Skip to content

R
rdiff-backup
Commit 1213096f authored by ben's avatar ben
Browse files
Options

Various changes to 0.9.3, see CHANGELOG 


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@157 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent 05dea566
Hide whitespace changes
Inline Side-by-side
Showing with 938 additions and 129 deletions
trunk/rdiff-backup/rdiff_backup/Globals.py
View file @ 1213096f
...@@ -85,9 +85,6 @@ isbackup_writer = None ...@@ -85,9 +85,6 @@ isbackup_writer = None
# Connection of the backup writer # Connection of the backup writer
backup_writer = None backup_writer = None
# True if this process is the client invoked by the user
isclient = None
# Connection of the client # Connection of the client
client_conn = None client_conn = None
...@@ -171,6 +168,22 @@ select_source, select_mirror = None, None ...@@ -171,6 +168,22 @@ select_source, select_mirror = None, None
# object. Access is provided to increment error counts. # object. Access is provided to increment error counts.
ITRB = None ITRB = None
# Percentage of time to spend sleeping. None means never sleep.
sleep_ratio = None
# security_level has 4 values and controls which requests from remote
# systems will be honored. "all" means anything goes. "read-only"
# means that the requests must not write to disk. "update-only" means
# that requests shouldn't destructively update the disk (but normal
# incremental updates are OK). "minimal" means only listen to a few
# basic requests.
security_level = "all"
# If this is set, it indicates that the remote connection should only
# deal with paths inside of restrict_path.
restrict_path = None
def get(name): def get(name):
"""Return the value of something in this module""" """Return the value of something in this module"""
return globals()[name] return globals()[name]
...@@ -199,6 +212,32 @@ def set_integer(name, val): ...@@ -199,6 +212,32 @@ def set_integer(name, val):
"received %s instead." % (name, val)) "received %s instead." % (name, val))
set(name, intval) set(name, intval)
def set_float(name, val, min = None, max = None, inclusive = 1):
"""Like set, but make sure val is float within given bounds"""
def error():
s = "Variable %s must be set to a float" % (name,)
if min is not None and max is not None:
s += " between %s and %s " % (min, max)
if inclusive: s += "inclusive"
else: s += "not inclusive"
elif min is not None or max is not None:
if inclusive: inclusive_string = "or equal to "
else: inclusive_string = ""
if min is not None:
s += " greater than %s%s" % (inclusive_string, min)
else: s+= " less than %s%s" % (inclusive_string, max)
Log.FatalError(s)
try: f = float(val)
except ValueError: error()
if min is not None:
if inclusive and f < min: error()
elif not inclusive and f <= min: error()
if max is not None:
if inclusive and f > max: error()
elif not inclusive and f >= max: error()
set(name, f)
def get_dict_val(name, key): def get_dict_val(name, key):
"""Return val from dictionary in this class""" """Return val from dictionary in this class"""
return globals()[name][key] return globals()[name][key]
......
trunk/rdiff-backup/rdiff_backup/Main.py
View file @ 1213096f
...@@ -44,16 +44,17 @@ def parse_cmdlineoptions(arglist): ...@@ -44,16 +44,17 @@ def parse_cmdlineoptions(arglist):
"checkpoint-interval=", "current-time=", "exclude=", "checkpoint-interval=", "current-time=", "exclude=",
"exclude-device-files", "exclude-filelist=", "exclude-device-files", "exclude-filelist=",
"exclude-filelist-stdin", "exclude-mirror=", "exclude-filelist-stdin", "exclude-mirror=",
"exclude-regexp=", "force", "include=", "exclude-other-filesystems", "exclude-regexp=", "force",
"include-filelist=", "include-filelist-stdin", "include=", "include-filelist=", "include-filelist-stdin",
"include-regexp=", "list-increments", "mirror-only", "include-regexp=", "list-increments", "mirror-only",
"no-compression", "no-compression-regexp=", "no-compression", "no-compression-regexp=", "no-hard-links",
"no-hard-links", "no-resume", "null-separator", "no-resume", "null-separator", "parsable-output",
"parsable-output", "print-statistics", "quoting-char=", "print-statistics", "quoting-char=", "remote-cmd=",
"remote-cmd=", "remote-schema=", "remove-older-than=", "remote-schema=", "remove-older-than=", "restore-as-of=",
"restore-as-of=", "resume", "resume-window=", "server", "restrict=", "restrict-read-only=", "restrict-update-only=",
"ssh-no-compression", "terminal-verbosity=", "resume", "resume-window=", "server", "sleep-ratio=",
"test-server", "verbosity", "version", "windows-mode", "ssh-no-compression", "terminal-verbosity=", "test-server",
"verbosity", "version", "windows-mode",
"windows-time-format"]) "windows-time-format"])
except getopt.error, e: except getopt.error, e:
commandline_error("Bad commandline options: %s" % str(e)) commandline_error("Bad commandline options: %s" % str(e))
...@@ -80,6 +81,8 @@ def parse_cmdlineoptions(arglist): ...@@ -80,6 +81,8 @@ def parse_cmdlineoptions(arglist):
select_files.append(sys.stdin) select_files.append(sys.stdin)
elif opt == "--exclude-mirror": elif opt == "--exclude-mirror":
select_mirror_opts.append(("--exclude", arg)) select_mirror_opts.append(("--exclude", arg))
elif opt == "--exclude-other-filesystems":
select_opts.append((opt, arg))
elif opt == "--exclude-regexp": select_opts.append((opt, arg)) elif opt == "--exclude-regexp": select_opts.append((opt, arg))
elif opt == "--force": force = 1 elif opt == "--force": force = 1
elif opt == "--include": select_opts.append((opt, arg)) elif opt == "--include": select_opts.append((opt, arg))
...@@ -99,23 +102,34 @@ def parse_cmdlineoptions(arglist): ...@@ -99,23 +102,34 @@ def parse_cmdlineoptions(arglist):
elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
elif opt == '--no-resume': Globals.resume = 0 elif opt == '--no-resume': Globals.resume = 0
elif opt == "--null-separator": Globals.set("null_separator", 1) elif opt == "--null-separator": Globals.set("null_separator", 1)
elif opt == "-r" or opt == "--restore-as-of":
restore_timestr, action = arg, "restore-as-of"
elif opt == "--parsable-output": Globals.set('parsable_output', 1) elif opt == "--parsable-output": Globals.set('parsable_output', 1)
elif opt == "--print-statistics": elif opt == "--print-statistics":
Globals.set('print_statistics', 1) Globals.set('print_statistics', 1)
elif opt == "--quoting-char": elif opt == "--quoting-char":
Globals.set('quoting_char', arg) Globals.set('quoting_char', arg)
Globals.set('quoting_enabled', 1) Globals.set('quoting_enabled', 1)
elif opt == "-r" or opt == "--restore-as-of":
restore_timestr, action = arg, "restore-as-of"
elif opt == "--remote-cmd": remote_cmd = arg elif opt == "--remote-cmd": remote_cmd = arg
elif opt == "--remote-schema": remote_schema = arg elif opt == "--remote-schema": remote_schema = arg
elif opt == "--remove-older-than": elif opt == "--remove-older-than":
remove_older_than_string = arg remove_older_than_string = arg
action = "remove-older-than" action = "remove-older-than"
elif opt == "--restrict": Globals.restrict_path = arg
elif opt == "--restrict-read-only":
Globals.security_level = "read-only"
Globals.restrict_path = arg
elif opt == "--restrict-update-only":
Globals.security_level = "update-only"
Globals.restrict_path = arg
elif opt == '--resume': Globals.resume = 1 elif opt == '--resume': Globals.resume = 1
elif opt == '--resume-window': elif opt == '--resume-window':
Globals.set_integer('resume_window', arg) Globals.set_integer('resume_window', arg)
elif opt == "-s" or opt == "--server": action = "server" elif opt == "-s" or opt == "--server":
action = "server"
Globals.server = 1
elif opt == "--sleep-ratio":
Globals.set_float("sleep_ratio", arg, 0, 1, inclusive=0)
elif opt == "--ssh-no-compression": elif opt == "--ssh-no-compression":
Globals.set('ssh_compression', None) Globals.set('ssh_compression', None)
elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg) elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
...@@ -176,7 +190,6 @@ def misc_setup(rps): ...@@ -176,7 +190,6 @@ def misc_setup(rps):
os.umask(077) os.umask(077)
Time.setcurtime(Globals.current_time) Time.setcurtime(Globals.current_time)
FilenameMapping.set_init_quote_vals() FilenameMapping.set_init_quote_vals()
Globals.set("isclient", 1)
SetConnections.UpdateGlobal("client_conn", Globals.local_connection) SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
# This is because I originally didn't think compiled regexps # This is because I originally didn't think compiled regexps
...@@ -209,7 +222,9 @@ def Main(arglist): ...@@ -209,7 +222,9 @@ def Main(arglist):
"""Start everything up!""" """Start everything up!"""
parse_cmdlineoptions(arglist) parse_cmdlineoptions(arglist)
set_action() set_action()
rps = SetConnections.InitRPs(args, remote_schema, remote_cmd) cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd)
Security.initialize(action, cmdpairs)
rps = map(SetConnections.cmdpair2rp, cmdpairs)
misc_setup(rps) misc_setup(rps)
take_action(rps) take_action(rps)
cleanup() cleanup()
...@@ -222,6 +237,7 @@ def Mirror(src_rp, dest_rp): ...@@ -222,6 +237,7 @@ def Mirror(src_rp, dest_rp):
# Since no "rdiff-backup-data" dir, use root of destination. # Since no "rdiff-backup-data" dir, use root of destination.
SetConnections.UpdateGlobal('rbdir', dest_rp) SetConnections.UpdateGlobal('rbdir', dest_rp)
SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn) SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn)
backup_init_select(src_rp, dest_rp)
HighLevel.Mirror(src_rp, dest_rp) HighLevel.Mirror(src_rp, dest_rp)
def mirror_check_paths(rpin, rpout): def mirror_check_paths(rpin, rpout):
...@@ -245,7 +261,7 @@ def Backup(rpin, rpout): ...@@ -245,7 +261,7 @@ def Backup(rpin, rpout):
Time.setprevtime(prevtime) Time.setprevtime(prevtime)
HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI) HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI)
else: HighLevel.Mirror(rpin, rpout, incdir, RSI) else: HighLevel.Mirror(rpin, rpout, incdir, RSI)
backup_touch_curmirror(rpin, rpout) rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout)
def backup_init_select(rpin, rpout): def backup_init_select(rpin, rpout):
"""Create Select objects on source and dest connections""" """Create Select objects on source and dest connections"""
...@@ -307,6 +323,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2) ...@@ -307,6 +323,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)
def backup_get_mirrorrps(): def backup_get_mirrorrps():
"""Return list of current_mirror rps""" """Return list of current_mirror rps"""
datadir = Globals.rbdir
if not datadir.isdir(): return [] if not datadir.isdir(): return []
mirrorrps = [datadir.append(fn) for fn in datadir.listdir() mirrorrps = [datadir.append(fn) for fn in datadir.listdir()
if fn.startswith("current_mirror.")] if fn.startswith("current_mirror.")]
...@@ -324,12 +341,14 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2) ...@@ -324,12 +341,14 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2)
timestr = mirrorrps[-1].getinctime() timestr = mirrorrps[-1].getinctime()
return Time.stringtotime(timestr) return Time.stringtotime(timestr)
def backup_touch_curmirror(rpin, rpout): def backup_touch_curmirror_local(rpin, rpout):
"""Make a file like current_mirror.time.data to record time """Make a file like current_mirror.time.data to record time
Also updates rpout so mod times don't get messed up. Also updates rpout so mod times don't get messed up. This should
be run on the destination connection.
""" """
datadir = Globals.rbdir
map(RPath.delete, backup_get_mirrorrps()) map(RPath.delete, backup_get_mirrorrps())
mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr, mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr,
"data")) "data"))
...@@ -337,7 +356,6 @@ def backup_touch_curmirror(rpin, rpout): ...@@ -337,7 +356,6 @@ def backup_touch_curmirror(rpin, rpout):
mirrorrp.touch() mirrorrp.touch()
RPath.copy_attribs(rpin, rpout) RPath.copy_attribs(rpin, rpout)
def restore(src_rp, dest_rp = None): def restore(src_rp, dest_rp = None):
"""Main restoring function """Main restoring function
...@@ -474,23 +492,24 @@ def RemoveOlderThan(rootrp): ...@@ -474,23 +492,24 @@ def RemoveOlderThan(rootrp):
(datadir.path,)) (datadir.path,))
try: time = Time.genstrtotime(remove_older_than_string) try: time = Time.genstrtotime(remove_older_than_string)
except TimeError, exc: Log.FatalError(str(exc)) except Time.TimeException, exc: Log.FatalError(str(exc))
timep = Time.timetopretty(time) timep = Time.timetopretty(time)
Log("Deleting increment(s) before %s" % timep, 4) Log("Deleting increment(s) before %s" % timep, 4)
itimes = [Time.stringtopretty(inc.getinctime()) times_in_secs = map(lambda inc: Time.stringtotime(inc.getinctime()),
for inc in Restore.get_inclist(datadir.append("increments")) Restore.get_inclist(datadir.append("increments")))
if Time.stringtotime(inc.getinctime()) < time] times_in_secs = filter(lambda t: t < time, times_in_secs)
if not times_in_secs:
if not itimes:
Log.FatalError("No increments older than %s found" % timep) Log.FatalError("No increments older than %s found" % timep)
inc_pretty_time = "\n".join(itimes)
if len(itimes) > 1 and not force: times_in_secs.sort()
inc_pretty_time = "\n".join(map(Time.timetopretty, times_in_secs))
if len(times_in_secs) > 1 and not force:
Log.FatalError("Found %d relevant increments, dated:\n%s" Log.FatalError("Found %d relevant increments, dated:\n%s"
"\nIf you want to delete multiple increments in this way, " "\nIf you want to delete multiple increments in this way, "
"use the --force." % (len(itimes), inc_pretty_time)) "use the --force." % (len(times_in_secs), inc_pretty_time))
Log("Deleting increment%sat times:\n%s" % Log("Deleting increment%sat times:\n%s" %
(len(itimes) == 1 and " " or "s ", inc_pretty_time), 3) (len(times_in_secs) == 1 and " " or "s ", inc_pretty_time), 3)
Manage.delete_earlier_than(datadir, time) Manage.delete_earlier_than(datadir, time)
trunk/rdiff-backup/rdiff_backup/Security.py 0 → 100644
View file @ 1213096f
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Functions to make sure remote requests are kosher"""
import sys
import Globals, tempfile
from rpath import *
class Violation(Exception):
"""Exception that indicates an improper request has been received"""
pass
# This will store the list of functions that will be honored from
# remote connections.
allowed_requests = None
# This stores the list of global variables that the client can not
# set on the server.
disallowed_server_globals = ["server", "security_level", "restrict_path"]
def initialize(action, cmdpairs):
"""Initialize allowable request list and chroot"""
global allowed_requests
set_security_level(action, cmdpairs)
set_allowed_requests(Globals.security_level)
def set_security_level(action, cmdpairs):
"""If running client, set security level and restrict_path
To find these settings, we must look at the action to see what is
supposed to happen, and then look at the cmdpairs to see what end
the client is on.
"""
def islocal(cmdpair): return not cmdpair[0]
def bothlocal(cp1, cp2): return islocal(cp1) and islocal(cp2)
def bothremote(cp1, cp2): return not islocal(cp1) and not islocal(cp2)
def getpath(cmdpair): return cmdpair[1]
if Globals.server: return
cp1 = cmdpairs[0]
if len(cmdpairs) > 1: cp2 = cmdpairs[1]
if action == "backup":
if bothlocal(cp1, cp2) or bothremote(cp1, cp2):
sec_level = "minimal"
rdir = tempfile.gettempdir()
elif islocal(cp1):
sec_level = "read-only"
rdir = getpath(cp1)
else:
assert islocal(cp2)
sec_level = "update-only"
rdir = getpath(cp2)
elif action == "restore" or action == "restore-as-of":
if len(cmdpairs) == 1 or bothlocal(cp1, cp2) or bothremote(cp1, cp2):
sec_level = "minimal"
rdir = tempfile.gettempdir()
elif islocal(cp1):
sec_level = "read-only"
else:
assert islocal(cp2)
sec_level = "all"
rdir = getpath(cp2)
elif action == "mirror":
if bothlocal(cp1, cp2) or bothremote(cp1, cp2):
sec_level = "minimal"
rdir = tempfile.gettempdir()
elif islocal(cp1):
sec_level = "read-only"
rdir = getpath(cp1)
else:
assert islocal(cp2)
sec_level = "all"
rdir = getpath(cp2)
elif (action == "test-server" or action == "list-increments" or
action == "calculate-average" or action == "remove-older-than"):
sec_level = "minimal"
rdir = tempfile.gettempdir()
else: assert 0, "Unknown action %s" % action
Globals.security_level = sec_level
Globals.restrict_path = rdir
def set_allowed_requests(sec_level):
"""Set the allowed requests list using the security level"""
global allowed_requests
if sec_level == "all": return
allowed_requests = ["VirtualFile.readfromid", "VirtualFile.closebyid",
"Globals.get", "Globals.is_not_None",
"Globals.get_dict_val",
"Log.open_logfile_allconn",
"Log.close_logfile_allconn",
"SetConnections.add_redirected_conn",
"RedirectedRun"]
if sec_level == "minimal": pass
elif sec_level == "read-only" or sec_level == "update-only":
allowed_requests.extend(["C.make_file_dict",
"os.getuid",
"os.listdir",
"Resume.ResumeCheck",
"HLSourceStruct.split_initial_dsiter",
"HLSourceStruct.get_diffs_and_finalize"])
if sec_level == "update-only":
allowed_requests. \
extend(["Log.open_logfile_local", "Log.close_logfile_local",
"Log.close_logfile_allconn", "Log.log_to_file",
"SaveState.init_filenames",
"SaveState.touch_last_file",
"HLDestinationStruct.get_sigs",
"HLDestinationStruct.patch_w_datadir_writes",
"HLDestinationStruct.patch_and_finalize",
"HLDestinationStruct.patch_increment_and_finalize",
"Main.backup_touch_curmirror_local",
"Globals.ITRB.increment_stat"])
if Globals.server:
allowed_requests.extend(["SetConnections.init_connection_remote",
"Log.setverbosity",
"Log.setterm_verbosity",
"Time.setcurtime_local",
"Time.setprevtime_local",
"FilenameMapping.set_init_quote_vals_local",
"Globals.postset_regexp_local",
"Globals.set_select",
"HLSourceStruct.set_session_info",
"HLDestinationStruct.set_session_info"])
def vet_request(request, arglist):
"""Examine request for security violations"""
#if Globals.server: sys.stderr.write(str(request) + "\n")
security_level = Globals.security_level
if Globals.restrict_path:
for arg in arglist:
if isinstance(arg, RPath): vet_rpath(arg)
if security_level == "all": return
if request.function_string in allowed_requests: return
if request.function_string == "Globals.set":
if Globals.server and arglist[0] not in disallowed_server_globals:
return
raise Violation("\nWarning Security Violation!\n"
"Bad request for function: %s\n"
"with arguments: %s\n" % (request.function_string,
arglist))
def vet_rpath(rpath):
"""Require rpath not to step outside retricted directory"""
if Globals.restrict_path and rpath.conn is Globals.local_connection:
normalized, restrict = rpath.normalize().path, Globals.restrict_path
components = normalized.split("/")
# 3 cases for restricted dir /usr/foo: /var, /usr/foobar, /usr/foo/..
if (not normalized.startswith(restrict) or
(len(normalized) > len(restrict) and
normalized[len(restrict)] != "/") or
".." in components):
raise Violation("\nWarning Security Violation!\n"
"Request to handle path %s\n"
"which doesn't appear to be within "
"restrict path %s.\n" % (normalized, restrict))
trunk/rdiff-backup/rdiff_backup/SetConnections.py
View file @ 1213096f
...@@ -28,8 +28,13 @@ __conn_remote_cmds = [None] ...@@ -28,8 +28,13 @@ __conn_remote_cmds = [None]
class SetConnectionsException(Exception): pass class SetConnectionsException(Exception): pass
def InitRPs(arglist, remote_schema = None, remote_cmd = None): def get_cmd_pairs(arglist, remote_schema = None, remote_cmd = None):
"""Map the given file descriptions into rpaths and return list""" """Map the given file descriptions into command pairs
Command pairs are tuples cmdpair with length 2. cmdpair[0] is
None iff it describes a local path, and cmdpair[1] is the path.
"""
global __cmd_schema global __cmd_schema
if remote_schema: __cmd_schema = remote_schema if remote_schema: __cmd_schema = remote_schema
elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress
...@@ -44,11 +49,10 @@ def InitRPs(arglist, remote_schema = None, remote_cmd = None): ...@@ -44,11 +49,10 @@ def InitRPs(arglist, remote_schema = None, remote_cmd = None):
elif remote_schema: elif remote_schema:
Log("Remote schema option ignored - no remote file " Log("Remote schema option ignored - no remote file "
"descriptions.", 2) "descriptions.", 2)
cmdpairs = map(desc2cmd_pairs, desc_pairs)
cmd_pairs = map(desc2cmd_pairs, desc_pairs)
if remote_cmd: # last file description gets remote_cmd if remote_cmd: # last file description gets remote_cmd
cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1]) cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1])
return map(cmdpair2rp, cmd_pairs) return cmdpairs
def cmdpair2rp(cmd_pair): def cmdpair2rp(cmd_pair):
"""Return RPath from cmd_pair (remote_cmd, filename)""" """Return RPath from cmd_pair (remote_cmd, filename)"""
......
trunk/rdiff-backup/rdiff_backup/Time.py
View file @ 1213096f
...@@ -25,17 +25,18 @@ _genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]" ...@@ -25,17 +25,18 @@ _genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
_genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]" _genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]"
"(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$") "(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$")
curtime = curtimestr = None curtime = curtimestr = None
been_awake_since = None # stores last time sleep() was run
def setcurtime(curtime = None): def setcurtime(curtime = None):
"""Sets the current time in curtime and curtimestr on all systems""" """Sets the current time in curtime and curtimestr on all systems"""
t = curtime or time.time() t = curtime or time.time()
for conn in Globals.connections: for conn in Globals.connections:
conn.Time.setcurtime_local(t, timetostring(t)) conn.Time.setcurtime_local(t)
def setcurtime_local(timeinseconds, timestr): def setcurtime_local(timeinseconds):
"""Only set the current time locally""" """Only set the current time locally"""
global curtime, curtimestr global curtime, curtimestr
curtime, curtimestr = timeinseconds, timestr curtime, curtimestr = timeinseconds, timetostring(timeinseconds)
def setprevtime(timeinseconds): def setprevtime(timeinseconds):
"""Sets the previous inc time in prevtime and prevtimestr""" """Sets the previous inc time in prevtime and prevtimestr"""
...@@ -168,6 +169,25 @@ def cmp(time1, time2): ...@@ -168,6 +169,25 @@ def cmp(time1, time2):
elif time1 == time2: return 0 elif time1 == time2: return 0
else: return 1 else: return 1
def sleep(sleep_ratio):
"""Sleep for period to maintain given sleep_ratio
On my system sleeping for periods less than 1/20th of a second
doesn't seem to work very accurately, so accumulate at least that
much time before sleeping.
"""
global been_awake_since
if been_awake_since is None: # first running
been_awake_since = time.time()
else:
elapsed_time = time.time() - been_awake_since
sleep_time = elapsed_time * (sleep_ratio/(1-sleep_ratio))
if sleep_time >= 0.05:
time.sleep(sleep_time)
been_awake_since = time.time()
def genstrtotime(timestr, curtime = None): def genstrtotime(timestr, curtime = None):
"""Convert a generic time string to a time in seconds""" """Convert a generic time string to a time in seconds"""
if curtime is None: curtime = globals()['curtime'] if curtime is None: curtime = globals()['curtime']
...@@ -203,5 +223,3 @@ the day).""" % timestr) ...@@ -203,5 +223,3 @@ the day).""" % timestr)
t = stringtotime(timestr) t = stringtotime(timestr)
if t: return t if t: return t
else: error() else: error()
trunk/rdiff-backup/rdiff_backup/cmodule.c
View file @ 1213096f
...@@ -49,7 +49,7 @@ static PyObject *c_make_file_dict(self, args) ...@@ -49,7 +49,7 @@ static PyObject *c_make_file_dict(self, args)
size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size); size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size);
inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino); inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino);
#else #else
size = PyInt_FromLong((long)sbuf.st_size); size = PyInt_FromLong(sbuf.st_size);
inode = PyInt_FromLong((long)sbuf.st_ino); inode = PyInt_FromLong((long)sbuf.st_ino);
#endif #endif
mode = (long)sbuf.st_mode; mode = (long)sbuf.st_mode;
...@@ -64,7 +64,7 @@ static PyObject *c_make_file_dict(self, args) ...@@ -64,7 +64,7 @@ static PyObject *c_make_file_dict(self, args)
atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime); atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime);
#else #else
mtime = PyInt_FromLong((long)sbuf.st_mtime); mtime = PyInt_FromLong((long)sbuf.st_mtime);
atime = PyLong_FromLongLong((long)sbuf.st_atime); atime = PyInt_FromLong((long)sbuf.st_atime);
#endif #endif
/* Build return dictionary from stat struct */ /* Build return dictionary from stat struct */
......
trunk/rdiff-backup/rdiff_backup/connection.py
View file @ 1213096f
...@@ -48,11 +48,9 @@ class LocalConnection(Connection): ...@@ -48,11 +48,9 @@ class LocalConnection(Connection):
elif isinstance(__builtins__, dict): return __builtins__[name] elif isinstance(__builtins__, dict): return __builtins__[name]
else: return __builtins__.__dict__[name] else: return __builtins__.__dict__[name]
def __setattr__(self, name, value): def __setattr__(self, name, value): globals()[name] = value
globals()[name] = value
def __delattr__(self, name): def __delattr__(self, name): del globals()[name]
del globals()[name]
def __str__(self): return "LocalConnection" def __str__(self): return "LocalConnection"
...@@ -329,7 +327,9 @@ class PipeConnection(LowLevelPipeConnection): ...@@ -329,7 +327,9 @@ class PipeConnection(LowLevelPipeConnection):
arg_req_num, arg = self._get() arg_req_num, arg = self._get()
assert arg_req_num == req_num assert arg_req_num == req_num
argument_list.append(arg) argument_list.append(arg)
try: result = apply(eval(request.function_string), argument_list) try:
Security.vet_request(request, argument_list)
result = apply(eval(request.function_string), argument_list)
except: result = self.extract_exception() except: result = self.extract_exception()
self._put(result, req_num) self._put(result, req_num)
self.unused_request_numbers[req_num] = None self.unused_request_numbers[req_num] = None
...@@ -407,14 +407,31 @@ class RedirectedConnection(Connection): ...@@ -407,14 +407,31 @@ class RedirectedConnection(Connection):
self.routing_number = routing_number self.routing_number = routing_number
self.routing_conn = Globals.connection_dict[routing_number] self.routing_conn = Globals.connection_dict[routing_number]
def reval(self, function_string, *args):
"""Evalution function_string on args on remote connection"""
return self.routing_conn.reval("RedirectedRun", self.conn_number,
function_string, *args)
def __str__(self): def __str__(self):
return "RedirectedConnection %d,%d" % (self.conn_number, return "RedirectedConnection %d,%d" % (self.conn_number,
self.routing_number) self.routing_number)
def __getattr__(self, name): def __getattr__(self, name):
return EmulateCallable(self.routing_conn, return EmulateCallableRedirected(self.conn_number, self.routing_conn,
"Globals.get_dict_val('connection_dict', %d).%s" name)
% (self.conn_number, name))
def RedirectedRun(conn_number, func, *args):
"""Run func with args on connection with conn number conn_number
This function is meant to redirect requests from one connection to
another, so conn_number must not be the local connection (and also
for security reasons since this function is always made
available).
"""
conn = Globals.connection_dict[conn_number]
assert conn is not Globals.local_connection, conn
return conn.reval(func, *args)
class EmulateCallable: class EmulateCallable:
...@@ -428,6 +445,18 @@ class EmulateCallable: ...@@ -428,6 +445,18 @@ class EmulateCallable:
return EmulateCallable(self.connection, return EmulateCallable(self.connection,
"%s.%s" % (self.name, attr_name)) "%s.%s" % (self.name, attr_name))
class EmulateCallableRedirected:
"""Used by RedirectedConnection in calls like conn.os.chmod(foo)"""
def __init__(self, conn_number, routing_conn, name):
self.conn_number, self.routing_conn = conn_number, routing_conn
self.name = name
def __call__(self, *args):
return apply(self.routing_conn.reval,
("RedirectedRun", self.conn_number, self.name) + args)
def __getattr__(self, attr_name):
return EmulateCallableRedirected(self.conn_number, self.routing_conn,
"%s.%s" % (self.name, attr_name))
class VirtualFile: class VirtualFile:
"""When the client asks for a file over the connection, it gets this """When the client asks for a file over the connection, it gets this
...@@ -499,7 +528,7 @@ class VirtualFile: ...@@ -499,7 +528,7 @@ class VirtualFile:
# everything has to be available here for remote connection's use, but # everything has to be available here for remote connection's use, but
# put at bottom to reduce circularities. # put at bottom to reduce circularities.
import Globals, Time, Rdiff, Hardlink, FilenameMapping, C import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, Main
from static import * from static import *
from lazy import * from lazy import *
from log import * from log import *
......
trunk/rdiff-backup/rdiff_backup/increment.py
View file @ 1213096f
...@@ -274,6 +274,7 @@ class IncrementITRB(StatsITRB): ...@@ -274,6 +274,7 @@ class IncrementITRB(StatsITRB):
def branch_process(self, branch): def branch_process(self, branch):
"""Update statistics, and the has_changed flag if change in branch""" """Update statistics, and the has_changed flag if change in branch"""
if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio)
if branch.changed: self.changed = 1 if branch.changed: self.changed = 1
self.add_file_stats(branch) self.add_file_stats(branch)
...@@ -288,7 +289,7 @@ class MirrorITRB(StatsITRB): ...@@ -288,7 +289,7 @@ class MirrorITRB(StatsITRB):
StatsITRB.__init__(self) StatsITRB.__init__(self)
def start_process(self, index, diff_rorp, mirror_dsrp): def start_process(self, index, diff_rorp, mirror_dsrp):
"""Initialize statistics, do actual writing to mirror""" """Initialize statistics and do actual writing to mirror"""
self.start_stats(mirror_dsrp) self.start_stats(mirror_dsrp)
if diff_rorp and not diff_rorp.isplaceholder(): if diff_rorp and not diff_rorp.isplaceholder():
RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute() RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute()
...@@ -312,6 +313,7 @@ class MirrorITRB(StatsITRB): ...@@ -312,6 +313,7 @@ class MirrorITRB(StatsITRB):
def branch_process(self, branch): def branch_process(self, branch):
"""Update statistics with subdirectory results""" """Update statistics with subdirectory results"""
if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio)
self.add_file_stats(branch) self.add_file_stats(branch)
......
trunk/rdiff-backup/rdiff_backup/rorpiter.py
View file @ 1213096f
...@@ -242,7 +242,9 @@ class RORPIter: ...@@ -242,7 +242,9 @@ class RORPIter:
def init(): Hardlink.link_rp(diff_rorp, tf, basisrp) def init(): Hardlink.link_rp(diff_rorp, tf, basisrp)
return Robust.make_tf_robustaction(init, tf, basisrp) return Robust.make_tf_robustaction(init, tf, basisrp)
elif basisrp and basisrp.isreg() and diff_rorp.isreg(): elif basisrp and basisrp.isreg() and diff_rorp.isreg():
assert diff_rorp.get_attached_filetype() == 'diff' if diff_rorp.get_attached_filetype() != 'diff':
raise RPathException("File %s appears to have changed during"
" processing, skipping" % (basisrp.path,))
return Rdiff.patch_with_attribs_action(basisrp, diff_rorp) return Rdiff.patch_with_attribs_action(basisrp, diff_rorp)
else: # Diff contains whole file, just copy it over else: # Diff contains whole file, just copy it over
if not basisrp: basisrp = base_rp.new_index(diff_rorp.index) if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
......
trunk/rdiff-backup/rdiff_backup/selection.py
View file @ 1213096f
...@@ -256,6 +256,8 @@ class Select: ...@@ -256,6 +256,8 @@ class Select:
self.add_selection_func(self.filelist_get_sf( self.add_selection_func(self.filelist_get_sf(
filelists[filelists_index], 0, arg)) filelists[filelists_index], 0, arg))
filelists_index += 1 filelists_index += 1
elif opt == "--exclude-other-filesystems":
self.add_selection_func(self.other_filesystems_get_sf(0))
elif opt == "--exclude-regexp": elif opt == "--exclude-regexp":
self.add_selection_func(self.regexp_get_sf(arg, 0)) self.add_selection_func(self.regexp_get_sf(arg, 0))
elif opt == "--include": elif opt == "--include":
...@@ -416,6 +418,17 @@ probably isn't what you meant.""" % ...@@ -416,6 +418,17 @@ probably isn't what you meant.""" %
else: return (None, None) # dsrp greater, not initial sequence else: return (None, None) # dsrp greater, not initial sequence
else: assert 0, "Include is %s, should be 0 or 1" % (include,) else: assert 0, "Include is %s, should be 0 or 1" % (include,)
def other_filesystems_get_sf(self, include):
"""Return selection function matching files on other filesystems"""
assert include == 0 or include == 1
root_devloc = self.dsrpath.getdevloc()
def sel_func(dsrp):
if dsrp.getdevloc() == root_devloc: return None
else: return include
sel_func.exclude = not include
sel_func.name = "Match other filesystems"
return sel_func
def regexp_get_sf(self, regexp_string, include): def regexp_get_sf(self, regexp_string, include):
"""Return selection function given by regexp_string""" """Return selection function given by regexp_string"""
assert include == 0 or include == 1 assert include == 0 or include == 1
......
trunk/rdiff-backup/rdiff_backup/statistics.py
View file @ 1213096f
...@@ -28,7 +28,7 @@ class StatsObj: ...@@ -28,7 +28,7 @@ class StatsObj:
'ChangedFiles', 'ChangedFiles',
'ChangedSourceSize', 'ChangedMirrorSize', 'ChangedSourceSize', 'ChangedMirrorSize',
'IncrementFiles', 'IncrementFileSize') 'IncrementFiles', 'IncrementFileSize')
stat_misc_attrs = ('Errors',) stat_misc_attrs = ('Errors', 'TotalDestinationSizeChange')
stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime') stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime')
stat_attrs = (('Filename',) + stat_time_attrs + stat_attrs = (('Filename',) + stat_time_attrs +
stat_misc_attrs + stat_file_attrs) stat_misc_attrs + stat_file_attrs)
...@@ -65,6 +65,26 @@ class StatsObj: ...@@ -65,6 +65,26 @@ class StatsObj:
"""Add 1 to value of attribute""" """Add 1 to value of attribute"""
self.__dict__[attr] += 1 self.__dict__[attr] += 1
def get_total_dest_size_change(self):
"""Return total destination size change
This represents the total change in the size of the
rdiff-backup destination directory.
"""
addvals = [self.NewFileSize, self.ChangedSourceSize,
self.IncrementFileSize]
subtractvals = [self.DeletedFileSize, self.ChangedMirrorSize]
for val in addvals + subtractvals:
if val is None:
result = None
break
else:
def addlist(l): return reduce(lambda x,y: x+y, l)
result = addlist(addvals) - addlist(subtractvals)
self.TotalDestinationSizeChange = result
return result
def get_stats_line(self, index, use_repr = 1): def get_stats_line(self, index, use_repr = 1):
"""Return one line abbreviated version of full stats string""" """Return one line abbreviated version of full stats string"""
file_attrs = map(lambda attr: str(self.get_stat(attr)), file_attrs = map(lambda attr: str(self.get_stat(attr)),
...@@ -95,7 +115,9 @@ class StatsObj: ...@@ -95,7 +115,9 @@ class StatsObj:
def get_stats_string(self): def get_stats_string(self):
"""Return extended string printing out statistics""" """Return extended string printing out statistics"""
return self.get_timestats_string() + self.get_filestats_string() return "%s%s%s" % (self.get_timestats_string(),
self.get_filestats_string(),
self.get_miscstats_string())
def get_timestats_string(self): def get_timestats_string(self):
"""Return portion of statistics string dealing with time""" """Return portion of statistics string dealing with time"""
...@@ -112,8 +134,6 @@ class StatsObj: ...@@ -112,8 +134,6 @@ class StatsObj:
self.ElapsedTime = self.EndTime - self.StartTime self.ElapsedTime = self.EndTime - self.StartTime
timelist.append("ElapsedTime %.2f (%s)\n" % timelist.append("ElapsedTime %.2f (%s)\n" %
(self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) (self.ElapsedTime, Time.inttopretty(self.ElapsedTime)))
if self.Errors is not None:
timelist.append("Errors %d\n" % self.Errors)
return "".join(timelist) return "".join(timelist)
def get_filestats_string(self): def get_filestats_string(self):
...@@ -130,8 +150,23 @@ class StatsObj: ...@@ -130,8 +150,23 @@ class StatsObj:
return "".join(map(fileline, self.stat_file_pairs)) return "".join(map(fileline, self.stat_file_pairs))
def get_miscstats_string(self):
"""Return portion of extended stat string about misc attributes"""
misc_string = ""
tdsc = self.get_total_dest_size_change()
if tdsc is not None:
misc_string += ("TotalDestinationSizeChange %s (%s)\n" %
(tdsc, self.get_byte_summary_string(tdsc)))
if self.Errors is not None: misc_string += "Errors %d\n" % self.Errors
return misc_string
def get_byte_summary_string(self, byte_count): def get_byte_summary_string(self, byte_count):
"""Turn byte count into human readable string like "7.23GB" """ """Turn byte count into human readable string like "7.23GB" """
if byte_count < 0:
sign = "-"
byte_count = -byte_count
else: sign = ""
for abbrev_bytes, abbrev_string in self.byte_abbrev_list: for abbrev_bytes, abbrev_string in self.byte_abbrev_list:
if byte_count >= abbrev_bytes: if byte_count >= abbrev_bytes:
# Now get 3 significant figures # Now get 3 significant figures
...@@ -139,11 +174,11 @@ class StatsObj: ...@@ -139,11 +174,11 @@ class StatsObj:
if abbrev_count >= 100: precision = 0 if abbrev_count >= 100: precision = 0
elif abbrev_count >= 10: precision = 1 elif abbrev_count >= 10: precision = 1
else: precision = 2 else: precision = 2
return "%%.%df %s" % (precision, abbrev_string) \ return "%s%%.%df %s" % (sign, precision, abbrev_string) \
% (abbrev_count,) % (abbrev_count,)
byte_count = round(byte_count) byte_count = round(byte_count)
if byte_count == 1: return "1 byte" if byte_count == 1: return sign + "1 byte"
else: return "%d bytes" % (byte_count,) else: return "%s%d bytes" % (sign, byte_count)
def get_stats_logstring(self, title): def get_stats_logstring(self, title):
"""Like get_stats_string, but add header and footer""" """Like get_stats_string, but add header and footer"""
......
trunk/rdiff-backup/src/Globals.py
View file @ 1213096f
...@@ -85,9 +85,6 @@ isbackup_writer = None ...@@ -85,9 +85,6 @@ isbackup_writer = None
# Connection of the backup writer # Connection of the backup writer
backup_writer = None backup_writer = None
# True if this process is the client invoked by the user
isclient = None
# Connection of the client # Connection of the client
client_conn = None client_conn = None
...@@ -171,6 +168,22 @@ select_source, select_mirror = None, None ...@@ -171,6 +168,22 @@ select_source, select_mirror = None, None
# object. Access is provided to increment error counts. # object. Access is provided to increment error counts.
ITRB = None ITRB = None
# Percentage of time to spend sleeping. None means never sleep.
sleep_ratio = None
# security_level has 4 values and controls which requests from remote
# systems will be honored. "all" means anything goes. "read-only"
# means that the requests must not write to disk. "update-only" means
# that requests shouldn't destructively update the disk (but normal
# incremental updates are OK). "minimal" means only listen to a few
# basic requests.
security_level = "all"
# If this is set, it indicates that the remote connection should only
# deal with paths inside of restrict_path.
restrict_path = None
def get(name): def get(name):
"""Return the value of something in this module""" """Return the value of something in this module"""
return globals()[name] return globals()[name]
...@@ -199,6 +212,32 @@ def set_integer(name, val): ...@@ -199,6 +212,32 @@ def set_integer(name, val):
"received %s instead." % (name, val)) "received %s instead." % (name, val))
set(name, intval) set(name, intval)
def set_float(name, val, min = None, max = None, inclusive = 1):
"""Like set, but make sure val is float within given bounds"""
def error():
s = "Variable %s must be set to a float" % (name,)
if min is not None and max is not None:
s += " between %s and %s " % (min, max)
if inclusive: s += "inclusive"
else: s += "not inclusive"
elif min is not None or max is not None:
if inclusive: inclusive_string = "or equal to "
else: inclusive_string = ""
if min is not None:
s += " greater than %s%s" % (inclusive_string, min)
else: s+= " less than %s%s" % (inclusive_string, max)
Log.FatalError(s)
try: f = float(val)
except ValueError: error()
if min is not None:
if inclusive and f < min: error()
elif not inclusive and f <= min: error()
if max is not None:
if inclusive and f > max: error()
elif not inclusive and f >= max: error()
set(name, f)
def get_dict_val(name, key): def get_dict_val(name, key):
"""Return val from dictionary in this class""" """Return val from dictionary in this class"""
return globals()[name][key] return globals()[name][key]
......
trunk/rdiff-backup/src/Main.py
View file @ 1213096f
...@@ -44,16 +44,17 @@ def parse_cmdlineoptions(arglist): ...@@ -44,16 +44,17 @@ def parse_cmdlineoptions(arglist):
"checkpoint-interval=", "current-time=", "exclude=", "checkpoint-interval=", "current-time=", "exclude=",
"exclude-device-files", "exclude-filelist=", "exclude-device-files", "exclude-filelist=",
"exclude-filelist-stdin", "exclude-mirror=", "exclude-filelist-stdin", "exclude-mirror=",
"exclude-regexp=", "force", "include=", "exclude-other-filesystems", "exclude-regexp=", "force",
"include-filelist=", "include-filelist-stdin", "include=", "include-filelist=", "include-filelist-stdin",
"include-regexp=", "list-increments", "mirror-only", "include-regexp=", "list-increments", "mirror-only",
"no-compression", "no-compression-regexp=", "no-compression", "no-compression-regexp=", "no-hard-links",
"no-hard-links", "no-resume", "null-separator", "no-resume", "null-separator", "parsable-output",
"parsable-output", "print-statistics", "quoting-char=", "print-statistics", "quoting-char=", "remote-cmd=",
"remote-cmd=", "remote-schema=", "remove-older-than=", "remote-schema=", "remove-older-than=", "restore-as-of=",
"restore-as-of=", "resume", "resume-window=", "server", "restrict=", "restrict-read-only=", "restrict-update-only=",
"ssh-no-compression", "terminal-verbosity=", "resume", "resume-window=", "server", "sleep-ratio=",
"test-server", "verbosity", "version", "windows-mode", "ssh-no-compression", "terminal-verbosity=", "test-server",
"verbosity", "version", "windows-mode",
"windows-time-format"]) "windows-time-format"])
except getopt.error, e: except getopt.error, e:
commandline_error("Bad commandline options: %s" % str(e)) commandline_error("Bad commandline options: %s" % str(e))
...@@ -80,6 +81,8 @@ def parse_cmdlineoptions(arglist): ...@@ -80,6 +81,8 @@ def parse_cmdlineoptions(arglist):
select_files.append(sys.stdin) select_files.append(sys.stdin)
elif opt == "--exclude-mirror": elif opt == "--exclude-mirror":
select_mirror_opts.append(("--exclude", arg)) select_mirror_opts.append(("--exclude", arg))
elif opt == "--exclude-other-filesystems":
select_opts.append((opt, arg))
elif opt == "--exclude-regexp": select_opts.append((opt, arg)) elif opt == "--exclude-regexp": select_opts.append((opt, arg))
elif opt == "--force": force = 1 elif opt == "--force": force = 1
elif opt == "--include": select_opts.append((opt, arg)) elif opt == "--include": select_opts.append((opt, arg))
...@@ -99,23 +102,34 @@ def parse_cmdlineoptions(arglist): ...@@ -99,23 +102,34 @@ def parse_cmdlineoptions(arglist):
elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
elif opt == '--no-resume': Globals.resume = 0 elif opt == '--no-resume': Globals.resume = 0
elif opt == "--null-separator": Globals.set("null_separator", 1) elif opt == "--null-separator": Globals.set("null_separator", 1)
elif opt == "-r" or opt == "--restore-as-of":
restore_timestr, action = arg, "restore-as-of"
elif opt == "--parsable-output": Globals.set('parsable_output', 1) elif opt == "--parsable-output": Globals.set('parsable_output', 1)
elif opt == "--print-statistics": elif opt == "--print-statistics":
Globals.set('print_statistics', 1) Globals.set('print_statistics', 1)
elif opt == "--quoting-char": elif opt == "--quoting-char":
Globals.set('quoting_char', arg) Globals.set('quoting_char', arg)
Globals.set('quoting_enabled', 1) Globals.set('quoting_enabled', 1)
elif opt == "-r" or opt == "--restore-as-of":
restore_timestr, action = arg, "restore-as-of"
elif opt == "--remote-cmd": remote_cmd = arg elif opt == "--remote-cmd": remote_cmd = arg
elif opt == "--remote-schema": remote_schema = arg elif opt == "--remote-schema": remote_schema = arg
elif opt == "--remove-older-than": elif opt == "--remove-older-than":
remove_older_than_string = arg remove_older_than_string = arg
action = "remove-older-than" action = "remove-older-than"
elif opt == "--restrict": Globals.restrict_path = arg
elif opt == "--restrict-read-only":
Globals.security_level = "read-only"
Globals.restrict_path = arg
elif opt == "--restrict-update-only":
Globals.security_level = "update-only"
Globals.restrict_path = arg
elif opt == '--resume': Globals.resume = 1 elif opt == '--resume': Globals.resume = 1
elif opt == '--resume-window': elif opt == '--resume-window':
Globals.set_integer('resume_window', arg) Globals.set_integer('resume_window', arg)
elif opt == "-s" or opt == "--server": action = "server" elif opt == "-s" or opt == "--server":
action = "server"
Globals.server = 1
elif opt == "--sleep-ratio":
Globals.set_float("sleep_ratio", arg, 0, 1, inclusive=0)
elif opt == "--ssh-no-compression": elif opt == "--ssh-no-compression":
Globals.set('ssh_compression', None) Globals.set('ssh_compression', None)
elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg) elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
...@@ -176,7 +190,6 @@ def misc_setup(rps): ...@@ -176,7 +190,6 @@ def misc_setup(rps):
os.umask(077) os.umask(077)
Time.setcurtime(Globals.current_time) Time.setcurtime(Globals.current_time)
FilenameMapping.set_init_quote_vals() FilenameMapping.set_init_quote_vals()
Globals.set("isclient", 1)
SetConnections.UpdateGlobal("client_conn", Globals.local_connection) SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
# This is because I originally didn't think compiled regexps # This is because I originally didn't think compiled regexps
...@@ -209,7 +222,9 @@ def Main(arglist): ...@@ -209,7 +222,9 @@ def Main(arglist):
"""Start everything up!""" """Start everything up!"""
parse_cmdlineoptions(arglist) parse_cmdlineoptions(arglist)
set_action() set_action()
rps = SetConnections.InitRPs(args, remote_schema, remote_cmd) cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd)
Security.initialize(action, cmdpairs)
rps = map(SetConnections.cmdpair2rp, cmdpairs)
misc_setup(rps) misc_setup(rps)
take_action(rps) take_action(rps)
cleanup() cleanup()
...@@ -222,6 +237,7 @@ def Mirror(src_rp, dest_rp): ...@@ -222,6 +237,7 @@ def Mirror(src_rp, dest_rp):
# Since no "rdiff-backup-data" dir, use root of destination. # Since no "rdiff-backup-data" dir, use root of destination.
SetConnections.UpdateGlobal('rbdir', dest_rp) SetConnections.UpdateGlobal('rbdir', dest_rp)
SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn) SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn)
backup_init_select(src_rp, dest_rp)
HighLevel.Mirror(src_rp, dest_rp) HighLevel.Mirror(src_rp, dest_rp)
def mirror_check_paths(rpin, rpout): def mirror_check_paths(rpin, rpout):
...@@ -245,7 +261,7 @@ def Backup(rpin, rpout): ...@@ -245,7 +261,7 @@ def Backup(rpin, rpout):
Time.setprevtime(prevtime) Time.setprevtime(prevtime)
HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI) HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI)
else: HighLevel.Mirror(rpin, rpout, incdir, RSI) else: HighLevel.Mirror(rpin, rpout, incdir, RSI)
backup_touch_curmirror(rpin, rpout) rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout)
def backup_init_select(rpin, rpout): def backup_init_select(rpin, rpout):
"""Create Select objects on source and dest connections""" """Create Select objects on source and dest connections"""
...@@ -307,6 +323,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2) ...@@ -307,6 +323,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)
def backup_get_mirrorrps(): def backup_get_mirrorrps():
"""Return list of current_mirror rps""" """Return list of current_mirror rps"""
datadir = Globals.rbdir
if not datadir.isdir(): return [] if not datadir.isdir(): return []
mirrorrps = [datadir.append(fn) for fn in datadir.listdir() mirrorrps = [datadir.append(fn) for fn in datadir.listdir()
if fn.startswith("current_mirror.")] if fn.startswith("current_mirror.")]
...@@ -324,12 +341,14 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2) ...@@ -324,12 +341,14 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2)
timestr = mirrorrps[-1].getinctime() timestr = mirrorrps[-1].getinctime()
return Time.stringtotime(timestr) return Time.stringtotime(timestr)
def backup_touch_curmirror(rpin, rpout): def backup_touch_curmirror_local(rpin, rpout):
"""Make a file like current_mirror.time.data to record time """Make a file like current_mirror.time.data to record time
Also updates rpout so mod times don't get messed up. Also updates rpout so mod times don't get messed up. This should
be run on the destination connection.
""" """
datadir = Globals.rbdir
map(RPath.delete, backup_get_mirrorrps()) map(RPath.delete, backup_get_mirrorrps())
mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr, mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr,
"data")) "data"))
...@@ -337,7 +356,6 @@ def backup_touch_curmirror(rpin, rpout): ...@@ -337,7 +356,6 @@ def backup_touch_curmirror(rpin, rpout):
mirrorrp.touch() mirrorrp.touch()
RPath.copy_attribs(rpin, rpout) RPath.copy_attribs(rpin, rpout)
def restore(src_rp, dest_rp = None): def restore(src_rp, dest_rp = None):
"""Main restoring function """Main restoring function
...@@ -474,23 +492,24 @@ def RemoveOlderThan(rootrp): ...@@ -474,23 +492,24 @@ def RemoveOlderThan(rootrp):
(datadir.path,)) (datadir.path,))
try: time = Time.genstrtotime(remove_older_than_string) try: time = Time.genstrtotime(remove_older_than_string)
except TimeError, exc: Log.FatalError(str(exc)) except Time.TimeException, exc: Log.FatalError(str(exc))
timep = Time.timetopretty(time) timep = Time.timetopretty(time)
Log("Deleting increment(s) before %s" % timep, 4) Log("Deleting increment(s) before %s" % timep, 4)
itimes = [Time.stringtopretty(inc.getinctime()) times_in_secs = map(lambda inc: Time.stringtotime(inc.getinctime()),
for inc in Restore.get_inclist(datadir.append("increments")) Restore.get_inclist(datadir.append("increments")))
if Time.stringtotime(inc.getinctime()) < time] times_in_secs = filter(lambda t: t < time, times_in_secs)
if not times_in_secs:
if not itimes:
Log.FatalError("No increments older than %s found" % timep) Log.FatalError("No increments older than %s found" % timep)
inc_pretty_time = "\n".join(itimes)
if len(itimes) > 1 and not force: times_in_secs.sort()
inc_pretty_time = "\n".join(map(Time.timetopretty, times_in_secs))
if len(times_in_secs) > 1 and not force:
Log.FatalError("Found %d relevant increments, dated:\n%s" Log.FatalError("Found %d relevant increments, dated:\n%s"
"\nIf you want to delete multiple increments in this way, " "\nIf you want to delete multiple increments in this way, "
"use the --force." % (len(itimes), inc_pretty_time)) "use the --force." % (len(times_in_secs), inc_pretty_time))
Log("Deleting increment%sat times:\n%s" % Log("Deleting increment%sat times:\n%s" %
(len(itimes) == 1 and " " or "s ", inc_pretty_time), 3) (len(times_in_secs) == 1 and " " or "s ", inc_pretty_time), 3)
Manage.delete_earlier_than(datadir, time) Manage.delete_earlier_than(datadir, time)
trunk/rdiff-backup/src/Security.py 0 → 100644
View file @ 1213096f
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Functions to make sure remote requests are kosher"""
import sys
import Globals, tempfile
from rpath import *
class Violation(Exception):
"""Exception that indicates an improper request has been received"""
pass
# This will store the list of functions that will be honored from
# remote connections.
allowed_requests = None
# This stores the list of global variables that the client can not
# set on the server.
disallowed_server_globals = ["server", "security_level", "restrict_path"]
def initialize(action, cmdpairs):
"""Initialize allowable request list and chroot"""
global allowed_requests
set_security_level(action, cmdpairs)
set_allowed_requests(Globals.security_level)
def set_security_level(action, cmdpairs):
"""If running client, set security level and restrict_path
To find these settings, we must look at the action to see what is
supposed to happen, and then look at the cmdpairs to see what end
the client is on.
"""
def islocal(cmdpair): return not cmdpair[0]
def bothlocal(cp1, cp2): return islocal(cp1) and islocal(cp2)
def bothremote(cp1, cp2): return not islocal(cp1) and not islocal(cp2)
def getpath(cmdpair): return cmdpair[1]
if Globals.server: return
cp1 = cmdpairs[0]
if len(cmdpairs) > 1: cp2 = cmdpairs[1]
if action == "backup":
if bothlocal(cp1, cp2) or bothremote(cp1, cp2):
sec_level = "minimal"
rdir = tempfile.gettempdir()
elif islocal(cp1):
sec_level = "read-only"
rdir = getpath(cp1)
else:
assert islocal(cp2)
sec_level = "update-only"
rdir = getpath(cp2)
elif action == "restore" or action == "restore-as-of":
if len(cmdpairs) == 1 or bothlocal(cp1, cp2) or bothremote(cp1, cp2):
sec_level = "minimal"
rdir = tempfile.gettempdir()
elif islocal(cp1):
sec_level = "read-only"
else:
assert islocal(cp2)
sec_level = "all"
rdir = getpath(cp2)
elif action == "mirror":
if bothlocal(cp1, cp2) or bothremote(cp1, cp2):
sec_level = "minimal"
rdir = tempfile.gettempdir()
elif islocal(cp1):
sec_level = "read-only"
rdir = getpath(cp1)
else:
assert islocal(cp2)
sec_level = "all"
rdir = getpath(cp2)
elif (action == "test-server" or action == "list-increments" or
action == "calculate-average" or action == "remove-older-than"):
sec_level = "minimal"
rdir = tempfile.gettempdir()
else: assert 0, "Unknown action %s" % action
Globals.security_level = sec_level
Globals.restrict_path = rdir
def set_allowed_requests(sec_level):
"""Set the allowed requests list using the security level"""
global allowed_requests
if sec_level == "all": return
allowed_requests = ["VirtualFile.readfromid", "VirtualFile.closebyid",
"Globals.get", "Globals.is_not_None",
"Globals.get_dict_val",
"Log.open_logfile_allconn",
"Log.close_logfile_allconn",
"SetConnections.add_redirected_conn",
"RedirectedRun"]
if sec_level == "minimal": pass
elif sec_level == "read-only" or sec_level == "update-only":
allowed_requests.extend(["C.make_file_dict",
"os.getuid",
"os.listdir",
"Resume.ResumeCheck",
"HLSourceStruct.split_initial_dsiter",
"HLSourceStruct.get_diffs_and_finalize"])
if sec_level == "update-only":
allowed_requests. \
extend(["Log.open_logfile_local", "Log.close_logfile_local",
"Log.close_logfile_allconn", "Log.log_to_file",
"SaveState.init_filenames",
"SaveState.touch_last_file",
"HLDestinationStruct.get_sigs",
"HLDestinationStruct.patch_w_datadir_writes",
"HLDestinationStruct.patch_and_finalize",
"HLDestinationStruct.patch_increment_and_finalize",
"Main.backup_touch_curmirror_local",
"Globals.ITRB.increment_stat"])
if Globals.server:
allowed_requests.extend(["SetConnections.init_connection_remote",
"Log.setverbosity",
"Log.setterm_verbosity",
"Time.setcurtime_local",
"Time.setprevtime_local",
"FilenameMapping.set_init_quote_vals_local",
"Globals.postset_regexp_local",
"Globals.set_select",
"HLSourceStruct.set_session_info",
"HLDestinationStruct.set_session_info"])
def vet_request(request, arglist):
"""Examine request for security violations"""
#if Globals.server: sys.stderr.write(str(request) + "\n")
security_level = Globals.security_level
if Globals.restrict_path:
for arg in arglist:
if isinstance(arg, RPath): vet_rpath(arg)
if security_level == "all": return
if request.function_string in allowed_requests: return
if request.function_string == "Globals.set":
if Globals.server and arglist[0] not in disallowed_server_globals:
return
raise Violation("\nWarning Security Violation!\n"
"Bad request for function: %s\n"
"with arguments: %s\n" % (request.function_string,
arglist))
def vet_rpath(rpath):
"""Require rpath not to step outside retricted directory"""
if Globals.restrict_path and rpath.conn is Globals.local_connection:
normalized, restrict = rpath.normalize().path, Globals.restrict_path
components = normalized.split("/")
# 3 cases for restricted dir /usr/foo: /var, /usr/foobar, /usr/foo/..
if (not normalized.startswith(restrict) or
(len(normalized) > len(restrict) and
normalized[len(restrict)] != "/") or
".." in components):
raise Violation("\nWarning Security Violation!\n"
"Request to handle path %s\n"
"which doesn't appear to be within "
"restrict path %s.\n" % (normalized, restrict))
trunk/rdiff-backup/src/SetConnections.py
View file @ 1213096f
...@@ -28,8 +28,13 @@ __conn_remote_cmds = [None] ...@@ -28,8 +28,13 @@ __conn_remote_cmds = [None]
class SetConnectionsException(Exception): pass class SetConnectionsException(Exception): pass
def InitRPs(arglist, remote_schema = None, remote_cmd = None): def get_cmd_pairs(arglist, remote_schema = None, remote_cmd = None):
"""Map the given file descriptions into rpaths and return list""" """Map the given file descriptions into command pairs
Command pairs are tuples cmdpair with length 2. cmdpair[0] is
None iff it describes a local path, and cmdpair[1] is the path.
"""
global __cmd_schema global __cmd_schema
if remote_schema: __cmd_schema = remote_schema if remote_schema: __cmd_schema = remote_schema
elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress
...@@ -44,11 +49,10 @@ def InitRPs(arglist, remote_schema = None, remote_cmd = None): ...@@ -44,11 +49,10 @@ def InitRPs(arglist, remote_schema = None, remote_cmd = None):
elif remote_schema: elif remote_schema:
Log("Remote schema option ignored - no remote file " Log("Remote schema option ignored - no remote file "
"descriptions.", 2) "descriptions.", 2)
cmdpairs = map(desc2cmd_pairs, desc_pairs)
cmd_pairs = map(desc2cmd_pairs, desc_pairs)
if remote_cmd: # last file description gets remote_cmd if remote_cmd: # last file description gets remote_cmd
cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1]) cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1])
return map(cmdpair2rp, cmd_pairs) return cmdpairs
def cmdpair2rp(cmd_pair): def cmdpair2rp(cmd_pair):
"""Return RPath from cmd_pair (remote_cmd, filename)""" """Return RPath from cmd_pair (remote_cmd, filename)"""
......
trunk/rdiff-backup/src/Time.py
View file @ 1213096f
...@@ -25,17 +25,18 @@ _genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]" ...@@ -25,17 +25,18 @@ _genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
_genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]" _genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]"
"(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$") "(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$")
curtime = curtimestr = None curtime = curtimestr = None
been_awake_since = None # stores last time sleep() was run
def setcurtime(curtime = None): def setcurtime(curtime = None):
"""Sets the current time in curtime and curtimestr on all systems""" """Sets the current time in curtime and curtimestr on all systems"""
t = curtime or time.time() t = curtime or time.time()
for conn in Globals.connections: for conn in Globals.connections:
conn.Time.setcurtime_local(t, timetostring(t)) conn.Time.setcurtime_local(t)
def setcurtime_local(timeinseconds, timestr): def setcurtime_local(timeinseconds):
"""Only set the current time locally""" """Only set the current time locally"""
global curtime, curtimestr global curtime, curtimestr
curtime, curtimestr = timeinseconds, timestr curtime, curtimestr = timeinseconds, timetostring(timeinseconds)
def setprevtime(timeinseconds): def setprevtime(timeinseconds):
"""Sets the previous inc time in prevtime and prevtimestr""" """Sets the previous inc time in prevtime and prevtimestr"""
...@@ -168,6 +169,25 @@ def cmp(time1, time2): ...@@ -168,6 +169,25 @@ def cmp(time1, time2):
elif time1 == time2: return 0 elif time1 == time2: return 0
else: return 1 else: return 1
def sleep(sleep_ratio):
"""Sleep for period to maintain given sleep_ratio
On my system sleeping for periods less than 1/20th of a second
doesn't seem to work very accurately, so accumulate at least that
much time before sleeping.
"""
global been_awake_since
if been_awake_since is None: # first running
been_awake_since = time.time()
else:
elapsed_time = time.time() - been_awake_since
sleep_time = elapsed_time * (sleep_ratio/(1-sleep_ratio))
if sleep_time >= 0.05:
time.sleep(sleep_time)
been_awake_since = time.time()
def genstrtotime(timestr, curtime = None): def genstrtotime(timestr, curtime = None):
"""Convert a generic time string to a time in seconds""" """Convert a generic time string to a time in seconds"""
if curtime is None: curtime = globals()['curtime'] if curtime is None: curtime = globals()['curtime']
...@@ -203,5 +223,3 @@ the day).""" % timestr) ...@@ -203,5 +223,3 @@ the day).""" % timestr)
t = stringtotime(timestr) t = stringtotime(timestr)
if t: return t if t: return t
else: error() else: error()
trunk/rdiff-backup/src/cmodule.c
View file @ 1213096f
...@@ -49,7 +49,7 @@ static PyObject *c_make_file_dict(self, args) ...@@ -49,7 +49,7 @@ static PyObject *c_make_file_dict(self, args)
size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size); size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size);
inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino); inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino);
#else #else
size = PyInt_FromLong((long)sbuf.st_size); size = PyInt_FromLong(sbuf.st_size);
inode = PyInt_FromLong((long)sbuf.st_ino); inode = PyInt_FromLong((long)sbuf.st_ino);
#endif #endif
mode = (long)sbuf.st_mode; mode = (long)sbuf.st_mode;
...@@ -64,7 +64,7 @@ static PyObject *c_make_file_dict(self, args) ...@@ -64,7 +64,7 @@ static PyObject *c_make_file_dict(self, args)
atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime); atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime);
#else #else
mtime = PyInt_FromLong((long)sbuf.st_mtime); mtime = PyInt_FromLong((long)sbuf.st_mtime);
atime = PyLong_FromLongLong((long)sbuf.st_atime); atime = PyInt_FromLong((long)sbuf.st_atime);
#endif #endif
/* Build return dictionary from stat struct */ /* Build return dictionary from stat struct */
......
trunk/rdiff-backup/src/connection.py
View file @ 1213096f
...@@ -48,11 +48,9 @@ class LocalConnection(Connection): ...@@ -48,11 +48,9 @@ class LocalConnection(Connection):
elif isinstance(__builtins__, dict): return __builtins__[name] elif isinstance(__builtins__, dict): return __builtins__[name]
else: return __builtins__.__dict__[name] else: return __builtins__.__dict__[name]
def __setattr__(self, name, value): def __setattr__(self, name, value): globals()[name] = value
globals()[name] = value
def __delattr__(self, name): def __delattr__(self, name): del globals()[name]
del globals()[name]
def __str__(self): return "LocalConnection" def __str__(self): return "LocalConnection"
...@@ -329,7 +327,9 @@ class PipeConnection(LowLevelPipeConnection): ...@@ -329,7 +327,9 @@ class PipeConnection(LowLevelPipeConnection):
arg_req_num, arg = self._get() arg_req_num, arg = self._get()
assert arg_req_num == req_num assert arg_req_num == req_num
argument_list.append(arg) argument_list.append(arg)
try: result = apply(eval(request.function_string), argument_list) try:
Security.vet_request(request, argument_list)
result = apply(eval(request.function_string), argument_list)
except: result = self.extract_exception() except: result = self.extract_exception()
self._put(result, req_num) self._put(result, req_num)
self.unused_request_numbers[req_num] = None self.unused_request_numbers[req_num] = None
...@@ -407,14 +407,31 @@ class RedirectedConnection(Connection): ...@@ -407,14 +407,31 @@ class RedirectedConnection(Connection):
self.routing_number = routing_number self.routing_number = routing_number
self.routing_conn = Globals.connection_dict[routing_number] self.routing_conn = Globals.connection_dict[routing_number]
def reval(self, function_string, *args):
"""Evalution function_string on args on remote connection"""
return self.routing_conn.reval("RedirectedRun", self.conn_number,
function_string, *args)
def __str__(self): def __str__(self):
return "RedirectedConnection %d,%d" % (self.conn_number, return "RedirectedConnection %d,%d" % (self.conn_number,
self.routing_number) self.routing_number)
def __getattr__(self, name): def __getattr__(self, name):
return EmulateCallable(self.routing_conn, return EmulateCallableRedirected(self.conn_number, self.routing_conn,
"Globals.get_dict_val('connection_dict', %d).%s" name)
% (self.conn_number, name))
def RedirectedRun(conn_number, func, *args):
"""Run func with args on connection with conn number conn_number
This function is meant to redirect requests from one connection to
another, so conn_number must not be the local connection (and also
for security reasons since this function is always made
available).
"""
conn = Globals.connection_dict[conn_number]
assert conn is not Globals.local_connection, conn
return conn.reval(func, *args)
class EmulateCallable: class EmulateCallable:
...@@ -428,6 +445,18 @@ class EmulateCallable: ...@@ -428,6 +445,18 @@ class EmulateCallable:
return EmulateCallable(self.connection, return EmulateCallable(self.connection,
"%s.%s" % (self.name, attr_name)) "%s.%s" % (self.name, attr_name))
class EmulateCallableRedirected:
"""Used by RedirectedConnection in calls like conn.os.chmod(foo)"""
def __init__(self, conn_number, routing_conn, name):
self.conn_number, self.routing_conn = conn_number, routing_conn
self.name = name
def __call__(self, *args):
return apply(self.routing_conn.reval,
("RedirectedRun", self.conn_number, self.name) + args)
def __getattr__(self, attr_name):
return EmulateCallableRedirected(self.conn_number, self.routing_conn,
"%s.%s" % (self.name, attr_name))
class VirtualFile: class VirtualFile:
"""When the client asks for a file over the connection, it gets this """When the client asks for a file over the connection, it gets this
...@@ -499,7 +528,7 @@ class VirtualFile: ...@@ -499,7 +528,7 @@ class VirtualFile:
# everything has to be available here for remote connection's use, but # everything has to be available here for remote connection's use, but
# put at bottom to reduce circularities. # put at bottom to reduce circularities.
import Globals, Time, Rdiff, Hardlink, FilenameMapping, C import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, Main
from static import * from static import *
from lazy import * from lazy import *
from log import * from log import *
......
trunk/rdiff-backup/src/increment.py
View file @ 1213096f
...@@ -274,6 +274,7 @@ class IncrementITRB(StatsITRB): ...@@ -274,6 +274,7 @@ class IncrementITRB(StatsITRB):
def branch_process(self, branch): def branch_process(self, branch):
"""Update statistics, and the has_changed flag if change in branch""" """Update statistics, and the has_changed flag if change in branch"""
if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio)
if branch.changed: self.changed = 1 if branch.changed: self.changed = 1
self.add_file_stats(branch) self.add_file_stats(branch)
...@@ -288,7 +289,7 @@ class MirrorITRB(StatsITRB): ...@@ -288,7 +289,7 @@ class MirrorITRB(StatsITRB):
StatsITRB.__init__(self) StatsITRB.__init__(self)
def start_process(self, index, diff_rorp, mirror_dsrp): def start_process(self, index, diff_rorp, mirror_dsrp):
"""Initialize statistics, do actual writing to mirror""" """Initialize statistics and do actual writing to mirror"""
self.start_stats(mirror_dsrp) self.start_stats(mirror_dsrp)
if diff_rorp and not diff_rorp.isplaceholder(): if diff_rorp and not diff_rorp.isplaceholder():
RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute() RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute()
...@@ -312,6 +313,7 @@ class MirrorITRB(StatsITRB): ...@@ -312,6 +313,7 @@ class MirrorITRB(StatsITRB):
def branch_process(self, branch): def branch_process(self, branch):
"""Update statistics with subdirectory results""" """Update statistics with subdirectory results"""
if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio)
self.add_file_stats(branch) self.add_file_stats(branch)
......
trunk/rdiff-backup/src/rorpiter.py
View file @ 1213096f
...@@ -242,7 +242,9 @@ class RORPIter: ...@@ -242,7 +242,9 @@ class RORPIter:
def init(): Hardlink.link_rp(diff_rorp, tf, basisrp) def init(): Hardlink.link_rp(diff_rorp, tf, basisrp)
return Robust.make_tf_robustaction(init, tf, basisrp) return Robust.make_tf_robustaction(init, tf, basisrp)
elif basisrp and basisrp.isreg() and diff_rorp.isreg(): elif basisrp and basisrp.isreg() and diff_rorp.isreg():
assert diff_rorp.get_attached_filetype() == 'diff' if diff_rorp.get_attached_filetype() != 'diff':
raise RPathException("File %s appears to have changed during"
" processing, skipping" % (basisrp.path,))
return Rdiff.patch_with_attribs_action(basisrp, diff_rorp) return Rdiff.patch_with_attribs_action(basisrp, diff_rorp)
else: # Diff contains whole file, just copy it over else: # Diff contains whole file, just copy it over
if not basisrp: basisrp = base_rp.new_index(diff_rorp.index) if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
......
trunk/rdiff-backup/src/selection.py
View file @ 1213096f
...@@ -256,6 +256,8 @@ class Select: ...@@ -256,6 +256,8 @@ class Select:
self.add_selection_func(self.filelist_get_sf( self.add_selection_func(self.filelist_get_sf(
filelists[filelists_index], 0, arg)) filelists[filelists_index], 0, arg))
filelists_index += 1 filelists_index += 1
elif opt == "--exclude-other-filesystems":
self.add_selection_func(self.other_filesystems_get_sf(0))
elif opt == "--exclude-regexp": elif opt == "--exclude-regexp":
self.add_selection_func(self.regexp_get_sf(arg, 0)) self.add_selection_func(self.regexp_get_sf(arg, 0))
elif opt == "--include": elif opt == "--include":
...@@ -416,6 +418,17 @@ probably isn't what you meant.""" % ...@@ -416,6 +418,17 @@ probably isn't what you meant.""" %
else: return (None, None) # dsrp greater, not initial sequence else: return (None, None) # dsrp greater, not initial sequence
else: assert 0, "Include is %s, should be 0 or 1" % (include,) else: assert 0, "Include is %s, should be 0 or 1" % (include,)
def other_filesystems_get_sf(self, include):
"""Return selection function matching files on other filesystems"""
assert include == 0 or include == 1
root_devloc = self.dsrpath.getdevloc()
def sel_func(dsrp):
if dsrp.getdevloc() == root_devloc: return None
else: return include
sel_func.exclude = not include
sel_func.name = "Match other filesystems"
return sel_func
def regexp_get_sf(self, regexp_string, include): def regexp_get_sf(self, regexp_string, include):
"""Return selection function given by regexp_string""" """Return selection function given by regexp_string"""
assert include == 0 or include == 1 assert include == 0 or include == 1
......
trunk/rdiff-backup/src/statistics.py
View file @ 1213096f
...@@ -28,7 +28,7 @@ class StatsObj: ...@@ -28,7 +28,7 @@ class StatsObj:
'ChangedFiles', 'ChangedFiles',
'ChangedSourceSize', 'ChangedMirrorSize', 'ChangedSourceSize', 'ChangedMirrorSize',
'IncrementFiles', 'IncrementFileSize') 'IncrementFiles', 'IncrementFileSize')
stat_misc_attrs = ('Errors',) stat_misc_attrs = ('Errors', 'TotalDestinationSizeChange')
stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime') stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime')
stat_attrs = (('Filename',) + stat_time_attrs + stat_attrs = (('Filename',) + stat_time_attrs +
stat_misc_attrs + stat_file_attrs) stat_misc_attrs + stat_file_attrs)
...@@ -65,6 +65,26 @@ class StatsObj: ...@@ -65,6 +65,26 @@ class StatsObj:
"""Add 1 to value of attribute""" """Add 1 to value of attribute"""
self.__dict__[attr] += 1 self.__dict__[attr] += 1
def get_total_dest_size_change(self):
"""Return total destination size change
This represents the total change in the size of the
rdiff-backup destination directory.
"""
addvals = [self.NewFileSize, self.ChangedSourceSize,
self.IncrementFileSize]
subtractvals = [self.DeletedFileSize, self.ChangedMirrorSize]
for val in addvals + subtractvals:
if val is None:
result = None
break
else:
def addlist(l): return reduce(lambda x,y: x+y, l)
result = addlist(addvals) - addlist(subtractvals)
self.TotalDestinationSizeChange = result
return result
def get_stats_line(self, index, use_repr = 1): def get_stats_line(self, index, use_repr = 1):
"""Return one line abbreviated version of full stats string""" """Return one line abbreviated version of full stats string"""
file_attrs = map(lambda attr: str(self.get_stat(attr)), file_attrs = map(lambda attr: str(self.get_stat(attr)),
...@@ -95,7 +115,9 @@ class StatsObj: ...@@ -95,7 +115,9 @@ class StatsObj:
def get_stats_string(self): def get_stats_string(self):
"""Return extended string printing out statistics""" """Return extended string printing out statistics"""
return self.get_timestats_string() + self.get_filestats_string() return "%s%s%s" % (self.get_timestats_string(),
self.get_filestats_string(),
self.get_miscstats_string())
def get_timestats_string(self): def get_timestats_string(self):
"""Return portion of statistics string dealing with time""" """Return portion of statistics string dealing with time"""
...@@ -112,8 +134,6 @@ class StatsObj: ...@@ -112,8 +134,6 @@ class StatsObj:
self.ElapsedTime = self.EndTime - self.StartTime self.ElapsedTime = self.EndTime - self.StartTime
timelist.append("ElapsedTime %.2f (%s)\n" % timelist.append("ElapsedTime %.2f (%s)\n" %
(self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) (self.ElapsedTime, Time.inttopretty(self.ElapsedTime)))
if self.Errors is not None:
timelist.append("Errors %d\n" % self.Errors)
return "".join(timelist) return "".join(timelist)
def get_filestats_string(self): def get_filestats_string(self):
...@@ -130,8 +150,23 @@ class StatsObj: ...@@ -130,8 +150,23 @@ class StatsObj:
return "".join(map(fileline, self.stat_file_pairs)) return "".join(map(fileline, self.stat_file_pairs))
def get_miscstats_string(self):
"""Return portion of extended stat string about misc attributes"""
misc_string = ""
tdsc = self.get_total_dest_size_change()
if tdsc is not None:
misc_string += ("TotalDestinationSizeChange %s (%s)\n" %
(tdsc, self.get_byte_summary_string(tdsc)))
if self.Errors is not None: misc_string += "Errors %d\n" % self.Errors
return misc_string
def get_byte_summary_string(self, byte_count): def get_byte_summary_string(self, byte_count):
"""Turn byte count into human readable string like "7.23GB" """ """Turn byte count into human readable string like "7.23GB" """
if byte_count < 0:
sign = "-"
byte_count = -byte_count
else: sign = ""
for abbrev_bytes, abbrev_string in self.byte_abbrev_list: for abbrev_bytes, abbrev_string in self.byte_abbrev_list:
if byte_count >= abbrev_bytes: if byte_count >= abbrev_bytes:
# Now get 3 significant figures # Now get 3 significant figures
...@@ -139,11 +174,11 @@ class StatsObj: ...@@ -139,11 +174,11 @@ class StatsObj:
if abbrev_count >= 100: precision = 0 if abbrev_count >= 100: precision = 0
elif abbrev_count >= 10: precision = 1 elif abbrev_count >= 10: precision = 1
else: precision = 2 else: precision = 2
return "%%.%df %s" % (precision, abbrev_string) \ return "%s%%.%df %s" % (sign, precision, abbrev_string) \
% (abbrev_count,) % (abbrev_count,)
byte_count = round(byte_count) byte_count = round(byte_count)
if byte_count == 1: return "1 byte" if byte_count == 1: return sign + "1 byte"
else: return "%d bytes" % (byte_count,) else: return "%s%d bytes" % (sign, byte_count)
def get_stats_logstring(self, title): def get_stats_logstring(self, title):
"""Like get_stats_string, but add header and footer""" """Like get_stats_string, but add header and footer"""
......
trunk/rdiff-backup/testing/commontest.py
View file @ 1213096f
...@@ -54,6 +54,15 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir, ...@@ -54,6 +54,15 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
os.system(" ".join(cmdargs)) os.system(" ".join(cmdargs))
def cmd_schemas2rps(schema_list, remote_schema):
"""Input list of file descriptions and the remote schema, return rps
File descriptions should be strings of the form 'hostname.net::foo'
"""
return map(SetConnections.cmdpair2rp,
SetConnections.get_cmd_pairs(schema_list, remote_schema))
def InternalBackup(source_local, dest_local, src_dir, dest_dir, def InternalBackup(source_local, dest_local, src_dir, dest_dir,
current_time = None): current_time = None):
"""Backup src to dest internally """Backup src to dest internally
...@@ -75,7 +84,7 @@ def InternalBackup(source_local, dest_local, src_dir, dest_dir, ...@@ -75,7 +84,7 @@ def InternalBackup(source_local, dest_local, src_dir, dest_dir,
dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
% (SourceDir, dest_dir) % (SourceDir, dest_dir)
rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema) rpin, rpout = cmd_schemas2rps([src_dir, dest_dir], remote_schema)
Main.misc_setup([rpin, rpout]) Main.misc_setup([rpin, rpout])
Main.Backup(rpin, rpout) Main.Backup(rpin, rpout)
Main.cleanup() Main.cleanup()
...@@ -92,7 +101,7 @@ def InternalMirror(source_local, dest_local, src_dir, dest_dir, ...@@ -92,7 +101,7 @@ def InternalMirror(source_local, dest_local, src_dir, dest_dir,
dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
% (SourceDir, dest_dir) % (SourceDir, dest_dir)
rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema) rpin, rpout = cmd_schemas2rps([src_dir, dest_dir], remote_schema)
Main.misc_setup([rpin, rpout]) Main.misc_setup([rpin, rpout])
Main.backup_init_select(rpin, rpout) Main.backup_init_select(rpin, rpout)
if not rpout.lstat(): rpout.mkdir() if not rpout.lstat(): rpout.mkdir()
...@@ -127,8 +136,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time): ...@@ -127,8 +136,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
% (SourceDir, dest_dir) % (SourceDir, dest_dir)
mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir], mirror_rp, dest_rp = cmd_schemas2rps([mirror_dir, dest_dir], remote_schema)
remote_schema)
Time.setcurtime() Time.setcurtime()
inc = get_increment_rp(mirror_rp, time) inc = get_increment_rp(mirror_rp, time)
if inc: Main.restore(get_increment_rp(mirror_rp, time), dest_rp) if inc: Main.restore(get_increment_rp(mirror_rp, time), dest_rp)
......
trunk/rdiff-backup/testing/connectiontest.py
View file @ 1213096f
...@@ -168,13 +168,22 @@ class RedirectedConnectionTest(unittest.TestCase): ...@@ -168,13 +168,22 @@ class RedirectedConnectionTest(unittest.TestCase):
def testBasic(self): def testBasic(self):
"""Test basic operations with redirection""" """Test basic operations with redirection"""
self.conna.Globals.set("tmp_val", 1)
self.connb.Globals.set("tmp_val", 2)
assert self.conna.Globals.get("tmp_val") == 1
assert self.connb.Globals.get("tmp_val") == 2
self.conna.Globals.set("tmp_connb", self.connb) self.conna.Globals.set("tmp_connb", self.connb)
self.connb.Globals.set("tmp_conna", self.conna) self.connb.Globals.set("tmp_conna", self.conna)
assert self.conna.Globals.get("tmp_connb") is self.connb assert self.conna.Globals.get("tmp_connb") is self.connb
assert self.connb.Globals.get("tmp_conna") is self.conna assert self.connb.Globals.get("tmp_conna") is self.conna
#self.conna.Test_SetConnGlobals(self.connb, "tmp_settest", 1) val = self.conna.reval("Globals.get('tmp_connb').Globals.get",
#assert self.connb.Globals.get("tmp_settest") "tmp_val")
assert val == 2, val
val = self.connb.reval("Globals.get('tmp_conna').Globals.get",
"tmp_val")
assert val == 1, val
assert self.conna.reval("Globals.get('tmp_connb').pow", 2, 3) == 8 assert self.conna.reval("Globals.get('tmp_connb').pow", 2, 3) == 8
self.conna.reval("Globals.tmp_connb.reval", self.conna.reval("Globals.tmp_connb.reval",
......
trunk/rdiff-backup/testing/finaltest.py
View file @ 1213096f
...@@ -17,6 +17,7 @@ class Local: ...@@ -17,6 +17,7 @@ class Local:
def get_local_rp(extension): def get_local_rp(extension):
return RPath(Globals.local_connection, "testfiles/" + extension) return RPath(Globals.local_connection, "testfiles/" + extension)
vftrp = get_local_rp('various_file_types')
inc1rp = get_local_rp('increment1') inc1rp = get_local_rp('increment1')
inc2rp = get_local_rp('increment2') inc2rp = get_local_rp('increment2')
inc3rp = get_local_rp('increment3') inc3rp = get_local_rp('increment3')
...@@ -71,6 +72,19 @@ class PathSetter(unittest.TestCase): ...@@ -71,6 +72,19 @@ class PathSetter(unittest.TestCase):
print "executing " + cmdstr print "executing " + cmdstr
assert not os.system(cmdstr) assert not os.system(cmdstr)
def exec_rb_extra_args(self, time, extra_args, *args):
"""Run rdiff-backup on given arguments"""
arglist = []
if time: arglist.append("--current-time %s" % str(time))
arglist.append(self.src_prefix + args[0])
if len(args) > 1:
arglist.append(self.dest_prefix + args[1])
assert len(args) == 2
cmdstr = "%s %s %s" % (self.rb_schema, extra_args, ' '.join(arglist))
print "executing " + cmdstr
assert not os.system(cmdstr)
def exec_rb_restore(self, time, *args): def exec_rb_restore(self, time, *args):
"""Restore using rdiff-backup's new syntax and given time""" """Restore using rdiff-backup's new syntax and given time"""
arglist = [] arglist = []
...@@ -174,6 +188,23 @@ class Final(PathSetter): ...@@ -174,6 +188,23 @@ class Final(PathSetter):
self.set_connections("test1/", '../', 'test2/tmp/', '../../') self.set_connections("test1/", '../', 'test2/tmp/', '../../')
self.runtest() self.runtest()
def testMirroringLocal(self):
"""Run mirroring only everything remote"""
self.delete_tmpdirs()
self.set_connections(None, None, None, None)
self.exec_rb_extra_args(10000, "-m",
"testfiles/various_file_types",
"testfiles/output")
assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None)
def testMirroringRemote(self):
"""Run mirroring only everything remote"""
self.delete_tmpdirs()
self.set_connections("test1/", "../", "test2/tmp/", "../../")
self.exec_rb_extra_args(10000, "-m",
"testfiles/various_file_types",
"testfiles/output")
assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None)
class FinalSelection(PathSetter): class FinalSelection(PathSetter):
"""Test selection options""" """Test selection options"""
......
trunk/rdiff-backup/testing/securitytest.py 0 → 100644
View file @ 1213096f
import os, unittest
from commontest import *
import rdiff_backup.Security, Security
#Log.setverbosity(5)
class SecurityTest(unittest.TestCase):
def assert_exc_sec(self, exc):
"""Fudge - make sure exception is a security violation
This is necessary because of some kind of pickling/module
problem.
"""
assert isinstance(exc, rdiff_backup.Security.Violation)
#assert str(exc).find("Security") >= 0, "%s\n%s" % (exc, repr(exc))
def test_vet_request_ro(self):
"""Test vetting of ConnectionRequests on read-only server"""
remote_cmd = "rdiff-backup --server --restrict-read-only foo"
conn = SetConnections.init_connection(remote_cmd)
assert type(conn.os.getuid()) is type(5)
try: conn.os.remove("/tmp/foobar")
except Exception, e: self.assert_exc_sec(e)
else: assert 0, "No exception raised"
SetConnections.CloseConnections()
def test_vet_request_minimal(self):
"""Test vetting of ConnectionRequests on minimal server"""
remote_cmd = "rdiff-backup --server --restrict-update-only foo"
conn = SetConnections.init_connection(remote_cmd)
assert type(conn.os.getuid()) is type(5)
try: conn.os.remove("/tmp/foobar")
except Exception, e: self.assert_exc_sec(e)
else: assert 0, "No exception raised"
SetConnections.CloseConnections()
def test_vet_rpath(self):
"""Test to make sure rpaths not in restricted path will be rejected"""
remote_cmd = "rdiff-backup --server --restrict-update-only foo"
conn = SetConnections.init_connection(remote_cmd)
for rp in [RPath(Globals.local_connection, "blahblah"),
RPath(conn, "foo/bar")]:
conn.Globals.set("TEST_var", rp)
assert conn.Globals.get("TEST_var").path == rp.path
for rp in [RPath(conn, "foobar"),
RPath(conn, "/usr/local"),
RPath(conn, "foo/../bar")]:
try: conn.Globals.set("TEST_var", rp)
except Exception, e:
self.assert_exc_sec(e)
continue
assert 0, "No violation raised by rp %s" % (rp,)
SetConnections.CloseConnections()
if __name__ == "__main__": unittest.main()
trunk/rdiff-backup/testing/selectiontest.py
View file @ 1213096f
...@@ -221,6 +221,18 @@ testfiles/select/1/1 ...@@ -221,6 +221,18 @@ testfiles/select/1/1
select.filelist_get_sf(StringIO.StringIO("/foo/bar"), 0, select.filelist_get_sf(StringIO.StringIO("/foo/bar"), 0,
"test")(root) == None "test")(root) == None
def testOtherFilesystems(self):
"""Test to see if --exclude-other-filesystems works correctly"""
root = DSRPath(1, Globals.local_connection, "/")
select = Select(root)
sf = select.other_filesystems_get_sf(0)
assert sf(root) is None
assert sf(RPath(Globals.local_connection, "/usr/bin")) is None, \
"Assumption: /usr/bin is on the same filesystem as /"
assert sf(RPath(Globals.local_connection, "/proc")) == 0, \
"Assumption: /proc is on a different filesystem"
assert sf(RPath(Globals.local_connection, "/boot")) == 0, \
"Assumption: /boot is on a different filesystem"
class ParseArgsTest(unittest.TestCase): class ParseArgsTest(unittest.TestCase):
"""Test argument parsing""" """Test argument parsing"""
......
trunk/rdiff-backup/testing/statisticstest.py
View file @ 1213096f
...@@ -57,6 +57,7 @@ ChangedSourceSize 8 (8 bytes) ...@@ -57,6 +57,7 @@ ChangedSourceSize 8 (8 bytes)
ChangedMirrorSize 9 (9 bytes) ChangedMirrorSize 9 (9 bytes)
IncrementFiles 15 IncrementFiles 15
IncrementFileSize 10 (10 bytes) IncrementFileSize 10 (10 bytes)
TotalDestinationSizeChange 7 (7 bytes)
""", "'%s'" % stats_string """, "'%s'" % stats_string
def test_line_string(self): def test_line_string(self):
......
trunk/rdiff-backup/testing/timetest.py
View file @ 1213096f
import unittest import unittest, time
from commontest import * from commontest import *
import Globals, Time import Globals, Time
...@@ -108,5 +108,29 @@ class TimeTest(unittest.TestCase): ...@@ -108,5 +108,29 @@ class TimeTest(unittest.TestCase):
self.assertRaises(Time.TimeException, g2t, "") self.assertRaises(Time.TimeException, g2t, "")
self.assertRaises(Time.TimeException, g2t, "3q") self.assertRaises(Time.TimeException, g2t, "3q")
def testSleeping(self):
"""Test sleep and sleep ratio"""
sleep_ratio = 0.5
time1 = time.time()
Time.sleep(0) # set initial time
time.sleep(1)
time2 = time.time()
Time.sleep(sleep_ratio)
time3 = time.time()
time.sleep(0.5)
time4 = time.time()
Time.sleep(sleep_ratio)
time5 = time.time()
sleep_ratio = 0.25
time.sleep(0.75)
time6 = time.time()
Time.sleep(sleep_ratio)
time7 = time.time()
assert 0.9 < time3 - time2 < 1.1, time3 - time2
assert 0.4 < time5 - time4 < 0.6, time5 - time4
assert 0.2 < time7 - time6 < 0.3, time7 - time6
if __name__ == '__main__': unittest.main() if __name__ == '__main__': unittest.main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7