Skip to content

R
rdiff-backup
Commit 00be1f4c authored by ben's avatar ben
Browse files
Options

Added compile.py to compile the c portion 

Added GPL headers to all the source code files

Fixed memory leaks and bad pointer casts in cmodule.c


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@141 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent 4076aba8
Hide whitespace changes
Inline Side-by-side
Showing with 996 additions and 450 deletions
trunk/rdiff-backup/rdiff_backup/FilenameMapping.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Coordinate corresponding files with different names
For instance, some source filenames may contain characters not allowed
on the mirror end. Also, if a source filename is very long (say 240
characters), the extra characters added to related increments may put
them over the usual 255 character limit.
"""
import re
from log import *
import Globals
#######################################################################
#
# filename_mapping - used to coordinate related filenames
#
# For instance, some source filenames may contain characters not
# allowed on the mirror end. Also, if a source filename is very long
# (say 240 characters), the extra characters added to related
# increments may put them over the usual 255 character limit.
#
"""Contains class methods which coordinate related filenames"""
max_filename_length = 255
# If true, enable character quoting, and set characters making
......
trunk/rdiff-backup/rdiff_backup/Globals.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Hold a variety of constants usually set at initialization."""
import re, os
# The current version of rdiff-backup
version = "0.9.0"
version = "0.9.1"
# If this is set, use this value in seconds as the current time
# instead of reading it from the clock.
......
trunk/rdiff-backup/rdiff_backup/Hardlink.py
View file @ 00be1f4c
from __future__ import generators
import cPickle
#######################################################################
# Copyright 2002 Ben Escoto
#
# hardlink - code for preserving and restoring hardlinks
#
# If the preserve_hardlinks option is selected, linked files in the
# source directory will be linked in the mirror directory. Linked
# files are treated like any other with respect to incrementing, but a
# database of all links will be recorded at each session, so linked
# files can still be restored from the increments.
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Preserve and restore hard links
"""Hardlink class methods and data
If the preserve_hardlinks option is selected, linked files in the
source directory will be linked in the mirror directory. Linked files
are treated like any other with respect to incrementing, but a
database of all links will be recorded at each session, so linked
files can still be restored from the increments.
All these functions are meant to be executed on the destination
side. The source side should only transmit inode information.
"""
from __future__ import generators
import cPickle
# In all of these lists of indicies are the values. The keys in
# _inode_ ones are (inode, devloc) pairs.
_src_inode_indicies = {}
......
trunk/rdiff-backup/rdiff_backup/Main.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Start (and end) here - read arguments, set global settings, etc."""
import getopt, sys, re
from log import *
from lazy import *
......@@ -10,10 +22,7 @@ from highlevel import *
from manage import *
import Globals, Time, SetConnections
#######################################################################
#
# main - Start here: Read arguments, set global settings, etc.
#
action = None
remote_cmd, remote_schema = None, None
force = None
......
trunk/rdiff-backup/rdiff_backup/MiscStats.py
View file @ 00be1f4c
from statistics import *
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Misc statistics methods, pertaining to dir and session stat files"""
from statistics import *
# This is the RPath of the directory statistics file, and the
# associated open file. It will hold a line of statistics for
# each directory that is backed up.
......
trunk/rdiff-backup/rdiff_backup/Rdiff.py
View file @ 00be1f4c
import os, popen2
#######################################################################
# Copyright 2002 Ben Escoto
#
# rdiff - Invoke rdiff utility to make signatures, deltas, or patch
# This file is part of rdiff-backup.
#
# All these operations should be done in a relatively safe manner
# using RobustAction and the like.
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Invoke rdiff utility to make signatures, deltas, or patch
All these operations should be done in a relatively safe manner using
RobustAction and the like.
"""
import os, popen2
class RdiffException(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/SetConnections.py
View file @ 00be1f4c
#######################################################################
# Copyright 2002 Ben Escoto
#
# setconnections - Parse initial arguments and establish connections
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Parse args and setup connections
The methods in this class are used once by Main to parse file
descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to
set up the related connections.
The functions in this module are used once by Main to parse file
descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to set up
the related connections.
"""
class SetConnectionsException(Exception): pass
# This is the schema that determines how rdiff-backup will open a
# pipe to the remote system. If the file is given as A::B, %s will
# be substituted with A in the schema.
......@@ -24,6 +26,8 @@ __cmd_schema_no_compress = 'ssh %s rdiff-backup --server'
# The first is None because it is the local connection.
__conn_remote_cmds = [None]
class SetConnectionsException(Exception): pass
def InitRPs(arglist, remote_schema = None, remote_cmd = None):
"""Map the given file descriptions into rpaths and return list"""
global __cmd_schema
......
trunk/rdiff-backup/rdiff_backup/Time.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Provide time related exceptions and functions"""
import time, types, re
import Globals
#######################################################################
#
# ttime - Provide Time class, which contains time related functions.
#
class TimeException(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/cmodule.c
View file @ 00be1f4c
/* ----------------------------------------------------------------------- *
*
* Copyright 2002 Ben Escoto
*
* This file is part of rdiff-backup.
*
* rdiff-backup is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation, Inc., 675 Mass Ave,
* Cambridge MA 02139, USA; either version 2 of the License, or (at
* your option) any later version; incorporated herein by reference.
*
* ----------------------------------------------------------------------- */
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <Python.h>
#include <errno.h>
static PyObject *UnknownFileTypeError;
static PyObject *c_make_file_dict(PyObject *self, PyObject *args);
static PyObject *long2str(PyObject *self, PyObject *args);
static PyObject *str2long(PyObject *self, PyObject *args);
/* Turn a stat structure into a python dictionary. The preprocessor
stuff taken from Python's posixmodule.c */
static PyObject *c_make_file_dict(self, args)
PyObject *self;
PyObject *args;
{
PyObject *size, *inode, *mtime, *atime, *devloc, *return_val;
char *filename, filetype[5];
struct stat sbuf;
mode_t mode;
long int mode, perms;
if (!PyArg_ParseTuple(args, "s", &filename)) return NULL;
if (lstat(filename, &sbuf) != 0) {
......@@ -21,7 +45,27 @@ static PyObject *c_make_file_dict(self, args)
return NULL;
}
}
mode = sbuf.st_mode;
#ifdef HAVE_LARGEFILE_SUPPORT
size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size);
inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino);
#else
size = PyInt_FromLong((long)sbuf.st_size);
inode = PyInt_FromLong((long)sbuf.st_ino);
#endif
mode = (long)sbuf.st_mode;
perms = mode & (S_IRWXU | S_IRWXG | S_IRWXO);
#if defined(HAVE_LONG_LONG) && !defined(MS_WINDOWS)
devloc = PyLong_FromLongLong((LONG_LONG)sbuf.st_dev);
#else
devloc = PyInt_FromLong((long)sbuf.st_dev);
#endif
#if SIZEOF_TIME_T > SIZEOF_LONG
mtime = PyLong_FromLongLong((LONG_LONG)sbuf.st_mtime);
atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime);
#else
mtime = PyInt_FromLong((long)sbuf.st_mtime);
atime = PyLong_FromLongLong((long)sbuf.st_atime);
#endif
/* Build return dictionary from stat struct */
if (S_ISREG(mode) || S_ISDIR(mode) || S_ISSOCK(mode) || S_ISFIFO(mode)) {
......@@ -30,17 +74,17 @@ static PyObject *c_make_file_dict(self, args)
else if S_ISDIR(mode) strcpy(filetype, "dir");
else if S_ISSOCK(mode) strcpy(filetype, "sock");
else strcpy(filetype, "fifo");
return Py_BuildValue("{s:s,s:l,s:i,s:i,s:i,s:l,s:i,s:i,s:l,s:l}",
return Py_BuildValue("{s:s,s:N,s:l,s:l,s:l,s:N,s:N,s:l,s:N,s:N}",
"type", filetype,
"size", (long int)sbuf.st_size,
"perms", (int)(mode & S_IRWXU),
"uid", (int)sbuf.st_uid,
"gid", (int)sbuf.st_gid,
"inode", (long int)sbuf.st_ino,
"devloc", (int)sbuf.st_dev,
"nlink", (int)sbuf.st_nlink,
"mtime", (long int)sbuf.st_mtime,
"atime", (long int)sbuf.st_atime);
"size", size,
"perms", perms,
"uid", (long)sbuf.st_uid,
"gid", (long)sbuf.st_gid,
"inode", inode,
"devloc", devloc,
"nlink", (long)sbuf.st_nlink,
"mtime", mtime,
"atime", atime);
} else if S_ISLNK(mode) {
/* Symbolic links */
char linkname[1024];
......@@ -51,89 +95,108 @@ static PyObject *c_make_file_dict(self, args)
}
linkname[len_link] = '\0';
return Py_BuildValue("{s:s,s:l,s:i,s:i,s:i,s:l,s:i,s:i,s:s}",
"type", "sym",
"size", (long int)sbuf.st_size,
"perms", (int)(mode & S_IRWXU),
"uid", (int)sbuf.st_uid,
"gid", (int)sbuf.st_gid,
"inode", (long int)sbuf.st_ino,
"devloc", (int)sbuf.st_dev,
"nlink", (int)sbuf.st_nlink,
"linkname", linkname);
return_val = Py_BuildValue("{s:s,s:N,s:l,s:l,s:l,s:N,s:N,s:l,s:s}",
"type", "sym",
"size", size,
"perms", perms,
"uid", (long)sbuf.st_uid,
"gid", (long)sbuf.st_gid,
"inode", inode,
"devloc", devloc,
"nlink", (long)sbuf.st_nlink,
"linkname", linkname);
Py_DECREF(mtime);
Py_DECREF(atime);
return return_val;
} else if (S_ISCHR(mode) || S_ISBLK(mode)) {
/* Device files */
char devtype[2];
int devnums = (int)sbuf.st_rdev;
#if defined(HAVE_LONG_LONG) && !defined(MS_WINDOWS)
LONG_LONG devnums = (LONG_LONG)sbuf.st_rdev;
PyObject *major_num = PyLong_FromLongLong(devnums >> 8);
#else
long int devnums = (long)sbuf.st_dev;
PyObject *major_num = PyInt_FromLong(devnums >> 8);
#endif
int minor_num = (int)(devnums & 0xff);
if S_ISCHR(mode) strcpy(devtype, "c");
else strcpy(devtype, "b");
return Py_BuildValue("{s:s,s:l,s:i,s:i,s:i,s:l,s:i,s:i,s:O}",
"type", "dev",
"size", (long int)sbuf.st_size,
"perms", (int)(mode & S_IRWXU),
"uid", (int)sbuf.st_uid,
"gid", (int)sbuf.st_gid,
"inode", (long int)sbuf.st_ino,
"devloc", (int)sbuf.st_dev,
"nlink", (int)sbuf.st_nlink,
"devnums", Py_BuildValue("(s,i,i)", devtype,
devnums >> 8,
devnums & 0xff),
"mtime", (long int)sbuf.st_mtime,
"atime", (long int)sbuf.st_atime);
return_val = Py_BuildValue("{s:s,s:N,s:l,s:l,s:l,s:N,s:N,s:l,s:N}",
"type", "dev",
"size", size,
"perms", perms,
"uid", (long)sbuf.st_uid,
"gid", (long)sbuf.st_gid,
"inode", inode,
"devloc", devloc,
"nlink", (long)sbuf.st_nlink,
"devnums", Py_BuildValue("(s,O,i)", devtype,
major_num, minor_num));
Py_DECREF(mtime);
Py_DECREF(atime);
return return_val;
} else {
/* Unrecognized file type - pretend it isn't there */
errno = ENOENT;
PyErr_SetFromErrno(PyExc_OSError);
/* Unrecognized file type - raise exception */
Py_DECREF(size);
Py_DECREF(inode);
Py_DECREF(devloc);
Py_DECREF(mtime);
Py_DECREF(atime);
PyErr_SetString(UnknownFileTypeError, filename);
return NULL;
}
}
/* Convert python long into 7 byte string */
static PyObject *long2str(self, args)
PyObject *self;
PyObject *args;
{
unsigned char s[7];
int sindex;
unsigned long long int l;
PyObject *pylong;
if (!PyArg_ParseTuple(args, "O", &pylong)) return NULL;
l = PyLong_AsUnsignedLongLong(pylong);
for(sindex = 0; sindex <= 6; sindex++) {
s[sindex] = l % 256;
l /= 256;
}
return Py_BuildValue("s#", s, 7);
PyLongObject *pylong;
PyObject *return_val;
if (!PyArg_ParseTuple(args, "O!", &PyLong_Type, &pylong)) return NULL;
if (_PyLong_AsByteArray(pylong, s, 7, 0, 0) != 0) return NULL;
else return Py_BuildValue("s#", s, 7);
return return_val;
}
/* Reverse of above; convert 7 byte string into python long */
static PyObject *str2long(self, args)
PyObject *self;
PyObject *args;
{
unsigned char *s;
unsigned long long int l = 0;
int sindex, ssize;
int ssize;
if (!PyArg_ParseTuple(args, "s#", &s, &ssize)) return NULL;
if (ssize != 7) return Py_BuildValue("i", -1);
for(sindex=6; sindex >= 0; sindex--)
l = l*256 + s[sindex];
return PyLong_FromLongLong(l);
if (ssize != 7) {
PyErr_SetString(PyExc_TypeError, "Single argument must be 7 char string");
return NULL;
}
return _PyLong_FromByteArray(s, 7, 0, 0);
}
static PyMethodDef CMethods[] = {
{"make_file_dict", c_make_file_dict, METH_VARARGS,
"Make dictionary from file stat"},
{"long2str", long2str, METH_VARARGS,
"Convert long int to 7 byte string"},
{"str2long", str2long, METH_VARARGS,
"Convert 7 byte string to long int"},
{"long2str", long2str, METH_VARARGS, "Convert python long to 7 byte string"},
{"str2long", str2long, METH_VARARGS, "Convert 7 byte string to python long"},
{NULL, NULL, 0, NULL}
};
void initC(void)
{
(void) Py_InitModule("C", CMethods);
PyObject *m, *d;
m = Py_InitModule("C", CMethods);
d = PyModule_GetDict(m);
UnknownFileTypeError = PyErr_NewException("C.UnknownFileTypeError",
NULL, NULL);
PyDict_SetItemString(d, "UnknownFileTypeError", UnknownFileTypeError);
}
trunk/rdiff-backup/rdiff_backup/compilec.py 0 → 100755
View file @ 00be1f4c
#!/usr/bin/env python
import sys, os
from distutils.core import setup, Extension
assert len(sys.argv) == 1
sys.argv.append("build")
setup(name="CModule",
version="0.9.0",
description="rdiff-backup's C component",
ext_modules=[Extension("C", ["cmodule.c"])])
assert not os.system("mv build/lib.linux-i686-2.2/C.so .")
assert not os.system("rm -rf build")
trunk/rdiff-backup/rdiff_backup/connection.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Support code for remote execution and data transfer"""
from __future__ import generators
import types, os, tempfile, cPickle, shutil, traceback, pickle
#######################################################################
#
# connection - Code that deals with remote execution
#
class ConnectionError(Exception): pass
class ConnectionReadError(ConnectionError): pass
class ConnectionQuit(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/destructive_stepping.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Deal with side effects from traversing trees"""
from __future__ import generators
import types
from rpath import *
from lazy import *
#######################################################################
#
# destructive-stepping - Deal with side effects from traversing trees
#
class DSRPPermError(Exception):
"""Exception used when a DSRPath can't get sufficient permissions"""
......
trunk/rdiff-backup/rdiff_backup/highlevel.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""High level functions for mirroring, mirror & inc, etc."""
from __future__ import generators
from static import *
from log import *
......@@ -8,10 +20,6 @@ from destructive_stepping import *
from rorpiter import *
import Globals, Hardlink, MiscStats
#######################################################################
#
# highlevel - High level functions for mirroring, mirror & inc, etc.
#
class SkipFileException(Exception):
"""Signal that the current file should be skipped but then continue
......
trunk/rdiff-backup/rdiff_backup/increment.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Provides Inc and *ITR classes, which relate to writing increment files"""
import traceback
from static import *
from statistics import *
from lazy import *
#######################################################################
#
# increment - Provides Inc class, which writes increment files
#
# This code is what writes files ending in .diff, .snapshot, etc.
#
class Inc:
"""Class containing increment functions"""
......
trunk/rdiff-backup/rdiff_backup/iterfile.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Convert an iterator to a file object and vice-versa"""
import cPickle, array
import Globals, C
#######################################################################
#
# iterfile - Convert an iterator to a file object and vice-versa
#
class IterFileException(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/lazy.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Define some lazy data structures and functions acting on them"""
from __future__ import generators
import os, stat, types
from static import *
#######################################################################
#
# lazy - Define some lazy data structures and functions acting on them
#
class Iter:
"""Hold static methods for the manipulation of lazy iterators"""
......@@ -210,7 +218,7 @@ class IterTreeReducer:
self.base_index = self.index = None
self.subinstances = [self]
self.finished = None
self.caught_exception = None
self.caught_exception = self.start_successful = None
def finish_subinstances(self, index):
"""Run Finish() on all subinstances index has passed
......@@ -235,9 +243,10 @@ class IterTreeReducer:
def call_end_proc(self):
"""Runs the end_process on self, checking for errors"""
if self.finished: self.caught_exception = 1
if self.caught_exception: self.log_pref_error(self.base_index)
Robust.check_common_error(self.on_error, self.end_process)
if self.finished or not self.start_successful:
self.caught_exception = 1
if self.caught_exception: self.log_prev_error(self.base_index)
else: Robust.check_common_error(self.on_error, self.end_process)
self.finished = 1
def add_subinstance(self):
......@@ -248,7 +257,9 @@ class IterTreeReducer:
def process_w_subinstance(self, index, subinst, args):
"""Run start_process on latest subinstance"""
Robust.check_common_error(self.on_error, subinst.start_process, args)
Robust.check_common_error(subinst.on_error,
subinst.start_process, args)
if not subinst.caught_exception: subinst.start_successful = 1
subinst.base_index = index
def start_process(self, *args):
......@@ -278,7 +289,8 @@ class IterTreeReducer:
def log_prev_error(self, index):
"""Call function if no pending exception"""
Log("Skipping %s because of previous error" % os.path.join(*index), 2)
Log("Skipping %s because of previous error" %
(os.path.join(*index),), 2)
def __call__(self, *args):
"""Process args, where args[0] is current position in iterator
......@@ -303,10 +315,10 @@ class IterTreeReducer:
if self.finish_subinstances(index) is None:
return None # We are no longer in the main tree
if self.caught_exception: self.log_prev_error(index)
else:
if self.subinstances[-1].start_successful:
subinst = self.add_subinstance()
self.process_w_subinstance(index, subinst, args)
else: self.log_prev_error(index)
self.index = index
return 1
......
trunk/rdiff-backup/rdiff_backup/log.py
View file @ 00be1f4c
import time, sys, traceback, types
#######################################################################
# Copyright 2002 Ben Escoto
#
# log - Manage logging
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Manage logging, displaying and recording messages with required verbosity"""
import time, sys, traceback, types
class LoggerError(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/manage.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""list, delete, and otherwise manage increments"""
from __future__ import generators
from static import *
from log import *
import Globals, Time
#######################################################################
#
# manage - list, delete, and otherwise manage increments
#
class ManageException(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/restore.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Read increment files and restore to original"""
from __future__ import generators
import tempfile
from static import *
#######################################################################
#
# restore - Read increment files and restore to original
#
class RestoreError(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/robust.py
View file @ 00be1f4c
import tempfile, errno, signal, cPickle
from static import *
#######################################################################
#
# robust - code which prevents mirror from being corrupted, error-recovery
# Copyright 2002 Ben Escoto
#
# Ideally no matter an instance of rdiff-backup gets aborted, no
# information should get lost. The target directory should be left in
# a coherent state, and later instances of rdiff-backup should clean
# things up so there is no sign that anything ever got aborted or
# failed.
# This file is part of rdiff-backup.
#
# Thus, files should be updated in an atomic way as possible. Each
# file should be updated (and the corresponding diff files written) or
# not, and it should be clear which happened. In general, I don't
# think this is possible, since the creation of the diff files and the
# changing of updated files cannot be guarateed to happen together.
# It is possible, I think, to record various information to files
# which would allow a later process to figure out what the last
# operation was, but this would add several file operations to the
# processing of each file, and I don't think, would be a good
# tradeoff.
#
# The compromise reached here is that diff files should be created
# just before the mirror files are updated, and each file update
# should be done with a rename operation on a file in the same
# directory. Furthermore, every once in a while, rdiff-backup will
# record which file it just finished processing. If any fatal errors
# are caught, it will also record the last processed file. Future
# instances may not know exactly when the previous instance was
# aborted, but they will be able to narrow down the possibilities.
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Prevent mirror from being corrupted; handle errors
Ideally no matter an instance of rdiff-backup gets aborted, no
information should get lost. The target directory should be left in a
coherent state, and later instances of rdiff-backup should clean
things up so there is no sign that anything ever got aborted or
failed.
Thus, files should be updated in an atomic way as possible. Each file
should be updated (and the corresponding diff files written) or not,
and it should be clear which happened. In general, I don't think this
is possible, since the creation of the diff files and the changing of
updated files cannot be guarateed to happen together. It is possible,
I think, to record various information to files which would allow a
later process to figure out what the last operation was, but this
would add several file operations to the processing of each file, and
I don't think, would be a good tradeoff.
The compromise reached here is that diff files should be created just
before the mirror files are updated, and each file update should be
done with a rename operation on a file in the same directory.
Furthermore, every once in a while, rdiff-backup will record which
file it just finished processing. If any fatal errors are caught, it
will also record the last processed file. Future instances may not
know exactly when the previous instance was aborted, but they will be
able to narrow down the possibilities.
"""
import tempfile, errno, signal, cPickle, C
from static import *
class RobustAction:
"""Represents a file operation to be accomplished later"""
......@@ -243,7 +253,8 @@ class Robust:
"""
try: return function(*args)
except (EnvironmentError, SkipFileException, DSRPPermError,
RPathException, Rdiff.RdiffException), exc:
RPathException, Rdiff.RdiffException,
C.UnknownFileTypeError), exc:
TracebackArchive.add()
if (not isinstance(exc, EnvironmentError) or
(errno.errorcode[exc[0]] in
......
trunk/rdiff-backup/rdiff_backup/rorpiter.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Operations on Iterators of Read Only Remote Paths"""
from __future__ import generators
import tempfile, UserList, types
from static import *
......@@ -7,10 +19,6 @@ from robust import *
from iterfile import *
import Globals, Rdiff, Hardlink
#######################################################################
#
# rorpiter - Operations on Iterators of Read Only Remote Paths
#
class RORPIterException(Exception): pass
......
trunk/rdiff-backup/rdiff_backup/rpath.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Wrapper class around a real path like "/usr/bin/env"
The RPath and associated classes make some function calls more
convenient (e.g. RPath.getperms()) and also make working with files on
remote systems transparent.
"""
import os, stat, re, sys, shutil, gzip
from static import *
#######################################################################
#
# rpath - Wrapper class around a real path like "/usr/bin/env"
#
# The RPath and associated classes make some function calls more
# convenient (e.g. RPath.getperms()) and also make working with files
# on remote systems transparent.
#
class RPathException(Exception): pass
......@@ -452,12 +462,11 @@ class RPath(RORPath):
"""Set data dictionary using C extension"""
self.data = self.conn.C.make_file_dict(self.path)
def setdata_old(self):
def make_file_dict_old(self):
"""Create the data dictionary"""
statblock = self.conn.RPathStatic.tupled_lstat(self.path)
if statblock is None:
self.data = {'type':None}
return
return {'type':None}
data = {}
mode = statblock[stat.ST_MODE]
......@@ -474,7 +483,7 @@ class RPath(RORPath):
type = 'sym'
data['linkname'] = self.conn.os.readlink(self.path)
elif stat.S_ISSOCK(mode): type = 'sock'
else: raise RPathException("Unknown type for %s" % self.path)
else: raise C.UnknownFileError(self.path)
data['type'] = type
data['size'] = statblock[stat.ST_SIZE]
data['perms'] = stat.S_IMODE(mode)
......@@ -487,10 +496,8 @@ class RPath(RORPath):
if not (type == 'sym' or type == 'dev'):
# mtimes on symlinks and dev files don't work consistently
data['mtime'] = long(statblock[stat.ST_MTIME])
if Globals.preserve_atime and not type == 'sym':
data['atime'] = long(statblock[stat.ST_ATIME])
self.data = data
return data
def check_consistency(self):
"""Raise an error if consistency of rp broken
......
trunk/rdiff-backup/rdiff_backup/selection.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Iterate exactly the requested files in a directory
Parses includes and excludes to yield correct files. More
documentation on what this code does can be found on the man page.
"""
from __future__ import generators
import re
from log import *
......@@ -5,14 +22,6 @@ from robust import *
from destructive_stepping import *
#######################################################################
#
# selection - Provides the iterator-like DSRPIterator class
#
# Parses includes and excludes to yield correct files. More
# documentation on what this code does can be found on the man page.
#
class SelectError(Exception):
"""Some error dealing with the Select class"""
pass
......
trunk/rdiff-backup/rdiff_backup/static.py
View file @ 00be1f4c
#######################################################################
# Copyright 2002 Ben Escoto
#
# static - MakeStatic and MakeClass
#
# These functions are used to make all the instance methods in a class
# into static or class methods.
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""MakeStatic and MakeClass
These functions are used to make all the instance methods in a class
into static or class methods.
class StaticMethodsError(Exception):
pass
"""
class StaticMethodsError(Exception): pass
def MakeStatic(cls):
"""turn instance methods into static ones
......@@ -20,7 +28,6 @@ def MakeStatic(cls):
if name[0] != "_":
cls.__dict__[name] = staticmethod(cls.__dict__[name])
def MakeClass(cls):
"""Turn instance methods into classmethods. Ignore _ like above"""
for name in dir(cls):
......
trunk/rdiff-backup/rdiff_backup/statistics.py
View file @ 00be1f4c
from lazy import *
#######################################################################
# Copyright 2002 Ben Escoto
#
# statistics - Generate and process aggregated backup information
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Generate and process aggregated backup information"""
from lazy import *
class StatsException(Exception): pass
......
trunk/rdiff-backup/src/FilenameMapping.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Coordinate corresponding files with different names
For instance, some source filenames may contain characters not allowed
on the mirror end. Also, if a source filename is very long (say 240
characters), the extra characters added to related increments may put
them over the usual 255 character limit.
"""
import re
from log import *
import Globals
#######################################################################
#
# filename_mapping - used to coordinate related filenames
#
# For instance, some source filenames may contain characters not
# allowed on the mirror end. Also, if a source filename is very long
# (say 240 characters), the extra characters added to related
# increments may put them over the usual 255 character limit.
#
"""Contains class methods which coordinate related filenames"""
max_filename_length = 255
# If true, enable character quoting, and set characters making
......
trunk/rdiff-backup/src/Globals.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Hold a variety of constants usually set at initialization."""
import re, os
# The current version of rdiff-backup
version = "0.9.0"
version = "0.9.1"
# If this is set, use this value in seconds as the current time
# instead of reading it from the clock.
......
trunk/rdiff-backup/src/Hardlink.py
View file @ 00be1f4c
from __future__ import generators
import cPickle
#######################################################################
# Copyright 2002 Ben Escoto
#
# hardlink - code for preserving and restoring hardlinks
#
# If the preserve_hardlinks option is selected, linked files in the
# source directory will be linked in the mirror directory. Linked
# files are treated like any other with respect to incrementing, but a
# database of all links will be recorded at each session, so linked
# files can still be restored from the increments.
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Preserve and restore hard links
"""Hardlink class methods and data
If the preserve_hardlinks option is selected, linked files in the
source directory will be linked in the mirror directory. Linked files
are treated like any other with respect to incrementing, but a
database of all links will be recorded at each session, so linked
files can still be restored from the increments.
All these functions are meant to be executed on the destination
side. The source side should only transmit inode information.
"""
from __future__ import generators
import cPickle
# In all of these lists of indicies are the values. The keys in
# _inode_ ones are (inode, devloc) pairs.
_src_inode_indicies = {}
......
trunk/rdiff-backup/src/Main.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Start (and end) here - read arguments, set global settings, etc."""
import getopt, sys, re
from log import *
from lazy import *
......@@ -10,10 +22,7 @@ from highlevel import *
from manage import *
import Globals, Time, SetConnections
#######################################################################
#
# main - Start here: Read arguments, set global settings, etc.
#
action = None
remote_cmd, remote_schema = None, None
force = None
......
trunk/rdiff-backup/src/MiscStats.py
View file @ 00be1f4c
from statistics import *
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Misc statistics methods, pertaining to dir and session stat files"""
from statistics import *
# This is the RPath of the directory statistics file, and the
# associated open file. It will hold a line of statistics for
# each directory that is backed up.
......
trunk/rdiff-backup/src/Rdiff.py
View file @ 00be1f4c
import os, popen2
#######################################################################
# Copyright 2002 Ben Escoto
#
# rdiff - Invoke rdiff utility to make signatures, deltas, or patch
# This file is part of rdiff-backup.
#
# All these operations should be done in a relatively safe manner
# using RobustAction and the like.
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Invoke rdiff utility to make signatures, deltas, or patch
All these operations should be done in a relatively safe manner using
RobustAction and the like.
"""
import os, popen2
class RdiffException(Exception): pass
......
trunk/rdiff-backup/src/SetConnections.py
View file @ 00be1f4c
#######################################################################
# Copyright 2002 Ben Escoto
#
# setconnections - Parse initial arguments and establish connections
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Parse args and setup connections
The methods in this class are used once by Main to parse file
descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to
set up the related connections.
The functions in this module are used once by Main to parse file
descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to set up
the related connections.
"""
class SetConnectionsException(Exception): pass
# This is the schema that determines how rdiff-backup will open a
# pipe to the remote system. If the file is given as A::B, %s will
# be substituted with A in the schema.
......@@ -24,6 +26,8 @@ __cmd_schema_no_compress = 'ssh %s rdiff-backup --server'
# The first is None because it is the local connection.
__conn_remote_cmds = [None]
class SetConnectionsException(Exception): pass
def InitRPs(arglist, remote_schema = None, remote_cmd = None):
"""Map the given file descriptions into rpaths and return list"""
global __cmd_schema
......
trunk/rdiff-backup/src/Time.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Provide time related exceptions and functions"""
import time, types, re
import Globals
#######################################################################
#
# ttime - Provide Time class, which contains time related functions.
#
class TimeException(Exception): pass
......
trunk/rdiff-backup/src/cmodule.c
View file @ 00be1f4c
/* ----------------------------------------------------------------------- *
*
* Copyright 2002 Ben Escoto
*
* This file is part of rdiff-backup.
*
* rdiff-backup is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation, Inc., 675 Mass Ave,
* Cambridge MA 02139, USA; either version 2 of the License, or (at
* your option) any later version; incorporated herein by reference.
*
* ----------------------------------------------------------------------- */
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <Python.h>
#include <errno.h>
static PyObject *UnknownFileTypeError;
static PyObject *c_make_file_dict(PyObject *self, PyObject *args);
static PyObject *long2str(PyObject *self, PyObject *args);
static PyObject *str2long(PyObject *self, PyObject *args);
/* Turn a stat structure into a python dictionary. The preprocessor
stuff taken from Python's posixmodule.c */
static PyObject *c_make_file_dict(self, args)
PyObject *self;
PyObject *args;
{
PyObject *size, *inode, *mtime, *atime, *devloc, *return_val;
char *filename, filetype[5];
struct stat sbuf;
mode_t mode;
long int mode, perms;
if (!PyArg_ParseTuple(args, "s", &filename)) return NULL;
if (lstat(filename, &sbuf) != 0) {
......@@ -21,7 +45,27 @@ static PyObject *c_make_file_dict(self, args)
return NULL;
}
}
mode = sbuf.st_mode;
#ifdef HAVE_LARGEFILE_SUPPORT
size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size);
inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino);
#else
size = PyInt_FromLong((long)sbuf.st_size);
inode = PyInt_FromLong((long)sbuf.st_ino);
#endif
mode = (long)sbuf.st_mode;
perms = mode & (S_IRWXU | S_IRWXG | S_IRWXO);
#if defined(HAVE_LONG_LONG) && !defined(MS_WINDOWS)
devloc = PyLong_FromLongLong((LONG_LONG)sbuf.st_dev);
#else
devloc = PyInt_FromLong((long)sbuf.st_dev);
#endif
#if SIZEOF_TIME_T > SIZEOF_LONG
mtime = PyLong_FromLongLong((LONG_LONG)sbuf.st_mtime);
atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime);
#else
mtime = PyInt_FromLong((long)sbuf.st_mtime);
atime = PyLong_FromLongLong((long)sbuf.st_atime);
#endif
/* Build return dictionary from stat struct */
if (S_ISREG(mode) || S_ISDIR(mode) || S_ISSOCK(mode) || S_ISFIFO(mode)) {
......@@ -30,17 +74,17 @@ static PyObject *c_make_file_dict(self, args)
else if S_ISDIR(mode) strcpy(filetype, "dir");
else if S_ISSOCK(mode) strcpy(filetype, "sock");
else strcpy(filetype, "fifo");
return Py_BuildValue("{s:s,s:l,s:i,s:i,s:i,s:l,s:i,s:i,s:l,s:l}",
return Py_BuildValue("{s:s,s:N,s:l,s:l,s:l,s:N,s:N,s:l,s:N,s:N}",
"type", filetype,
"size", (long int)sbuf.st_size,
"perms", (int)(mode & S_IRWXU),
"uid", (int)sbuf.st_uid,
"gid", (int)sbuf.st_gid,
"inode", (long int)sbuf.st_ino,
"devloc", (int)sbuf.st_dev,
"nlink", (int)sbuf.st_nlink,
"mtime", (long int)sbuf.st_mtime,
"atime", (long int)sbuf.st_atime);
"size", size,
"perms", perms,
"uid", (long)sbuf.st_uid,
"gid", (long)sbuf.st_gid,
"inode", inode,
"devloc", devloc,
"nlink", (long)sbuf.st_nlink,
"mtime", mtime,
"atime", atime);
} else if S_ISLNK(mode) {
/* Symbolic links */
char linkname[1024];
......@@ -51,89 +95,108 @@ static PyObject *c_make_file_dict(self, args)
}
linkname[len_link] = '\0';
return Py_BuildValue("{s:s,s:l,s:i,s:i,s:i,s:l,s:i,s:i,s:s}",
"type", "sym",
"size", (long int)sbuf.st_size,
"perms", (int)(mode & S_IRWXU),
"uid", (int)sbuf.st_uid,
"gid", (int)sbuf.st_gid,
"inode", (long int)sbuf.st_ino,
"devloc", (int)sbuf.st_dev,
"nlink", (int)sbuf.st_nlink,
"linkname", linkname);
return_val = Py_BuildValue("{s:s,s:N,s:l,s:l,s:l,s:N,s:N,s:l,s:s}",
"type", "sym",
"size", size,
"perms", perms,
"uid", (long)sbuf.st_uid,
"gid", (long)sbuf.st_gid,
"inode", inode,
"devloc", devloc,
"nlink", (long)sbuf.st_nlink,
"linkname", linkname);
Py_DECREF(mtime);
Py_DECREF(atime);
return return_val;
} else if (S_ISCHR(mode) || S_ISBLK(mode)) {
/* Device files */
char devtype[2];
int devnums = (int)sbuf.st_rdev;
#if defined(HAVE_LONG_LONG) && !defined(MS_WINDOWS)
LONG_LONG devnums = (LONG_LONG)sbuf.st_rdev;
PyObject *major_num = PyLong_FromLongLong(devnums >> 8);
#else
long int devnums = (long)sbuf.st_dev;
PyObject *major_num = PyInt_FromLong(devnums >> 8);
#endif
int minor_num = (int)(devnums & 0xff);
if S_ISCHR(mode) strcpy(devtype, "c");
else strcpy(devtype, "b");
return Py_BuildValue("{s:s,s:l,s:i,s:i,s:i,s:l,s:i,s:i,s:O}",
"type", "dev",
"size", (long int)sbuf.st_size,
"perms", (int)(mode & S_IRWXU),
"uid", (int)sbuf.st_uid,
"gid", (int)sbuf.st_gid,
"inode", (long int)sbuf.st_ino,
"devloc", (int)sbuf.st_dev,
"nlink", (int)sbuf.st_nlink,
"devnums", Py_BuildValue("(s,i,i)", devtype,
devnums >> 8,
devnums & 0xff),
"mtime", (long int)sbuf.st_mtime,
"atime", (long int)sbuf.st_atime);
return_val = Py_BuildValue("{s:s,s:N,s:l,s:l,s:l,s:N,s:N,s:l,s:N}",
"type", "dev",
"size", size,
"perms", perms,
"uid", (long)sbuf.st_uid,
"gid", (long)sbuf.st_gid,
"inode", inode,
"devloc", devloc,
"nlink", (long)sbuf.st_nlink,
"devnums", Py_BuildValue("(s,O,i)", devtype,
major_num, minor_num));
Py_DECREF(mtime);
Py_DECREF(atime);
return return_val;
} else {
/* Unrecognized file type - pretend it isn't there */
errno = ENOENT;
PyErr_SetFromErrno(PyExc_OSError);
/* Unrecognized file type - raise exception */
Py_DECREF(size);
Py_DECREF(inode);
Py_DECREF(devloc);
Py_DECREF(mtime);
Py_DECREF(atime);
PyErr_SetString(UnknownFileTypeError, filename);
return NULL;
}
}
/* Convert python long into 7 byte string */
static PyObject *long2str(self, args)
PyObject *self;
PyObject *args;
{
unsigned char s[7];
int sindex;
unsigned long long int l;
PyObject *pylong;
if (!PyArg_ParseTuple(args, "O", &pylong)) return NULL;
l = PyLong_AsUnsignedLongLong(pylong);
for(sindex = 0; sindex <= 6; sindex++) {
s[sindex] = l % 256;
l /= 256;
}
return Py_BuildValue("s#", s, 7);
PyLongObject *pylong;
PyObject *return_val;
if (!PyArg_ParseTuple(args, "O!", &PyLong_Type, &pylong)) return NULL;
if (_PyLong_AsByteArray(pylong, s, 7, 0, 0) != 0) return NULL;
else return Py_BuildValue("s#", s, 7);
return return_val;
}
/* Reverse of above; convert 7 byte string into python long */
static PyObject *str2long(self, args)
PyObject *self;
PyObject *args;
{
unsigned char *s;
unsigned long long int l = 0;
int sindex, ssize;
int ssize;
if (!PyArg_ParseTuple(args, "s#", &s, &ssize)) return NULL;
if (ssize != 7) return Py_BuildValue("i", -1);
for(sindex=6; sindex >= 0; sindex--)
l = l*256 + s[sindex];
return PyLong_FromLongLong(l);
if (ssize != 7) {
PyErr_SetString(PyExc_TypeError, "Single argument must be 7 char string");
return NULL;
}
return _PyLong_FromByteArray(s, 7, 0, 0);
}
static PyMethodDef CMethods[] = {
{"make_file_dict", c_make_file_dict, METH_VARARGS,
"Make dictionary from file stat"},
{"long2str", long2str, METH_VARARGS,
"Convert long int to 7 byte string"},
{"str2long", str2long, METH_VARARGS,
"Convert 7 byte string to long int"},
{"long2str", long2str, METH_VARARGS, "Convert python long to 7 byte string"},
{"str2long", str2long, METH_VARARGS, "Convert 7 byte string to python long"},
{NULL, NULL, 0, NULL}
};
void initC(void)
{
(void) Py_InitModule("C", CMethods);
PyObject *m, *d;
m = Py_InitModule("C", CMethods);
d = PyModule_GetDict(m);
UnknownFileTypeError = PyErr_NewException("C.UnknownFileTypeError",
NULL, NULL);
PyDict_SetItemString(d, "UnknownFileTypeError", UnknownFileTypeError);
}
trunk/rdiff-backup/src/compilec.py 0 → 100755
View file @ 00be1f4c
#!/usr/bin/env python
import sys, os
from distutils.core import setup, Extension
assert len(sys.argv) == 1
sys.argv.append("build")
setup(name="CModule",
version="0.9.0",
description="rdiff-backup's C component",
ext_modules=[Extension("C", ["cmodule.c"])])
assert not os.system("mv build/lib.linux-i686-2.2/C.so .")
assert not os.system("rm -rf build")
trunk/rdiff-backup/src/connection.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Support code for remote execution and data transfer"""
from __future__ import generators
import types, os, tempfile, cPickle, shutil, traceback, pickle
#######################################################################
#
# connection - Code that deals with remote execution
#
class ConnectionError(Exception): pass
class ConnectionReadError(ConnectionError): pass
class ConnectionQuit(Exception): pass
......
trunk/rdiff-backup/src/destructive_stepping.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Deal with side effects from traversing trees"""
from __future__ import generators
import types
from rpath import *
from lazy import *
#######################################################################
#
# destructive-stepping - Deal with side effects from traversing trees
#
class DSRPPermError(Exception):
"""Exception used when a DSRPath can't get sufficient permissions"""
......
trunk/rdiff-backup/src/highlevel.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""High level functions for mirroring, mirror & inc, etc."""
from __future__ import generators
from static import *
from log import *
......@@ -8,10 +20,6 @@ from destructive_stepping import *
from rorpiter import *
import Globals, Hardlink, MiscStats
#######################################################################
#
# highlevel - High level functions for mirroring, mirror & inc, etc.
#
class SkipFileException(Exception):
"""Signal that the current file should be skipped but then continue
......
trunk/rdiff-backup/src/increment.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Provides Inc and *ITR classes, which relate to writing increment files"""
import traceback
from static import *
from statistics import *
from lazy import *
#######################################################################
#
# increment - Provides Inc class, which writes increment files
#
# This code is what writes files ending in .diff, .snapshot, etc.
#
class Inc:
"""Class containing increment functions"""
......
trunk/rdiff-backup/src/iterfile.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Convert an iterator to a file object and vice-versa"""
import cPickle, array
import Globals, C
#######################################################################
#
# iterfile - Convert an iterator to a file object and vice-versa
#
class IterFileException(Exception): pass
......
trunk/rdiff-backup/src/lazy.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Define some lazy data structures and functions acting on them"""
from __future__ import generators
import os, stat, types
from static import *
#######################################################################
#
# lazy - Define some lazy data structures and functions acting on them
#
class Iter:
"""Hold static methods for the manipulation of lazy iterators"""
......@@ -210,7 +218,7 @@ class IterTreeReducer:
self.base_index = self.index = None
self.subinstances = [self]
self.finished = None
self.caught_exception = None
self.caught_exception = self.start_successful = None
def finish_subinstances(self, index):
"""Run Finish() on all subinstances index has passed
......@@ -235,9 +243,10 @@ class IterTreeReducer:
def call_end_proc(self):
"""Runs the end_process on self, checking for errors"""
if self.finished: self.caught_exception = 1
if self.caught_exception: self.log_pref_error(self.base_index)
Robust.check_common_error(self.on_error, self.end_process)
if self.finished or not self.start_successful:
self.caught_exception = 1
if self.caught_exception: self.log_prev_error(self.base_index)
else: Robust.check_common_error(self.on_error, self.end_process)
self.finished = 1
def add_subinstance(self):
......@@ -248,7 +257,9 @@ class IterTreeReducer:
def process_w_subinstance(self, index, subinst, args):
"""Run start_process on latest subinstance"""
Robust.check_common_error(self.on_error, subinst.start_process, args)
Robust.check_common_error(subinst.on_error,
subinst.start_process, args)
if not subinst.caught_exception: subinst.start_successful = 1
subinst.base_index = index
def start_process(self, *args):
......@@ -278,7 +289,8 @@ class IterTreeReducer:
def log_prev_error(self, index):
"""Call function if no pending exception"""
Log("Skipping %s because of previous error" % os.path.join(*index), 2)
Log("Skipping %s because of previous error" %
(os.path.join(*index),), 2)
def __call__(self, *args):
"""Process args, where args[0] is current position in iterator
......@@ -303,10 +315,10 @@ class IterTreeReducer:
if self.finish_subinstances(index) is None:
return None # We are no longer in the main tree
if self.caught_exception: self.log_prev_error(index)
else:
if self.subinstances[-1].start_successful:
subinst = self.add_subinstance()
self.process_w_subinstance(index, subinst, args)
else: self.log_prev_error(index)
self.index = index
return 1
......
trunk/rdiff-backup/src/log.py
View file @ 00be1f4c
import time, sys, traceback, types
#######################################################################
# Copyright 2002 Ben Escoto
#
# log - Manage logging
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Manage logging, displaying and recording messages with required verbosity"""
import time, sys, traceback, types
class LoggerError(Exception): pass
......
trunk/rdiff-backup/src/manage.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""list, delete, and otherwise manage increments"""
from __future__ import generators
from static import *
from log import *
import Globals, Time
#######################################################################
#
# manage - list, delete, and otherwise manage increments
#
class ManageException(Exception): pass
......
trunk/rdiff-backup/src/restore.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Read increment files and restore to original"""
from __future__ import generators
import tempfile
from static import *
#######################################################################
#
# restore - Read increment files and restore to original
#
class RestoreError(Exception): pass
......
trunk/rdiff-backup/src/robust.py
View file @ 00be1f4c
import tempfile, errno, signal, cPickle
from static import *
#######################################################################
#
# robust - code which prevents mirror from being corrupted, error-recovery
# Copyright 2002 Ben Escoto
#
# Ideally no matter an instance of rdiff-backup gets aborted, no
# information should get lost. The target directory should be left in
# a coherent state, and later instances of rdiff-backup should clean
# things up so there is no sign that anything ever got aborted or
# failed.
# This file is part of rdiff-backup.
#
# Thus, files should be updated in an atomic way as possible. Each
# file should be updated (and the corresponding diff files written) or
# not, and it should be clear which happened. In general, I don't
# think this is possible, since the creation of the diff files and the
# changing of updated files cannot be guarateed to happen together.
# It is possible, I think, to record various information to files
# which would allow a later process to figure out what the last
# operation was, but this would add several file operations to the
# processing of each file, and I don't think, would be a good
# tradeoff.
#
# The compromise reached here is that diff files should be created
# just before the mirror files are updated, and each file update
# should be done with a rename operation on a file in the same
# directory. Furthermore, every once in a while, rdiff-backup will
# record which file it just finished processing. If any fatal errors
# are caught, it will also record the last processed file. Future
# instances may not know exactly when the previous instance was
# aborted, but they will be able to narrow down the possibilities.
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Prevent mirror from being corrupted; handle errors
Ideally no matter an instance of rdiff-backup gets aborted, no
information should get lost. The target directory should be left in a
coherent state, and later instances of rdiff-backup should clean
things up so there is no sign that anything ever got aborted or
failed.
Thus, files should be updated in an atomic way as possible. Each file
should be updated (and the corresponding diff files written) or not,
and it should be clear which happened. In general, I don't think this
is possible, since the creation of the diff files and the changing of
updated files cannot be guarateed to happen together. It is possible,
I think, to record various information to files which would allow a
later process to figure out what the last operation was, but this
would add several file operations to the processing of each file, and
I don't think, would be a good tradeoff.
The compromise reached here is that diff files should be created just
before the mirror files are updated, and each file update should be
done with a rename operation on a file in the same directory.
Furthermore, every once in a while, rdiff-backup will record which
file it just finished processing. If any fatal errors are caught, it
will also record the last processed file. Future instances may not
know exactly when the previous instance was aborted, but they will be
able to narrow down the possibilities.
"""
import tempfile, errno, signal, cPickle, C
from static import *
class RobustAction:
"""Represents a file operation to be accomplished later"""
......@@ -243,7 +253,8 @@ class Robust:
"""
try: return function(*args)
except (EnvironmentError, SkipFileException, DSRPPermError,
RPathException, Rdiff.RdiffException), exc:
RPathException, Rdiff.RdiffException,
C.UnknownFileTypeError), exc:
TracebackArchive.add()
if (not isinstance(exc, EnvironmentError) or
(errno.errorcode[exc[0]] in
......
trunk/rdiff-backup/src/rorpiter.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Operations on Iterators of Read Only Remote Paths"""
from __future__ import generators
import tempfile, UserList, types
from static import *
......@@ -7,10 +19,6 @@ from robust import *
from iterfile import *
import Globals, Rdiff, Hardlink
#######################################################################
#
# rorpiter - Operations on Iterators of Read Only Remote Paths
#
class RORPIterException(Exception): pass
......
trunk/rdiff-backup/src/rpath.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Wrapper class around a real path like "/usr/bin/env"
The RPath and associated classes make some function calls more
convenient (e.g. RPath.getperms()) and also make working with files on
remote systems transparent.
"""
import os, stat, re, sys, shutil, gzip
from static import *
#######################################################################
#
# rpath - Wrapper class around a real path like "/usr/bin/env"
#
# The RPath and associated classes make some function calls more
# convenient (e.g. RPath.getperms()) and also make working with files
# on remote systems transparent.
#
class RPathException(Exception): pass
......@@ -452,12 +462,11 @@ class RPath(RORPath):
"""Set data dictionary using C extension"""
self.data = self.conn.C.make_file_dict(self.path)
def setdata_old(self):
def make_file_dict_old(self):
"""Create the data dictionary"""
statblock = self.conn.RPathStatic.tupled_lstat(self.path)
if statblock is None:
self.data = {'type':None}
return
return {'type':None}
data = {}
mode = statblock[stat.ST_MODE]
......@@ -474,7 +483,7 @@ class RPath(RORPath):
type = 'sym'
data['linkname'] = self.conn.os.readlink(self.path)
elif stat.S_ISSOCK(mode): type = 'sock'
else: raise RPathException("Unknown type for %s" % self.path)
else: raise C.UnknownFileError(self.path)
data['type'] = type
data['size'] = statblock[stat.ST_SIZE]
data['perms'] = stat.S_IMODE(mode)
......@@ -487,10 +496,8 @@ class RPath(RORPath):
if not (type == 'sym' or type == 'dev'):
# mtimes on symlinks and dev files don't work consistently
data['mtime'] = long(statblock[stat.ST_MTIME])
if Globals.preserve_atime and not type == 'sym':
data['atime'] = long(statblock[stat.ST_ATIME])
self.data = data
return data
def check_consistency(self):
"""Raise an error if consistency of rp broken
......
trunk/rdiff-backup/src/selection.py
View file @ 00be1f4c
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Iterate exactly the requested files in a directory
Parses includes and excludes to yield correct files. More
documentation on what this code does can be found on the man page.
"""
from __future__ import generators
import re
from log import *
......@@ -5,14 +22,6 @@ from robust import *
from destructive_stepping import *
#######################################################################
#
# selection - Provides the iterator-like DSRPIterator class
#
# Parses includes and excludes to yield correct files. More
# documentation on what this code does can be found on the man page.
#
class SelectError(Exception):
"""Some error dealing with the Select class"""
pass
......
trunk/rdiff-backup/src/static.py
View file @ 00be1f4c
#######################################################################
# Copyright 2002 Ben Escoto
#
# static - MakeStatic and MakeClass
#
# These functions are used to make all the instance methods in a class
# into static or class methods.
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""MakeStatic and MakeClass
These functions are used to make all the instance methods in a class
into static or class methods.
class StaticMethodsError(Exception):
pass
"""
class StaticMethodsError(Exception): pass
def MakeStatic(cls):
"""turn instance methods into static ones
......@@ -20,7 +28,6 @@ def MakeStatic(cls):
if name[0] != "_":
cls.__dict__[name] = staticmethod(cls.__dict__[name])
def MakeClass(cls):
"""Turn instance methods into classmethods. Ignore _ like above"""
for name in dir(cls):
......
trunk/rdiff-backup/src/statistics.py
View file @ 00be1f4c
from lazy import *
#######################################################################
# Copyright 2002 Ben Escoto
#
# statistics - Generate and process aggregated backup information
# This file is part of rdiff-backup.
#
# rdiff-backup is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
# 02139, USA; either version 2 of the License, or (at your option) any
# later version; incorporated herein by reference.
"""Generate and process aggregated backup information"""
from lazy import *
class StatsException(Exception): pass
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7