Skip to content

S
slapos
Commit 9c59c5fc authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

software/theia: Fix resilient signature checking

parent 5763522b
Hide whitespace changes
Inline Side-by-side
Showing with 203 additions and 116 deletions
software/theia/buildout.hash.cfg
View file @ 9c59c5fc
......@@ -35,15 +35,15 @@ md5sum = d78a9f885bdebf6720197209e0c21aa0
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = 38eba0fb605953677b5a2a2e686a66a2
md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
[theia-export]
_update_hash_filename_ = theia_export.py
md5sum = d338b2d3ba1dcd7a919c0baf51dd11da
md5sum = e2f6c483cce09f87ab1e63ae8be0daf4
[theia-import]
_update_hash_filename_ = theia_import.py
md5sum = 3580f1eec1099bebd550ba1eacd9c116
md5sum = 5dea99b0106cccba65f8ae90d110e111
[yarn.lock]
_update_hash_filename_ = yarn.lock
......
software/theia/theia_common.py
View file @ 9c59c5fc
......@@ -81,7 +81,6 @@ def remove(path):
def parse_installed(partition):
paths = []
custom_script = os.path.join(partition, 'srv', '.backup_identity_script')
for cfg in glob.glob(os.path.join(partition, '.installed*.cfg')):
try:
with open(cfg) as f:
......@@ -93,7 +92,7 @@ def parse_installed(partition):
for section in six.itervalues(installed_cfg):
for p in section.get('__buildout_installed__', '').splitlines():
p = p.strip()
if p and p != custom_script:
if p:
paths.append(p)
return paths
......@@ -108,31 +107,44 @@ def sha256sum(file_path, chunk_size=1024 * 1024):
return sha256.hexdigest()
def hashwalk(backup_dir, mirror_partitions):
scripts = {}
for p in mirror_partitions:
script_path = os.path.join(p, 'srv', '.backup_identity_script')
if os.path.exists(script_path):
scripts[os.path.abspath(p)] = script_path
for dirpath, dirnames, filenames in os.walk(backup_dir):
filenames.sort()
def fast_hashwalk(root_dir):
for dirpath, dirnames, filenames in os.walk(root_dir):
for f in filenames:
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath):
displaypath = os.path.relpath(filepath, start=backup_dir)
displaypath = os.path.relpath(filepath, start=root_dir)
yield '%s %s' % (sha256sum(filepath), displaypath)
remaining_dirnames = []
for subdir in dirnames:
subdirpath = os.path.abspath(os.path.join(dirpath, subdir))
custom_hashscript = scripts.get(subdirpath)
if custom_hashscript:
print('Using custom signature script %s' % custom_hashscript)
for s in hashcustom(subdirpath, backup_dir, custom_hashscript):
yield s
else:
remaining_dirnames.append(subdir)
remaining_dirnames.sort()
dirnames[:] = remaining_dirnames
def exclude_hashwalk(root_dir, instance_dir):
root_dir = os.path.abspath(root_dir)
instance_dir = os.path.abspath(instance_dir)
for dirpath, dirnames, filenames in os.walk(root_dir):
for f in filenames:
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath):
displaypath = os.path.relpath(filepath, start=root_dir)
yield '%s %s' % (sha256sum(filepath), displaypath)
if dirpath == instance_dir:
remaining_dirs = []
for d in dirnames:
if not d.startswith('slappart'):
remaining_dirs.append(d)
dirnames[:] = remaining_dirs
def hashwalk(root_dir, instance_dir=None):
if instance_dir and not os.path.relpath(
instance_dir, start=root_dir).startswith(os.pardir):
return exclude_hashwalk(root_dir, instance_dir)
return fast_hashwalk(root_dir)
def hashscript(partition):
script = os.path.join(partition, 'srv', '.backup_identity_script')
if os.path.exists(script):
return script
return None
@contextlib.contextmanager
......@@ -145,10 +157,11 @@ def cwd(path):
os.chdir(old_path)
def hashcustom(mirrordir, backup_dir, custom_hashscript):
workingdir = os.path.join(mirrordir, os.pardir, os.pardir, os.pardir)
def hashcustom(partition, script):
workingdir = os.path.join(partition, os.pardir, os.pardir, os.pardir)
with cwd(os.path.abspath(workingdir)):
for dirpath, _, filenames in os.walk(mirrordir):
for dirpath, dirnames, filenames in os.walk(partition):
dirnames.sort()
filepaths = []
for f in filenames:
path = os.path.join(dirpath, f)
......@@ -157,16 +170,16 @@ def hashcustom(mirrordir, backup_dir, custom_hashscript):
if not filepaths:
continue
hashprocess = sp.Popen(
custom_hashscript, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
script, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = hashprocess.communicate(str2bytes('\0'.join(filepaths)))
if hashprocess.returncode != 0:
template = "Custom signature script %s failed on inputs:\n%s"
msg = template % (custom_hashscript, '\n'.join(filepaths))
msg = template % (script, '\n'.join(filepaths))
msg += "\nwith stdout:\n%s" % bytes2str(out)
msg += "\nand stderr:\n%s" % bytes2str(err)
raise Exception(msg)
signatures = bytes2str(out).strip('\n').split('\n')
signatures.sort()
displaypath = os.path.relpath(dirpath, start=backup_dir)
displaypath = os.path.relpath(dirpath, start=partition)
for s in signatures:
yield '%s %s/ (custom)' % (s, displaypath)
yield '%s %s' % (s, displaypath)
software/theia/theia_export.py
View file @ 9c59c5fc
......@@ -55,49 +55,74 @@ class TheiaExport(object):
self.copytree_partitions_args = {}
self.logs = []
def mirrorpath(self, src):
def mirror_path(self, src):
return os.path.abspath(os.path.join(
self.backup_dir, os.path.relpath(src, start=self.root_dir)))
def backuptree(self, src, exclude=(), extrargs=(), verbosity='-v'):
dst = self.mirrorpath(src)
return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
def backup_tree(self, src):
return copytree(self.rsync_bin, src, self.mirror_path(src))
def backupfile(self, src):
dst = self.mirrorpath(src)
return copyfile(src, dst)
def backup_file(self, src):
return copyfile(src, self.mirror_path(src))
def backupdb(self):
copydb(self.sqlite3_bin, self.proxy_db, self.mirrorpath(self.proxy_db))
def backup_db(self):
copydb(self.sqlite3_bin, self.proxy_db, self.mirror_path(self.proxy_db))
def backuppartition(self, partition):
def backup_partition(self, partition):
installed = parse_installed(partition)
rules = os.path.join(partition, 'srv', 'exporter.exclude')
extrargs = ('--filter=.-/ ' + rules,) if os.path.exists(rules) else ()
self.backuptree(partition, exclude=installed, extrargs=extrargs)
self.copytree_partitions_args[partition] = (installed, extrargs)
dst = self.mirror_path(partition)
copytree(self.rsync_bin, partition, dst, installed, extrargs)
self.copytree_partitions_args[partition] = (dst, installed, extrargs)
def sign(self, signaturefile):
def sign(self, signaturefile, signatures):
remove(signaturefile)
pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
tmpfile = os.path.join(pardir, 'backup.signature.tmp')
mirror_partitions = [self.mirrorpath(p) for p in self.partition_dirs]
tmpfile = os.path.join(pardir, os.path.basename(signaturefile) + '.tmp')
with open(tmpfile, 'w') as f:
for s in hashwalk(self.backup_dir, mirror_partitions):
for s in signatures:
f.write(s + '\n')
os.rename(tmpfile, signaturefile)
def checkpartition(self, partition, pattern='/srv/backup/'):
installed, extrargs = self.copytree_partitions_args[partition]
output = self.backuptree(
def sign_root(self):
signaturefile = os.path.join(self.backup_dir, 'backup.signature')
signatures = hashwalk(self.backup_dir, self.mirror_path(self.instance_dir))
self.sign(signaturefile, signatures)
def sign_partition(self, partition):
dst = self.mirror_path(partition)
filename = os.path.basename(partition) + '.backup.signature'
signaturefile = os.path.join(self.backup_dir, filename)
script = hashscript(partition)
if script:
signaturefile += '.custom'
self.sign(signaturefile, hashcustom(dst, script))
else:
self.sign(signaturefile, hashwalk(dst))
def remove_signatures(self):
pattern = os.path.join(self.backup_dir, '*backup.signature*')
signature_files = glob.glob(pattern)
for f in signature_files:
try:
os.remove(f)
except OSError:
pass
def check_partition(self, partition, pattern='/srv/backup/'):
dst, installed, extrargs = self.copytree_partitions_args[partition]
output = copytree(
self.rsync_bin,
partition,
dst,
exclude=installed,
extrargs=extrargs + ('--dry-run', '--update'),
verbosity='--out-format=%n',
)
return [path for path in output.splitlines() if pattern in path]
def loginfo(self, msg):
def log(self, msg):
print(msg)
self.logs.append(msg)
......@@ -126,36 +151,42 @@ class TheiaExport(object):
with open(timestamp, 'w') as f:
f.write(str(export_start_date))
self.loginfo('Backup resilient timestamp ' + timestamp)
self.backupfile(timestamp)
self.remove_signatures()
self.log('Backup resilient timestamp ' + timestamp)
self.backup_file(timestamp)
for d in self.dirs:
self.loginfo('Backup directory ' + d)
self.backuptree(d)
self.log('Backup directory ' + d)
self.backup_tree(d)
self.loginfo('Backup slapproxy database')
self.backupdb()
self.log('Backup slapproxy database')
self.backup_db()
self.loginfo('Backup partitions')
self.log('Backup partitions')
for p in self.partition_dirs:
self.backuppartition(p)
self.backup_partition(p)
self.log('Compute root backup signature')
self.sign_root()
self.loginfo('Compute backup signature')
self.sign(os.path.join(self.backup_dir, 'backup.signature'))
self.log('Compute partitions backup signatures')
for p in self.partition_dirs:
self.sign_partition(p)
time.sleep(10)
self.loginfo('Check partitions')
self.log('Check partitions')
modified = list(itertools.chain.from_iterable(
self.checkpartition(p) for p in self.partition_dirs))
self.check_partition(p) for p in self.partition_dirs))
if modified:
msg = 'Some files have been modified since the backup started'
self.loginfo(msg + ':')
self.loginfo('\n'.join(modified))
self.loginfo("Let's wait %d minutes and try again" % BACKUP_WAIT)
self.log(msg + ':')
self.log('\n'.join(modified))
self.log("Let's wait %d minutes and try again" % BACKUP_WAIT)
time.sleep(BACKUP_WAIT * 60)
raise Exception(msg)
self.loginfo('Done')
self.log('Done')
if __name__ == '__main__':
......
software/theia/theia_import.py
View file @ 9c59c5fc
......@@ -57,32 +57,32 @@ class TheiaImport(object):
configp.read(cfg)
self.proxy_db = configp.get('slapproxy', 'database_uri')
self.instance_dir = configp.get('slapos', 'instance_root')
mirror_dir = self.mirrorpath(self.instance_dir)
mirror_dir = self.mirror_path(self.instance_dir)
partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
self.mirror_partition_dirs = [p for p in partitions if os.path.isdir(p)]
self.logs = []
def mirrorpath(self, dst):
def mirror_path(self, dst):
return os.path.abspath(os.path.join(
self.backup_dir, os.path.relpath(dst, start=self.root_dir)))
def dstpath(self, src):
def dst_path(self, src):
return os.path.abspath(os.path.join(
self.root_dir, os.path.relpath(src, start=self.backup_dir)))
def restoretree(self, dst, exclude=(), extrargs=(), verbosity='-v'):
src = self.mirrorpath(dst)
def restore_tree(self, dst, exclude=(), extrargs=(), verbosity='-v'):
src = self.mirror_path(dst)
return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
def restorefile(self, dst):
src = self.mirrorpath(dst)
def restore_file(self, dst):
src = self.mirror_path(dst)
return copyfile(src, dst)
def restoredb(self):
copydb(self.sqlite3_bin, self.mirrorpath(self.proxy_db), self.proxy_db)
def restore_db(self):
copydb(self.sqlite3_bin, self.mirror_path(self.proxy_db), self.proxy_db)
def restorepartition(self, mirror_partition):
p = self.dstpath(mirror_partition)
def restore_partition(self, mirror_partition):
p = self.dst_path(mirror_partition)
installed = parse_installed(p) if os.path.exists(p) else []
copytree(self.rsync_bin, mirror_partition, p, exclude=installed)
......@@ -97,31 +97,60 @@ class TheiaImport(object):
print(' '.join(command))
sp.check_call(command)
def verify(self, signaturefile):
pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
moved = os.path.join(pardir, 'backup.signature.moved')
proof = os.path.join(pardir, 'backup.signature.proof')
if os.path.exists(signaturefile):
os.rename(signaturefile, moved)
if not os.path.exists(moved):
msg = 'ERROR the backup signature file is missing'
print(msg)
def sign(self, signaturefile, root_dir):
with open(signaturefile, 'r') as f:
for line in f:
try:
_, relpath = line.strip().split(None, 1)
except ValueError:
yield 'Could not parse: %s' % line
continue
filepath = os.path.join(root_dir, relpath)
try:
signature = sha256sum(filepath)
except IOError:
yield 'Could not read: %s' % filepath
continue
yield '%s %s' % (signature, relpath)
def sign_custom(self, root_dir):
partition = self.dst_path(root_dir)
script = hashscript(partition)
if not script:
msg = 'ERROR: missing custom signature script for partition ' + partition
raise Exception(msg)
return hashcustom(root_dir, script)
def find_signature_file(self, partition):
filename = os.path.basename(partition) + '.backup.signature'
signaturefile = os.path.join(self.backup_dir, filename)
if os.path.exists(signaturefile):
return signaturefile, False
signaturefile += '.custom'
if os.path.exists(signaturefile):
return signaturefile, True
raise Exception('ERROR: missing signature file for partition ' + partition)
def verify(self, signaturefile, root_dir, custom=False):
proof = signaturefile + '.proof'
if custom:
signatures = self.sign_custom(root_dir)
else:
signatures = self.sign(signaturefile, root_dir)
with open(proof, 'w') as f:
for s in hashwalk(self.backup_dir, self.mirror_partition_dirs):
for s in signatures:
f.write(s + '\n')
diffcommand = ('diff', moved, proof)
print(' '.join(diffcommand))
diffcommand = ('diff', signaturefile, proof)
try:
sp.check_output(
diffcommand, stderr=sp.STDOUT, universal_newlines=True)
except sp.CalledProcessError as e:
template = 'ERROR the backup signatures do not match\n\n%s'
msg = template % e.output
template = 'ERROR the backup signatures do not match\n\n%s\n%s'
msg = template % (' '.join(diffcommand), e.output)
print(msg)
raise Exception(msg)
def loginfo(self, msg):
def log(self, msg):
print(msg)
self.logs.append(msg)
......@@ -144,44 +173,54 @@ class TheiaImport(object):
sys.exit(exitcode)
def restore(self):
self.loginfo('Verify backup signature')
self.verify(os.path.join(self.backup_dir, 'backup.signature'))
self.log('Verify main backup signature')
signaturefile = os.path.join(self.backup_dir, 'backup.signature')
self.verify(signaturefile, self.backup_dir)
self.loginfo('Stop slapproxy')
custom_partition_signatures = []
for m in self.mirror_partition_dirs:
signaturefile, custom = self.find_signature_file(m)
if custom:
custom_partition_signatures.append((signaturefile, m))
else:
self.log('Verify backup signature for ' + m)
self.verify(signaturefile, m)
self.log('Stop slapproxy')
self.supervisorctl('stop', 'slapos-proxy')
self.loginfo('Restore partitions')
self.log('Restore partitions')
for m in self.mirror_partition_dirs:
self.restorepartition(m)
self.restore_partition(m)
for d in self.dirs:
self.loginfo('Restore directory ' + d)
self.restoretree(d)
self.log('Restore directory ' + d)
self.restore_tree(d)
self.loginfo('Restore slapproxy database')
self.restoredb()
self.log('Restore slapproxy database')
self.restore_db()
timestamp = os.path.join(self.root_dir, 'etc', '.resilient_timestamp')
self.loginfo('Restore resilient timestamp ' + timestamp)
self.restorefile(timestamp)
self.log('Restore resilient timestamp ' + timestamp)
self.restore_file(timestamp)
custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
if os.path.exists(custom_script):
self.loginfo('Run custom restore script %s' % custom_script)
self.log('Run custom restore script %s' % custom_script)
sp.check_call(custom_script)
self.loginfo('Start slapproxy again')
self.log('Start slapproxy again')
self.supervisorctl('start', 'slapos-proxy')
self.loginfo('Reformat partitions')
self.log('Reformat partitions')
self.slapos('node', 'format', '--now')
self.loginfo('Remove old supervisord configuration files')
self.log('Remove old supervisord configuration files')
conf_dir = os.path.join(self.instance_dir, 'etc', 'supervisor.conf.d')
for f in glob.glob(os.path.join(conf_dir, '*')):
os.remove(f)
self.loginfo('Build Software Releases')
self.log('Build Software Releases')
for i in range(3):
try:
self.slapos('node', 'software', '--all', '--logfile', self.sr_log)
......@@ -191,18 +230,18 @@ class TheiaImport(object):
else:
break
self.loginfo('Remove old custom instance scripts')
self.log('Remove old custom instance scripts')
partitions_glob = os.path.join(self.instance_dir, 'slappart*')
scripts = os.path.join(partitions_glob, 'srv', 'runner-import-restore')
for f in glob.glob(scripts):
remove(f)
self.loginfo('Remove partition timestamps')
self.log('Remove partition timestamps')
timestamps = os.path.join(partitions_glob, '.timestamp')
for f in glob.glob(timestamps):
remove(f)
self.loginfo('Build Instances')
self.log('Build Instances')
cp_log = self.cp_log
for i in range(3):
try:
......@@ -213,11 +252,15 @@ class TheiaImport(object):
else:
break
self.log('Verify custom backup signatures')
for signaturefile, m in custom_partition_signatures:
self.verify(signaturefile, m, True)
for custom_script in glob.glob(scripts):
self.loginfo('Running custom instance script %s' % custom_script)
self.log('Running custom instance script %s' % custom_script)
sp.check_call(custom_script)
self.loginfo('Done')
self.log('Done')
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7