Commit bc3008c6 authored by Marius Gedminas's avatar Marius Gedminas

repozo --verify now correctly handles gzipped backups

parent 15b62484
......@@ -287,6 +287,22 @@ def checksum(fp, n):
return sum.hexdigest()
def file_size(fp):
# Compute number of bytes that can be read from fp
def func(data):
pass
return dofile(func, fp, None)
def checksum_and_size(fp):
# Checksum and return it with the size of the file
sum = md5()
def func(data):
sum.update(data)
size = dofile(func, fp, None)
return sum.hexdigest(), size
def copyfile(options, dst, start, n):
# Copy bytes from file src, to file dst, starting at offset start, for n
# length of bytes. For robustness, we first write, flush and fsync
......@@ -654,21 +670,39 @@ def do_verify(options):
os.path.basename(fn))
expected_size = endpos - startpos
log("Verifying %s", filename)
# XXX: if the file is gzipped, we need to unzip it
try:
if filename.endswith('fsz'):
fp = gzip.open(filename, 'rb')
when_uncompressed = ' (when uncompressed)'
else:
fp = open(filename, 'rb')
when_uncompressed = ''
except IOError:
error("%s is missing", filename)
continue
try:
fp.seek(0, 2)
except ValueError:
# can't seek in gzipped files
if options.quick:
size = file_size(fp)
actual_sum = None
else:
size = os.fstat(fp.fileno()).st_size
actual_sum, size = checksum_and_size(fp)
else:
size = fp.tell()
if options.quick or size != expected_size:
actual_sum = None
else:
fp.seek(0)
actual_sum = checksum(fp, size)
if size != expected_size:
error("%s is %d bytes, should be %d bytes", filename,
size, expected_size)
error("%s is %d bytes%s, should be %d bytes", filename,
size, when_uncompressed, expected_size)
elif not options.quick:
actual_sum = checksum(fp, size)
if actual_sum != sum:
error("%s has checksum %s instead of %s", filename,
actual_sum, sum)
error("%s has checksum %s%s instead of %s", filename,
actual_sum, when_uncompressed, sum)
fp.close()
......
......@@ -14,12 +14,7 @@
from __future__ import print_function
import unittest
import os
try:
# the hashlib package is available from Python 2.5
from hashlib import md5
except ImportError:
# the md5 package is deprecated in Python 2.6
from md5 import new as md5
from hashlib import md5
import ZODB.tests.util # layer used at class scope
......@@ -812,7 +807,13 @@ class Test_do_verify(OptionsTestBase, unittest.TestCase):
if text is None:
text = name
fqn = os.path.join(self._repository_directory, name)
f = _write_file(fqn, text.encode())
if ext.endswith('fsz'):
_opener = _GzipCloser
else:
_opener = open
with _opener(fqn, 'wb') as f:
f.write(text)
f.flush()
return fqn
def test_no_files(self):
......@@ -829,6 +830,15 @@ class Test_do_verify(OptionsTestBase, unittest.TestCase):
'/backup/2010-05-14-04-05-06.deltafs 3 7 f50881ced34c7d9e6bce100bf33dec60\n')
self.assertEqual(self._callFUT(options), [])
def test_all_is_fine_gzip(self):
options = self._makeOptions(quick=False)
self._makeFile(2, 3, 4, '.fsz', 'AAA')
self._makeFile(4, 5, 6, '.deltafsz', 'BBBB')
self._makeFile(2, 3, 4, '.dat',
'/backup/2010-05-14-02-03-04.fsz 0 3 e1faffb3e614e6c2fba74296962386b7\n'
'/backup/2010-05-14-04-05-06.deltafsz 3 7 f50881ced34c7d9e6bce100bf33dec60\n')
self.assertEqual(self._callFUT(options), [])
def test_missing_file(self):
options = self._makeOptions(quick=True)
self._makeFile(2, 3, 4, '.fs', 'AAA')
......@@ -839,6 +849,16 @@ class Test_do_verify(OptionsTestBase, unittest.TestCase):
[options.repository + os.path.sep +
'2010-05-14-04-05-06.deltafs is missing'])
def test_missing_file_gzip(self):
options = self._makeOptions(quick=True)
self._makeFile(2, 3, 4, '.fsz', 'AAA')
self._makeFile(2, 3, 4, '.dat',
'/backup/2010-05-14-02-03-04.fsz 0 3 e1faffb3e614e6c2fba74296962386b7\n'
'/backup/2010-05-14-04-05-06.deltafsz 3 7 f50881ced34c7d9e6bce100bf33dec60\n')
self.assertEqual(self._callFUT(options),
[options.repository + os.path.sep +
'2010-05-14-04-05-06.deltafsz is missing'])
def test_bad_size(self):
options = self._makeOptions(quick=False)
self._makeFile(2, 3, 4, '.fs', 'AAA')
......@@ -851,6 +871,18 @@ class Test_do_verify(OptionsTestBase, unittest.TestCase):
'2010-05-14-04-05-06.deltafs is 3 bytes,'
' should be 4 bytes'])
def test_bad_size_gzip(self):
options = self._makeOptions(quick=False)
self._makeFile(2, 3, 4, '.fsz', 'AAA')
self._makeFile(4, 5, 6, '.deltafsz', 'BBB')
self._makeFile(2, 3, 4, '.dat',
'/backup/2010-05-14-02-03-04.fsz 0 3 e1faffb3e614e6c2fba74296962386b7\n'
'/backup/2010-05-14-04-05-06.deltafsz 3 7 f50881ced34c7d9e6bce100bf33dec60\n')
self.assertEqual(self._callFUT(options),
[options.repository + os.path.sep +
'2010-05-14-04-05-06.deltafsz is 3 bytes (when uncompressed),'
' should be 4 bytes'])
def test_bad_checksum(self):
options = self._makeOptions(quick=False)
self._makeFile(2, 3, 4, '.fs', 'AAA')
......@@ -864,6 +896,19 @@ class Test_do_verify(OptionsTestBase, unittest.TestCase):
' 36486440db255f0ee6ab109d5d231406 instead of'
' f50881ced34c7d9e6bce100bf33dec60'])
def test_bad_checksum_gzip(self):
options = self._makeOptions(quick=False)
self._makeFile(2, 3, 4, '.fsz', 'AAA')
self._makeFile(4, 5, 6, '.deltafsz', 'BbBB')
self._makeFile(2, 3, 4, '.dat',
'/backup/2010-05-14-02-03-04.fsz 0 3 e1faffb3e614e6c2fba74296962386b7\n'
'/backup/2010-05-14-04-05-06.deltafsz 3 7 f50881ced34c7d9e6bce100bf33dec60\n')
self.assertEqual(self._callFUT(options),
[options.repository + os.path.sep +
'2010-05-14-04-05-06.deltafsz has checksum'
' 36486440db255f0ee6ab109d5d231406 (when uncompressed) instead of'
' f50881ced34c7d9e6bce100bf33dec60'])
def test_quick_ignores_checksums(self):
options = self._makeOptions(quick=True)
self._makeFile(2, 3, 4, '.fs', 'AAA')
......@@ -873,6 +918,15 @@ class Test_do_verify(OptionsTestBase, unittest.TestCase):
'/backup/2010-05-14-04-05-06.deltafs 3 7 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n')
self.assertEqual(self._callFUT(options), [])
def test_quick_ignores_checksums_gzip(self):
options = self._makeOptions(quick=True)
self._makeFile(2, 3, 4, '.fsz', 'AAA')
self._makeFile(4, 5, 6, '.deltafsz', 'BBBB')
self._makeFile(2, 3, 4, '.dat',
'/backup/2010-05-14-02-03-04.fsz 0 3 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n'
'/backup/2010-05-14-04-05-06.deltafsz 3 7 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n')
self.assertEqual(self._callFUT(options), [])
class MonteCarloTests(unittest.TestCase):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment