SVN: ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/ Add tests for 'do_recover'; add support for saving / restoring index.
Tres Seaver <[email protected]>
| Newsgroups | gmane.comp.python.zope.zodb.cvs |
|---|---|
| Message-ID | <[email protected]> |
Log message for revision 112339:
Add tests for 'do_recover'; add support for saving / restoring index.
Changed:
U ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/repozo.py
U ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/tests/test_repozo.py
-=-
Modified: ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/repozo.py
===================================================================
--- ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/repozo.py 2010-05-15 16:04:20 UTC (rev 112338)
+++ ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/repozo.py 2010-05-15 17:15:09 UTC (rev 112339)
@@ -69,6 +69,7 @@
"""
import os
+import shutil
import sys
try:
# the hashlib package is available from Python 2.5
@@ -96,6 +97,11 @@
class WouldOverwriteFiles(Exception):
pass
+
+class NoFiles(Exception):
+ pass
+
+
def usage(code, msg=''):
outfp = sys.stderr
if code == 0:
@@ -412,6 +418,10 @@
os.unlink(os.path.join(options.repository, fname))
def do_full_backup(options):
+ options.full = True
+ dest = os.path.join(options.repository, gen_filename(options))
+ if os.path.exists(dest):
+ raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
# Find the file position of the last completed transaction.
fs = FileStorage(options.file, read_only=True)
# Note that the FileStorage ctor calls read_index() which scans the file
@@ -420,11 +430,12 @@
# because we only want to copy stuff from the beginning of the file to the
# last valid transaction record.
pos = fs.getSize()
+ # Save the storage index into the repository
+ index_file = os.path.join(options.repository,
+ gen_filename(options, '.index'))
+ log('writing index')
+ fs._index.save(pos, index_file)
fs.close()
- options.full = True
- dest = os.path.join(options.repository, gen_filename(options))
- if os.path.exists(dest):
- raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
log('writing full backup: %s bytes to %s', pos, dest)
sum = copyfile(options, dest, 0, pos)
# Write the data file for this full backup
@@ -439,6 +450,10 @@
def do_incremental_backup(options, reposz, repofiles):
+ options.full = False
+ dest = os.path.join(options.repository, gen_filename(options))
+ if os.path.exists(dest):
+ raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
# Find the file position of the last completed transaction.
fs = FileStorage(options.file, read_only=True)
# Note that the FileStorage ctor calls read_index() which scans the file
@@ -447,11 +462,11 @@
# because we only want to copy stuff from the beginning of the file to the
# last valid transaction record.
pos = fs.getSize()
+ log('writing index')
+ index_file = os.path.join(options.repository,
+ gen_filename(options, '.index'))
+ fs._index.save(pos, index_file)
fs.close()
- options.full = False
- dest = os.path.join(options.repository, gen_filename(options))
- if os.path.exists(dest):
- raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
log('writing incremental: %s bytes to %s', pos-reposz, dest)
sum = copyfile(options, dest, reposz, pos - reposz)
# The first file in repofiles points to the last full backup. Use this to
@@ -552,10 +567,9 @@
repofiles = find_files(options)
if not repofiles:
if options.date:
- log('No files in repository before %s', options.date)
+ raise NoFiles('No files in repository before %s', options.date)
else:
- log('No files in repository')
- return
+ raise NoFiles('No files in repository')
if options.output is None:
log('Recovering file to stdout')
outfp = sys.stdout
@@ -567,7 +581,17 @@
outfp.close()
log('Recovered %s bytes, md5: %s', reposz, reposum)
+ if options.output is not None:
+ last_base = os.path.splitext(repofiles[-1])[0]
+ source_index = '%s.index' % last_base
+ target_index = '%s.index' % options.output
+ if os.path.exists(source_index):
+ log('Restoring index file %s to %s', source_index, target_index)
+ shutil.copyfile(source_index, target_index)
+ else:
+ log('No index file to restore: %s', source_index)
+
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
@@ -577,10 +601,14 @@
do_backup(options)
except WouldOverwriteFiles, e:
print >> sys.stderr, str(e)
- sys.exit(2)
+ sys.exit(1)
else:
assert options.mode == RECOVER
- do_recover(options)
+ try:
+ do_recover(options)
+ except NoFiles, e:
+ print >> sys.stderr, str(e)
+ sys.exit(1)
if __name__ == '__main__':
Modified: ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/tests/test_repozo.py
===================================================================
--- ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/tests/test_repozo.py 2010-05-15 16:04:20 UTC (rev 112338)
+++ ZODB/branches/tseaver-repozo_index/src/ZODB/scripts/tests/test_repozo.py 2010-05-15 17:15:09 UTC (rev 112339)
@@ -74,6 +74,7 @@
del tree[keys[0]]
transaction.commit()
self.pos = self.db.storage._pos
+ self.maxkey = self.db.storage._oid
self.close()
@@ -326,6 +327,11 @@
f.close()
return fqn
+ def test_no_files(self):
+ options = self._makeOptions(date='2010-05-14-13-30-57')
+ found = self._callFUT(options)
+ self.assertEqual(found, [])
+
def test_explicit_date(self):
options = self._makeOptions(date='2010-05-14-13-30-57')
files = []
@@ -525,7 +531,9 @@
self.assertRaises(WouldOverwriteFiles, self._callFUT, options)
def test_empty(self):
+ import struct
from ZODB.scripts.repozo import gen_filename
+ from ZODB.fsIndex import fsIndex
db = self._makeDB()
options = self._makeOptions(file=db._file_name,
gzip=False,
@@ -542,6 +550,15 @@
self.assertEqual(open(datfile).read(),
'%s 0 %d %s\n' %
(target, len(original), md5(original).hexdigest()))
+ ndxfile = os.path.join(self._repository_directory,
+ gen_filename(options, '.index'))
+ ndx_info = fsIndex.load(ndxfile)
+ self.assertEqual(ndx_info['pos'], len(original))
+ index = ndx_info['index']
+ pZero = struct.pack(">Q", 0)
+ pOne = struct.pack(">Q", 1)
+ self.assertEqual(index.minKey(), pZero)
+ self.assertEqual(index.maxKey(), pOne)
class Test_do_incremental_backup(OptionsTestBase, unittest.TestCase):
@@ -576,7 +593,9 @@
self._callFUT, options, 0, repofiles)
def test_no_changes(self):
+ import struct
from ZODB.scripts.repozo import gen_filename
+ from ZODB.fsIndex import fsIndex
db = self._makeDB()
oldpos = db.pos
options = self._makeOptions(file=db._file_name,
@@ -603,9 +622,20 @@
self.assertEqual(open(datfile).read(),
'%s %d %d %s\n' %
(target, oldpos, oldpos, md5('').hexdigest()))
+ ndxfile = os.path.join(self._repository_directory,
+ gen_filename(options, '.index'))
+ ndx_info = fsIndex.load(ndxfile)
+ self.assertEqual(ndx_info['pos'], oldpos)
+ index = ndx_info['index']
+ pZero = struct.pack(">Q", 0)
+ pOne = struct.pack(">Q", 1)
+ self.assertEqual(index.minKey(), pZero)
+ self.assertEqual(index.maxKey(), pOne)
def test_w_changes(self):
+ import struct
from ZODB.scripts.repozo import gen_filename
+ from ZODB.fsIndex import fsIndex
db = self._makeDB()
oldpos = db.pos
options = self._makeOptions(file=db._file_name,
@@ -637,8 +667,109 @@
'%s %d %d %s\n' %
(target, oldpos, newpos,
md5(increment).hexdigest()))
+ ndxfile = os.path.join(self._repository_directory,
+ gen_filename(options, '.index'))
+ ndx_info = fsIndex.load(ndxfile)
+ self.assertEqual(ndx_info['pos'], newpos)
+ index = ndx_info['index']
+ pZero = struct.pack(">Q", 0)
+ self.assertEqual(index.minKey(), pZero)
+ self.assertEqual(index.maxKey(), db.maxkey)
+class Test_do_recover(OptionsTestBase, unittest.TestCase):
+
+ def _callFUT(self, options):
+ from ZODB.scripts.repozo import do_recover
+ return do_recover(options)
+
+ def _makeFile(self, hour, min, sec, ext, text=None):
+ # call _makeOptions first!
+ name = '2010-05-14-%02d-%02d-%02d%s' % (hour, min, sec, ext)
+ if text is None:
+ text = name
+ fqn = os.path.join(self._repository_directory, name)
+ f = open(fqn, 'wb')
+ f.write(text)
+ f.flush()
+ f.close()
+ return fqn
+
+ def test_no_files(self):
+ from ZODB.scripts.repozo import NoFiles
+ options = self._makeOptions(date=None,
+ test_now=(2010, 5, 15, 13, 30, 57))
+ self.assertRaises(NoFiles, self._callFUT, options)
+
+ def test_no_files_before_explicit_date(self):
+ from ZODB.scripts.repozo import NoFiles
+ options = self._makeOptions(date='2010-05-13-13-30-57')
+ files = []
+ for h, m, s, e in [(2, 13, 14, '.fs'),
+ (2, 13, 14, '.dat'),
+ (3, 14, 15, '.deltafs'),
+ (4, 14, 15, '.deltafs'),
+ (5, 14, 15, '.deltafs'),
+ (12, 13, 14, '.fs'),
+ (12, 13, 14, '.dat'),
+ (13, 14, 15, '.deltafs'),
+ (14, 15, 16, '.deltafs'),
+ ]:
+ files.append(self._makeFile(h, m, s, e))
+ self.assertRaises(NoFiles, self._callFUT, options)
+
+ def test_w_full_backup_latest_no_index(self):
+ import tempfile
+ dd = self._data_directory = tempfile.mkdtemp()
+ output = os.path.join(dd, 'Data.fs')
+ index = os.path.join(dd, 'Data.fs.index')
+ options = self._makeOptions(date='2010-05-15-13-30-57',
+ output=output)
+ self._makeFile(2, 3, 4, '.fs', 'AAA')
+ self._makeFile(4, 5, 6, '.fs', 'BBB')
+ self._callFUT(options)
+ self.assertEqual(open(output, 'rb').read(), 'BBB')
+
+ def test_w_full_backup_latest_index(self):
+ import tempfile
+ dd = self._data_directory = tempfile.mkdtemp()
+ output = os.path.join(dd, 'Data.fs')
+ index = os.path.join(dd, 'Data.fs.index')
+ options = self._makeOptions(date='2010-05-15-13-30-57',
+ output=output)
+ self._makeFile(2, 3, 4, '.fs', 'AAA')
+ self._makeFile(4, 5, 6, '.fs', 'BBB')
+ self._makeFile(4, 5, 6, '.index', 'CCC')
+ self._callFUT(options)
+ self.assertEqual(open(output, 'rb').read(), 'BBB')
+ self.assertEqual(open(index, 'rb').read(), 'CCC')
+
+ def test_w_incr_backup_latest_no_index(self):
+ import tempfile
+ dd = self._data_directory = tempfile.mkdtemp()
+ output = os.path.join(dd, 'Data.fs')
+ index = os.path.join(dd, 'Data.fs.index')
+ options = self._makeOptions(date='2010-05-15-13-30-57',
+ output=output)
+ self._makeFile(2, 3, 4, '.fs', 'AAA')
+ self._makeFile(4, 5, 6, '.deltafs', 'BBB')
+ self._callFUT(options)
+ self.assertEqual(open(output, 'rb').read(), 'AAABBB')
+
+ def test_w_incr_backup_latest_index(self):
+ import tempfile
+ dd = self._data_directory = tempfile.mkdtemp()
+ output = os.path.join(dd, 'Data.fs')
+ index = os.path.join(dd, 'Data.fs.index')
+ options = self._makeOptions(date='2010-05-15-13-30-57',
+ output=output)
+ self._makeFile(2, 3, 4, '.fs', 'AAA')
+ self._makeFile(4, 5, 6, '.deltafs', 'BBB')
+ self._makeFile(4, 5, 6, '.index', 'CCC')
+ self._callFUT(options)
+ self.assertEqual(open(output, 'rb').read(), 'AAABBB')
+ self.assertEqual(open(index, 'rb').read(), 'CCC')
+
class MonteCarloTests(unittest.TestCase):
layer = ZODB.tests.util.MininalTestLayer('repozo')
@@ -754,5 +885,9 @@
unittest.makeSuite(Test_delete_old_backups),
unittest.makeSuite(Test_do_full_backup),
unittest.makeSuite(Test_do_incremental_backup),
+ #unittest.makeSuite(Test_do_backup), #TODO
+ unittest.makeSuite(Test_do_recover),
+ # N.B.: this test take forever to run (~40sec on a fast laptop),
+ # *and* it is non-deterministic.
unittest.makeSuite(MonteCarloTests),
])