SVN: ZODB/branches/3.9/src/ Merged the test_repozo branch, which converted the old basic manual
Jim Fulton <[email protected]>
| Newsgroups | gmane.comp.python.zope.zodb.cvs |
|---|---|
| Message-ID | <[email protected]> |
Log message for revision 106199:
Merged the test_repozo branch, which converted the old basic manual
test into an automated test, and included Chris Wither's repozo fix
(as monified by Godefroid Chapelle) to avoid a deprecation warning.
Changed:
U ZODB/branches/3.9/src/CHANGES.txt
D ZODB/branches/3.9/src/ZODB/scripts/fstail.txt
D ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py
D ZODB/branches/3.9/src/ZODB/scripts/referrers.txt
U ZODB/branches/3.9/src/ZODB/scripts/repozo.py
A ZODB/branches/3.9/src/ZODB/scripts/tests/
U ZODB/branches/3.9/src/ZODB/scripts/tests/test_repozo.py
D ZODB/branches/3.9/src/ZODB/scripts/tests.py
-=-
Modified: ZODB/branches/3.9/src/CHANGES.txt
===================================================================
--- ZODB/branches/3.9/src/CHANGES.txt 2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/CHANGES.txt 2009-12-03 19:50:05 UTC (rev 106199)
@@ -16,6 +16,9 @@
- Internal ZEO errors were logged at the INFO level, rather
than at the error level.
+- The FileStorage backup and restore script, repozo, gave a
+ deprecation warning under Python 2.6.
+
3.9.3 (2009-10-23)
==================
Deleted: ZODB/branches/3.9/src/ZODB/scripts/fstail.txt
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/fstail.txt 2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/fstail.txt 2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,40 +0,0 @@
-====================
-The `fstail` utility
-====================
-
-The `fstail` utility shows information for a FileStorage about the last `n`
-transactions:
-
-We have to prepare a FileStorage first:
-
- >>> from ZODB.FileStorage import FileStorage
- >>> from ZODB.DB import DB
- >>> import transaction
- >>> from tempfile import mktemp
- >>> storagefile = mktemp()
- >>> base_storage = FileStorage(storagefile)
- >>> database = DB(base_storage)
- >>> connection1 = database.open()
- >>> root = connection1.root()
- >>> root['foo'] = 1
- >>> transaction.commit()
-
-Now lets have a look at the last transactions of this FileStorage:
-
- >>> from ZODB.scripts.fstail import main
- >>> main(storagefile, 5)
- 2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3
- user='' description='' length=132 offset=185
- <BLANKLINE>
- 2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3
- user='' description='initial database creation' length=150 offset=52
- <BLANKLINE>
-
-Now clean up the storage again:
-
- >>> import os
- >>> base_storage.close()
- >>> os.unlink(storagefile)
- >>> os.unlink(storagefile+'.index')
- >>> os.unlink(storagefile+'.lock')
- >>> os.unlink(storagefile+'.tmp')
Deleted: ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py 2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py 2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,164 +0,0 @@
-#!/usr/bin/env python
-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-"""Test repozo.py.
-
-This is a by-hand test. It succeeds iff it doesn't blow up. Run it with
-its home directory as the current directory. It will destroy all files
-matching Data.* and Copy.* in this directory, and anything in a
-subdirectory of name 'backup'.
-
-Usage:
-
-python testrepozo.py [repozo_script]
-
- repozo_script, if provided, is a path to a script that runs repozo,
- such as that generated by buildout.
-
-eg:
-$ ../../../../bin/py testrepozo.py ../../../../bin/repozo
-"""
-
-import os
-import random
-import time
-import glob
-import sys
-import shutil
-
-import ZODB
-from ZODB import FileStorage
-import transaction
-
-def cleanup():
- for fname in glob.glob('Data.*') + glob.glob('Copy.*'):
- os.remove(fname)
-
- if os.path.isdir('backup'):
- for fname in os.listdir('backup'):
- os.remove(os.path.join('backup', fname))
- os.rmdir('backup')
-
-class OurDB:
- def __init__(self):
- from BTrees.OOBTree import OOBTree
- self.getdb()
- conn = self.db.open()
- conn.root()['tree'] = OOBTree()
- transaction.commit()
- self.close()
-
- def getdb(self):
- storage = FileStorage.FileStorage('Data.fs')
- self.db = ZODB.DB(storage)
-
- def gettree(self):
- self.getdb()
- conn = self.db.open()
- return conn.root()['tree']
-
- def pack(self):
- self.getdb()
- self.db.pack()
-
- def close(self):
- if self.db is not None:
- self.db.close()
- self.db = None
-
-# Do recovery to time 'when', and check that it's identical to correctpath.
-def check(correctpath='Data.fs', when=None):
- if when is None:
- extra = ''
- else:
- extra = ' -D ' + when
- cmd = PYTHON + REPOZO + ' -vRr backup -o Copy.fs' + extra
- os.system(cmd)
- f = file(correctpath, 'rb')
- g = file('Copy.fs', 'rb')
- fguts = f.read()
- gguts = g.read()
- f.close()
- g.close()
- if fguts != gguts:
- raise ValueError("guts don't match\n"
- " correctpath=%r when=%r\n"
- " cmd=%r" % (correctpath, when, cmd))
-
-def mutatedb(db):
- # Make random mutations to the btree in the database.
- tree = db.gettree()
- for dummy in range(100):
- if random.random() < 0.6:
- tree[random.randrange(100000)] = random.randrange(100000)
- else:
- keys = tree.keys()
- if keys:
- del tree[keys[0]]
- transaction.commit()
- db.close()
-
-def main():
- cleanup()
- os.mkdir('backup')
- d = OurDB()
- # Every 9th time thru the loop, we save a full copy of Data.fs,
- # and at the end we ensure we can reproduce those too.
- saved_snapshots = [] # list of (name, time) pairs for copies.
-
- for i in range(100):
- # Make some mutations.
- mutatedb(d)
-
- # Pack about each tenth time.
- if random.random() < 0.1:
- print "packing"
- d.pack()
- d.close()
-
- # Make an incremental backup, half the time with gzip (-z).
- if random.random() < 0.5:
- os.system(PYTHON + REPOZO + ' -vBQr backup -f Data.fs')
- else:
- os.system(PYTHON + REPOZO + ' -zvBQr backup -f Data.fs')
-
- if i % 9 == 0:
- copytime = '%04d-%02d-%02d-%02d-%02d-%02d' % (time.gmtime()[:6])
- copyname = os.path.join('backup', "Data%d" % i) + '.fs'
- shutil.copyfile('Data.fs', copyname)
- saved_snapshots.append((copyname, copytime))
-
- # Make sure the clock moves at least a second.
- time.sleep(1.01)
-
- # Verify current Data.fs can be reproduced exactly.
- check()
-
- # Verify snapshots can be reproduced exactly.
- for copyname, copytime in saved_snapshots:
- print "Checking that", copyname, "at", copytime, "is reproducible."
- check(copyname, copytime)
-
- # Tear it all down.
- cleanup()
- print 'Test passed!'
-
-if __name__ == '__main__':
- PYTHON = sys.executable + ' '
- if len(sys.argv)>1:
- REPOZO = sys.argv[1]
- else:
- REPOZO = '../repozo.py'
- main()
Deleted: ZODB/branches/3.9/src/ZODB/scripts/referrers.txt
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/referrers.txt 2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/referrers.txt 2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,43 +0,0 @@
-Getting Object Referrers
-========================
-
-The referrers module provides a way to get object referrers. It
-provides a referrers method that takes an iterable storage object. It
-returns a dictionary mapping object ids to lists of referrer object
-versions, which each version is a tuple an object id nd serial
-nummber.
-
-To see how this works, we'll create a small database:
-
- >>> import transaction
- >>> from persistent.mapping import PersistentMapping
- >>> from ZODB.FileStorage import FileStorage
- >>> from ZODB.DB import DB
- >>> import os, tempfile
- >>> dest = tempfile.mkdtemp()
- >>> fs = FileStorage(os.path.join(dest, 'Data.fs'))
- >>> db = DB(fs)
- >>> conn = db.open()
- >>> conn.root()['a'] = PersistentMapping()
- >>> conn.root()['b'] = PersistentMapping()
- >>> transaction.commit()
- >>> roid = conn.root()._p_oid
- >>> aoid = conn.root()['a']._p_oid
- >>> boid = conn.root()['b']._p_oid
- >>> s1 = conn.root()['b']._p_serial
-
- >>> conn.root()['a']['b'] = conn.root()['b']
- >>> transaction.commit()
- >>> s2 = conn.root()['a']._p_serial
-
-Now we'll get the storage and compute the referrers:
-
- >>> import ZODB.scripts.referrers
- >>> referrers = ZODB.scripts.referrers.referrers(fs)
-
- >>> referrers[boid] == [(roid, s1), (aoid, s2)]
- True
-
-.. Cleanup
-
- >>> db.close()
Modified: ZODB/branches/3.9/src/ZODB/scripts/repozo.py
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/repozo.py 2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/repozo.py 2009-12-03 19:50:05 UTC (rev 106199)
@@ -65,7 +65,12 @@
import os
import sys
-import md5
+try:
+ # the hashlib package is available from Python 2.5
+ from hashlib import md5
+except ImportError:
+ # the md5 package is deprecated in Python 2.6
+ from md5 import new as md5
import gzip
import time
import errno
@@ -101,10 +106,10 @@
print >> sys.stderr, msg % args
-def parseargs():
+def parseargs(argv):
global VERBOSE
try:
- opts, args = getopt.getopt(sys.argv[1:], 'BRvhf:r:FD:o:Qz',
+ opts, args = getopt.getopt(argv, 'BRvhf:r:FD:o:Qz',
['backup', 'recover', 'verbose', 'help',
'file=', 'repository=', 'full', 'date=',
'output=', 'quick', 'gzip'])
@@ -210,7 +215,7 @@
def checksum(fp, n):
# Checksum the first n bytes of the specified file
- sum = md5.new()
+ sum = md5()
def func(data):
sum.update(data)
dofile(func, fp, n)
@@ -221,7 +226,7 @@
# Copy bytes from file src, to file dst, starting at offset start, for n
# length of bytes. For robustness, we first write, flush and fsync
# to a temp file, then rename the temp file at the end.
- sum = md5.new()
+ sum = md5()
ifp = open(options.file, 'rb')
ifp.seek(start)
tempname = os.path.join(os.path.dirname(dst), 'tmp.tmp')
@@ -248,7 +253,7 @@
# Concatenate a bunch of files from the repository, output to `outfile' if
# given. Return the number of bytes written and the md5 checksum of the
# bytes.
- sum = md5.new()
+ sum = md5()
def func(data):
sum.update(data)
if ofp:
@@ -504,8 +509,10 @@
log('Recovered %s bytes, md5: %s', reposz, reposum)
-def main():
- options = parseargs()
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv[1:]
+ options = parseargs(argv)
if options.mode == BACKUP:
do_backup(options)
else:
Modified: ZODB/branches/3.9/src/ZODB/scripts/tests/test_repozo.py
===================================================================
--- ZODB/branches/test_repozo/src/ZODB/scripts/tests/test_repozo.py 2009-12-03 17:41:51 UTC (rev 106197)
+++ ZODB/branches/3.9/src/ZODB/scripts/tests/test_repozo.py 2009-12-03 19:50:05 UTC (rev 106199)
@@ -15,6 +15,7 @@
import unittest
import os
+import ZODB.tests.util
_NOISY = os.environ.get('NOISY_REPOZO_TEST_OUTPUT')
@@ -67,14 +68,10 @@
self.close()
-class BasicRepozoTests(unittest.TestCase):
-
- def test_importability(self):
- from ZODB.scripts import repozo
-
-
class RepozoTests(unittest.TestCase):
+ layer = ZODB.tests.util.MininalTestLayer('repozo')
+
def setUp(self):
# compute directory names
import tempfile
@@ -176,7 +173,6 @@
def test_suite():
suite = unittest.TestSuite()
- suite.addTest(unittest.makeSuite(BasicRepozoTests))
suite.addTest(unittest.makeSuite(RepozoTests))
return suite
Deleted: ZODB/branches/3.9/src/ZODB/scripts/tests.py
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/tests.py 2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/tests.py 2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,35 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Test harness for scripts.
-
-$Id$
-"""
-import unittest
-import re
-from zope.testing import doctest, renormalizing
-import ZODB.tests.util
-
-checker = renormalizing.RENormalizing([
- (re.compile('[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+'),
- '2007-11-10 15:18:48.543001'),
- (re.compile('hash=[0-9a-f]{40}'),
- 'hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3')])
-
-def test_suite():
- return unittest.TestSuite((
- doctest.DocFileSuite(
- 'referrers.txt', 'fstail.txt',
- setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown,
- checker=checker),
- ))