SVN: zope.testing/trunk/ Merge sdouche's work on shuffling tests.
Christian Theune <[email protected]>
| Newsgroups | gmane.comp.web.zope.zope3.cvs |
|---|---|
| Message-ID | <[email protected]> |
Log message for revision 106739:
Merge sdouche's work on shuffling tests.
Add some more comments, make some of the language clearer.:
Changed:
U zope.testing/trunk/CHANGES.txt
U zope.testing/trunk/src/zope/testing/testrunner/options.py
U zope.testing/trunk/src/zope/testing/testrunner/runner.py
A zope.testing/trunk/src/zope/testing/testrunner/shuffle.py
A zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt
U zope.testing/trunk/src/zope/testing/testrunner/tests.py
-=-
Modified: zope.testing/trunk/CHANGES.txt
===================================================================
--- zope.testing/trunk/CHANGES.txt 2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/CHANGES.txt 2009-12-18 08:23:21 UTC (rev 106739)
@@ -9,6 +9,9 @@
- Cleaned up unused imports reported by pyflakes.
+- Added two new options to generate randomly ordered list of tests and to
+ select a specific order of tests.
+
- RENormalizing checkers can be combined via `` + `` now: ``checker1 +
checker2`` creates a checker with the transformations of both checkers.
Modified: zope.testing/trunk/src/zope/testing/testrunner/options.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/options.py 2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/src/zope/testing/testrunner/options.py 2009-12-18 08:23:21 UTC (rev 106739)
@@ -398,6 +398,19 @@
Specifies the name of a directory to ignore when looking for tests.
""")
+setup.add_option(
+ '--shuffle', action="store_true", dest='shuffle',
+ help="""\
+Shuffles the order in which tests are ran.
+""")
+
+setup.add_option(
+ '--shuffle-seed', action="store", dest='shuffle_seed', type="int",
+ help="""\
+Value used to initialize the tests shuffler. Specify a value to create
+repeatable random ordered tests.
+""")
+
parser.add_option_group(setup)
######################################################################
Modified: zope.testing/trunk/src/zope/testing/testrunner/runner.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/runner.py 2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/src/zope/testing/testrunner/runner.py 2009-12-18 08:23:21 UTC (rev 106739)
@@ -46,6 +46,7 @@
import zope.testing.testrunner.interfaces
import zope.testing.testrunner.debug
import zope.testing.testrunner.tb_format
+import zope.testing.testrunner.shuffle
PYREFCOUNT_PATTERN = re.compile('\[[0-9]+ refs\]')
@@ -188,6 +189,7 @@
zope.testing.testrunner.garbagecollection.Debug(self))
self.features.append(zope.testing.testrunner.find.Find(self))
+ self.features.append(zope.testing.testrunner.shuffle.Shuffle(self))
self.features.append(zope.testing.testrunner.process.SubProcess(self))
self.features.append(zope.testing.testrunner.filter.Filter(self))
self.features.append(zope.testing.testrunner.listing.Listing(self))
Copied: zope.testing/trunk/src/zope/testing/testrunner/shuffle.py (from rev 105102, zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/shuffle.py)
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/shuffle.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/shuffle.py 2009-12-18 08:23:21 UTC (rev 106739)
@@ -0,0 +1,51 @@
+##############################################################################
+#
+# Copyright (c) 2004-2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Shuffle tests discovered before executing them.
+
+$Id$
+"""
+
+import time
+import random
+import unittest
+import zope.testing.testrunner.feature
+
+
+class Shuffle(zope.testing.testrunner.feature.Feature):
+ """Take the tests found so far and shuffle them."""
+
+ def __init__(self, runner):
+ super(Shuffle, self).__init__(runner)
+ self.active = runner.options.shuffle
+ self.seed = runner.options.shuffle_seed
+ if self.seed is None:
+ # We can't rely on the random modules seed initialization because
+ # we can't introspect the seed later for reporting. This is a
+ # simple emulation of what random.Random.seed does anyway.
+ self.seed = long(time.time() * 256) # use fractional seconds
+
+ def global_setup(self):
+ rng = random.Random(self.seed)
+ for layer, suite in list(self.runner.tests_by_layer_name.items()):
+ # Test suites cannot be modified through a public API. We thus
+ # take a mutable copy of the list of tests of that suite, shuffle
+ # that and replace the test suite instance with a new one of the
+ # same class.
+ tests = list(suite)
+ rng.shuffle(tests)
+ self.runner.tests_by_layer_name[layer] = suite.__class__(tests)
+
+ def report(self):
+ msg = "Tests were shuffled using seed number %d." % self.seed
+ self.runner.options.output.info(msg)
Copied: zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt (from rev 105102, zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/testrunner-shuffle.txt)
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt 2009-12-18 08:23:21 UTC (rev 106739)
@@ -0,0 +1,134 @@
+Shuffling tests
+===============
+
+By default, every time you launch the testrunner it will run the tests in a
+specific order. However, if you want to ensure that your tests are well
+isolated then running them in a varying order can be helpful. (Proper
+isolation meaning your tests don't depend on each others outcomes or side
+effects and thus the setup and tear down methods are written properly.)
+
+The ``--shuffle`` option tells the test runner to shuffle the list of tests
+before running them:
+
+ >>> import os.path, sys
+ >>> directory_with_tests = os.path.join(this_directory, 'testrunner-ex')
+ >>> defaults = [
+ ... '--path', directory_with_tests,
+ ... '--tests-pattern', '^sampletestsf?$',
+ ... ]
+
+ >>> from zope.testing import testrunner
+ >>> default_argv = 'test -u -m sample1 -t test_y0 --list-tests '
+
+Running shuffled tests
+----------------------
+
+When specifying the ``--shuffle`` option tests are ordered differently each
+time you run the tests:
+
+ >>> argv = (default_argv + '--shuffle').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number ...
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ ...
+ False
+
+Note: The runner prints out a new piece of information which is the seed number used
+to generate the shuffled list of tests. This seed number can later be used to
+re-run the tests in exactly the same order to support debugging.
+
+Specifying a seed number to control tests shuffling
+---------------------------------------------------
+
+Along with the ``--shuffle`` option comes the ``--shuffle-seed`` option which
+takes a seed number as an argument. If you want to reproduce test failures
+that happened during a randomly shuffled test run then simply write down the
+seed number that was printed out and run it again using the ``--shuffle-seed``
+option. The order is guaranteed to be the same.
+
+For example, using the seed number 0 will give us the following, stable, list of
+tests:
+
+ >>> argv = (default_argv + '--shuffle --shuffle-seed 0').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number 0.
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletests.test_one)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sampletests.test1)
+ False
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number 0.
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletests.test_one)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sampletests.test1)
+ False
+
+Whereas using the seed number 42 will give us the following, different but
+stable, list of tests:
+
+ >>> argv = (default_argv + '--shuffle --shuffle-seed 42').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number 42.
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sampletests.test1)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sampletests.test_one)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ False
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number 42.
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sampletests.test1)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sampletests.test_one)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ False
+
+Selecting a seed number without ``--shuffle``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Note that the ``--shuffle-seed`` option must be used along with ``--shuffle``
+option or tests will not be re-ordered:
+
+ >>> argv = (default_argv + '--shuffle-seed 42').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sampletests.test1)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ test_y0 (sample1.sampletests.test_one)
+ False
Modified: zope.testing/trunk/src/zope/testing/testrunner/tests.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/tests.py 2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/src/zope/testing/testrunner/tests.py 2009-12-18 08:23:21 UTC (rev 106739)
@@ -163,6 +163,7 @@
'testrunner-repeat.txt',
'testrunner-gc.txt',
'testrunner-knit.txt',
+ 'testrunner-shuffle.txt',
setUp=setUp, tearDown=tearDown,
optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE,
checker=checker),