SVN: zope.testing/branches/3.5/ Fix LP#253959: in some invoking running the test runner from inside a

Log message for revision 89164:
  Fix LP#253959: in some invoking running the test runner from inside a
  test case could cause problems with layers.
  
  

Changed:
  U   zope.testing/branches/3.5/README.txt
  A   zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/
  A   zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/innertests.py
  A   zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/outertests.py
  A   zope.testing/branches/3.5/src/zope/testing/testrunner-reentrancy.txt
  U   zope.testing/branches/3.5/src/zope/testing/testrunner.py

-=-
Modified: zope.testing/branches/3.5/README.txt
===================================================================
--- zope.testing/branches/3.5/README.txt	2008-08-01 16:54:08 UTC (rev 89163)
+++ zope.testing/branches/3.5/README.txt	2008-08-01 17:16:10 UTC (rev 89164)
@@ -67,7 +67,8 @@
 - Launchpad #242851: committed the last missing testing part
   of the patch
 
-- Launchpad #253959: added extra diagnostics.
+- Launchpad #253959: in some invoking running the test runner from inside a
+  test case could cause problems with layers.
 
 
 3.5.3 (2008/07/08)

Added: zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/innertests.py
===================================================================
--- zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/innertests.py	                        (rev 0)
+++ zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/innertests.py	2008-08-01 17:16:10 UTC (rev 89164)
@@ -0,0 +1,30 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Sample test layers
+
+$Id$
+"""
+
+import unittest
+
+
+class InnerTest(unittest.TestCase):
+
+    def test(self):
+        pass
+
+
+def test_suite():
+    return unittest.makeSuite(InnerTest)
+


Property changes on: zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/innertests.py
___________________________________________________________________
Name: svn:keywords
   + Id

Added: zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/outertests.py
===================================================================
--- zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/outertests.py	                        (rev 0)
+++ zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/outertests.py	2008-08-01 17:16:10 UTC (rev 89164)
@@ -0,0 +1,57 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Sample test layers
+
+$Id$
+"""
+
+import os
+import unittest
+
+
+class TestLayer:
+
+    __bases__ = ()
+
+    def __init__(self, module, name):
+        self.__module__ = module
+        self.__name__ = name
+
+
+class UnitTest(unittest.TestCase):
+
+    def test(self):
+        from zope.testing import testrunner
+        this_directory = os.path.dirname(__file__)
+        defaults = [
+            '--path', this_directory,
+            '--tests-pattern', '^innertests$',
+            ]
+        print "-- inner test run starts --"
+        testrunner.run(defaults)
+        print "-- inner test run ends --"
+
+
+class LayeredTest(unittest.TestCase):
+
+    layer = TestLayer(__name__, 'HardToAccessTestLayer')
+
+    def test(self):
+        pass
+
+
+def test_suite():
+    return unittest.TestSuite([unittest.makeSuite(UnitTest),
+                               unittest.makeSuite(LayeredTest)])
+


Property changes on: zope.testing/branches/3.5/src/zope/testing/testrunner-ex-r/outertests.py
___________________________________________________________________
Name: svn:keywords
   + Id

Added: zope.testing/branches/3.5/src/zope/testing/testrunner-reentrancy.txt
===================================================================
--- zope.testing/branches/3.5/src/zope/testing/testrunner-reentrancy.txt	                        (rev 0)
+++ zope.testing/branches/3.5/src/zope/testing/testrunner-reentrancy.txt	2008-08-01 17:16:10 UTC (rev 89164)
@@ -0,0 +1,40 @@
+Using the testrunner recursively
+================================
+
+It's pretty obvious that the tests that are run by the testrunner can
+themselves invoke the testrunner.  That's how the testrunner is tested,
+after all!
+
+However there's a corner case that used to break.  We have a set of tests
+split into unit tests (no layer) and layered tests (where the layer cannot be
+accessed directly by importing the module and taking the attribute named by
+layer.__name__ in that module).  The first test in the unit test layer
+imports zope.testing.testrunner and runs it recursively.  It should work:
+
+    >>> import os.path, sys
+    >>> directory_with_tests = os.path.join(this_directory, 'testrunner-ex-r')
+    >>> defaults = [
+    ...     '--path', directory_with_tests,
+    ...     '--tests-pattern', '^outertests$',
+    ...     ]
+
+    >>> sys.argv = []
+    >>> from zope.testing import testrunner
+    >>> testrunner.run(defaults)
+    Running unit tests:
+    -- inner test run starts --
+    Running unit tests:
+      Ran 1 tests with 0 failures and 0 errors in N.NNN seconds.
+    -- inner test run ends --
+      Ran 1 tests with 0 failures and 0 errors in N.NNN seconds.
+    Running outertests.HardToAccessTestLayer tests:
+      Set up outertests.HardToAccessTestLayer in N.NNN seconds.
+      Ran 1 tests with 0 failures and 0 errors in N.NNN seconds.
+    Tearing down left over layers:
+      Tear down outertests.HardToAccessTestLayer in N.NNN seconds.
+    Total: 2 tests, 0 failures, 0 errors in N.NNN seconds.
+    False
+
+https://bugs.launchpad.net/zope.testing/+bug/253959 was caused by the
+inner test runner clearing the global layer name cache that was needed
+by the outer test runner.


Property changes on: zope.testing/branches/3.5/src/zope/testing/testrunner-reentrancy.txt
___________________________________________________________________
Name: svn:eol-style
   + native

Modified: zope.testing/branches/3.5/src/zope/testing/testrunner.py
===================================================================
--- zope.testing/branches/3.5/src/zope/testing/testrunner.py	2008-08-01 16:54:08 UTC (rev 89163)
+++ zope.testing/branches/3.5/src/zope/testing/testrunner.py	2008-08-01 17:16:10 UTC (rev 89164)
@@ -972,174 +972,180 @@
     """
 
     global _layer_name_cache
-    _layer_name_cache = {} # Reset to enforce test isolation
+    old_layer_name_cache = _layer_name_cache
+    try:
+        _layer_name_cache = {} # Reset to enforce test isolation
 
-    output = options.output
+        output = options.output
 
-    if options.resume_layer:
-        original_stderr = sys.stderr
-        sys.stderr = sys.stdout
-    elif options.verbose:
-        if options.all:
-            msg = "Running tests at all levels"
-        else:
-            msg = "Running tests at level %d" % options.at_level
-        output.info(msg)
+        if options.resume_layer:
+            original_stderr = sys.stderr
+            sys.stderr = sys.stdout
+        elif options.verbose:
+            if options.all:
+                msg = "Running tests at all levels"
+            else:
+                msg = "Running tests at level %d" % options.at_level
+            output.info(msg)
 
 
-    old_threshold = gc.get_threshold()
-    if options.gc:
-        if len(options.gc) > 3:
-            output.error("Too many --gc options")
-            sys.exit(1)
-        if options.gc[0]:
-            output.info("Cyclic garbage collection threshold set to: %s" %
-                        repr(tuple(options.gc)))
-        else:
-            output.info("Cyclic garbage collection is disabled.")
+        old_threshold = gc.get_threshold()
+        if options.gc:
+            if len(options.gc) > 3:
+                output.error("Too many --gc options")
+                sys.exit(1)
+            if options.gc[0]:
+                output.info("Cyclic garbage collection threshold set to: %s" %
+                            repr(tuple(options.gc)))
+            else:
+                output.info("Cyclic garbage collection is disabled.")
 
-        gc.set_threshold(*options.gc)
+            gc.set_threshold(*options.gc)
 
-    old_flags = gc.get_debug()
-    if options.gc_option:
-        new_flags = 0
-        for op in options.gc_option:
-            new_flags |= getattr(gc, op)
-        gc.set_debug(new_flags)
+        old_flags = gc.get_debug()
+        if options.gc_option:
+            new_flags = 0
+            for op in options.gc_option:
+                new_flags |= getattr(gc, op)
+            gc.set_debug(new_flags)
 
-    old_reporting_flags = doctest.set_unittest_reportflags(0)
-    reporting_flags = 0
-    if options.ndiff:
-        reporting_flags = doctest.REPORT_NDIFF
-    if options.udiff:
+        old_reporting_flags = doctest.set_unittest_reportflags(0)
+        reporting_flags = 0
+        if options.ndiff:
+            reporting_flags = doctest.REPORT_NDIFF
+        if options.udiff:
+            if reporting_flags:
+                output.error("Can only give one of --ndiff, --udiff, or --cdiff")
+                sys.exit(1)
+            reporting_flags = doctest.REPORT_UDIFF
+        if options.cdiff:
+            if reporting_flags:
+                output.error("Can only give one of --ndiff, --udiff, or --cdiff")
+                sys.exit(1)
+            reporting_flags = doctest.REPORT_CDIFF
+        if options.report_only_first_failure:
+            reporting_flags |= doctest.REPORT_ONLY_FIRST_FAILURE
+
         if reporting_flags:
-            output.error("Can only give one of --ndiff, --udiff, or --cdiff")
-            sys.exit(1)
-        reporting_flags = doctest.REPORT_UDIFF
-    if options.cdiff:
-        if reporting_flags:
-            output.error("Can only give one of --ndiff, --udiff, or --cdiff")
-            sys.exit(1)
-        reporting_flags = doctest.REPORT_CDIFF
-    if options.report_only_first_failure:
-        reporting_flags |= doctest.REPORT_ONLY_FIRST_FAILURE
+            doctest.set_unittest_reportflags(reporting_flags)
+        else:
+            doctest.set_unittest_reportflags(old_reporting_flags)
 
-    if reporting_flags:
-        doctest.set_unittest_reportflags(reporting_flags)
-    else:
-        doctest.set_unittest_reportflags(old_reporting_flags)
 
+        # Add directories to the path
+        for path in options.path:
+            if path not in sys.path:
+                sys.path.append(path)
 
-    # Add directories to the path
-    for path in options.path:
-        if path not in sys.path:
-            sys.path.append(path)
+        remove_stale_bytecode(options)
 
-    remove_stale_bytecode(options)
+        tests_by_layer_name = find_tests(options, found_suites)
 
-    tests_by_layer_name = find_tests(options, found_suites)
+        ran = 0
+        failures = []
+        errors = []
+        nlayers = 0
+        import_errors = tests_by_layer_name.pop(None, None)
 
-    ran = 0
-    failures = []
-    errors = []
-    nlayers = 0
-    import_errors = tests_by_layer_name.pop(None, None)
+        output.import_errors(import_errors)
 
-    output.import_errors(import_errors)
+        if 'unit' in tests_by_layer_name:
+            tests = tests_by_layer_name.pop('unit')
+            if (not options.non_unit) and not options.resume_layer:
+                if options.layer:
+                    should_run = False
+                    for pat in options.layer:
+                        if pat('unit'):
+                            should_run = True
+                            break
+                else:
+                    should_run = True
 
-    if 'unit' in tests_by_layer_name:
-        tests = tests_by_layer_name.pop('unit')
-        if (not options.non_unit) and not options.resume_layer:
-            if options.layer:
-                should_run = False
-                for pat in options.layer:
-                    if pat('unit'):
-                        should_run = True
-                        break
-            else:
-                should_run = True
+                if should_run:
+                    if options.list_tests:
+                        output.list_of_tests(tests, 'unit')
+                    else:
+                        output.info("Running unit tests:")
+                        nlayers += 1
+                        ran += run_tests(options, tests, 'unit', failures, errors)
 
-            if should_run:
-                if options.list_tests:
-                    output.list_of_tests(tests, 'unit')
-                else:
-                    output.info("Running unit tests:")
-                    nlayers += 1
-                    ran += run_tests(options, tests, 'unit', failures, errors)
+        setup_layers = {}
 
-    setup_layers = {}
+        layers_to_run = list(ordered_layers(tests_by_layer_name))
+        if options.resume_layer is not None:
+            layers_to_run = [
+                (layer_name, layer, tests)
+                for (layer_name, layer, tests) in layers_to_run
+                if layer_name == options.resume_layer
+            ]
+        elif options.layer:
+            layers_to_run = [
+                (layer_name, layer, tests)
+                for (layer_name, layer, tests) in layers_to_run
+                if filter(None, [pat(layer_name) for pat in options.layer])
+            ]
 
-    layers_to_run = list(ordered_layers(tests_by_layer_name))
-    if options.resume_layer is not None:
-        layers_to_run = [
-            (layer_name, layer, tests)
-            for (layer_name, layer, tests) in layers_to_run
-            if layer_name == options.resume_layer
-        ]
-    elif options.layer:
-        layers_to_run = [
-            (layer_name, layer, tests)
-            for (layer_name, layer, tests) in layers_to_run
-            if filter(None, [pat(layer_name) for pat in options.layer])
-        ]
 
+        if options.list_tests:
+            for layer_name, layer, tests in layers_to_run:
+                output.list_of_tests(tests, layer_name)
+            return True
 
-    if options.list_tests:
+        start_time = time.time()
+
         for layer_name, layer, tests in layers_to_run:
-            output.list_of_tests(tests, layer_name)
-        return True
+            nlayers += 1
+            try:
+                ran += run_layer(options, layer_name, layer, tests,
+                                 setup_layers, failures, errors)
+            except CanNotTearDown:
+                setup_layers = None
+                if not options.resume_layer:
+                    ran += resume_tests(options, layer_name, layers_to_run,
+                                        failures, errors)
+                    break
 
-    start_time = time.time()
+        if setup_layers:
+            if options.resume_layer == None:
+                output.info("Tearing down left over layers:")
+            tear_down_unneeded(options, (), setup_layers, True)
 
-    for layer_name, layer, tests in layers_to_run:
-        nlayers += 1
-        try:
-            ran += run_layer(options, layer_name, layer, tests,
-                             setup_layers, failures, errors)
-        except CanNotTearDown:
-            setup_layers = None
-            if not options.resume_layer:
-                ran += resume_tests(options, layer_name, layers_to_run,
-                                    failures, errors)
-                break
+        total_time = time.time() - start_time
 
-    if setup_layers:
-        if options.resume_layer == None:
-            output.info("Tearing down left over layers:")
-        tear_down_unneeded(options, (), setup_layers, True)
+        if options.resume_layer:
+            sys.stdout.close()
+            # Communicate with the parent.  The protocol is obvious:
+            print >> original_stderr, ran, len(failures), len(errors)
+            for test, exc_info in failures:
+                print >> original_stderr, ' '.join(str(test).strip().split('\n'))
+            for test, exc_info in errors:
+                print >> original_stderr, ' '.join(str(test).strip().split('\n'))
 
-    total_time = time.time() - start_time
+        else:
+            if options.verbose:
+                output.tests_with_errors(errors)
+                output.tests_with_failures(failures)
 
-    if options.resume_layer:
-        sys.stdout.close()
-        # Communicate with the parent.  The protocol is obvious:
-        print >> original_stderr, ran, len(failures), len(errors)
-        for test, exc_info in failures:
-            print >> original_stderr, ' '.join(str(test).strip().split('\n'))
-        for test, exc_info in errors:
-            print >> original_stderr, ' '.join(str(test).strip().split('\n'))
+            if nlayers != 1:
+                output.totals(ran, len(failures), len(errors), total_time)
 
-    else:
-        if options.verbose:
-            output.tests_with_errors(errors)
-            output.tests_with_failures(failures)
+            output.modules_with_import_problems(import_errors)
 
-        if nlayers != 1:
-            output.totals(ran, len(failures), len(errors), total_time)
+        doctest.set_unittest_reportflags(old_reporting_flags)
 
-        output.modules_with_import_problems(import_errors)
+        if options.gc_option:
+            gc.set_debug(old_flags)
 
-    doctest.set_unittest_reportflags(old_reporting_flags)
+        if options.gc:
+            gc.set_threshold(*old_threshold)
 
-    if options.gc_option:
-        gc.set_debug(old_flags)
+        return not bool(import_errors or failures or errors)
+    finally:
+        # restore old value to prevent cleaning the global cache when one of
+        # the tests decides to invoke testrunner.run() inside itself.
+        _layer_name_cache = old_layer_name_cache
 
-    if options.gc:
-        gc.set_threshold(*old_threshold)
 
-    return not bool(import_errors or failures or errors)
-
-
 def run_tests(options, tests, name, failures, errors):
     repeat = options.repeat or 1
     repeat_range = iter(range(repeat))
@@ -2707,6 +2713,7 @@
         'testrunner-gc.txt',
         'testrunner-knit.txt',
         'testrunner-package-normalization.txt',
+        'testrunner-reentrancy.txt',
         setUp=setUp, tearDown=tearDown,
         optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE,
         checker=checker),