SVN: zope.testing/trunk/ Added import checker that warns when re-importing symbols and not
Christian Theune <[email protected]>
| Newsgroups | gmane.comp.web.zope.zope3.cvs |
|---|---|
| Message-ID | <[email protected]> |
Log message for revision 95313:
Added import checker that warns when re-importing symbols and not
getting them from their original location, applying some heuristics to
filter out regular re-import scenarios.
Changed:
U zope.testing/trunk/CHANGES.txt
U zope.testing/trunk/src/zope/testing/module.txt
A zope.testing/trunk/src/zope/testing/testrunner/importcheck.py
A zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py
A zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck2/
U zope.testing/trunk/src/zope/testing/testrunner/options.py
U zope.testing/trunk/src/zope/testing/testrunner/runner.py
U zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test
U zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt
U zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt
U zope.testing/trunk/src/zope/testing/testrunner/tests.py
-=-
Modified: zope.testing/trunk/CHANGES.txt
===================================================================
--- zope.testing/trunk/CHANGES.txt 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/CHANGES.txt 2009-01-28 13:55:39 UTC (rev 95313)
@@ -1,6 +1,13 @@
zope.testing Changelog
**********************
+3.7.2 (unreleased)
+==================
+
+- Added feature that warns about re-imports of symbols that originate
+ from a different location. This allows detecting when classes and
+ functions get moved from one module to another.
+
3.7.1 (2008-10-17)
==================
Modified: zope.testing/trunk/src/zope/testing/module.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/module.txt 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/module.txt 2009-01-28 13:55:39 UTC (rev 95313)
@@ -89,7 +89,7 @@
>>> import zope.testing.unlikelymodulename
Traceback (most recent call last):
...
- ImportError: No module named unlikelymodulename
+ ImportError: No module named zope.testing.unlikelymodulename
This only works for packages that already exist::
Added: zope.testing/trunk/src/zope/testing/testrunner/importcheck.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importcheck.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importcheck.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,113 @@
+# vim:fileencoding=utf-8
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+
+import ihooks
+import os.path
+import sys
+import types
+import zope.testing.testrunner.feature
+import inspect
+
+
+WHITELIST = [('re', 'match', 'sre'),
+ ('os', 'error', 'exceptions')]
+
+
+class IndirectAttributeAccessChecker(types.ModuleType):
+
+ def __init__(self, module):
+ self.__import_checker_module = module
+
+ def __setattr__(self, name, value):
+ if name.startswith('_IndirectAttributeAccessChecker__import_checker_'):
+ object.__setattr__(self, name.replace('_IndirectAttributeAccessChecker', ''), value)
+ else:
+ module = self.__import_checker_module
+ setattr(module, name, value)
+
+ def __getattribute__(self, name):
+ if name.startswith('_IndirectAttributeAccessChecker__import_checker_'):
+ return object.__getattribute__(self, name.replace('_IndirectAttributeAccessChecker', ''))
+ module = self.__import_checker_module
+ attr = getattr(module, name)
+ if getattr(attr, '__module__', None) is None:
+ return attr
+ if attr.__module__ != module.__name__:
+ import_mod, real_mod = module.__name__, attr.__module__
+ if (import_mod, name, real_mod) in WHITELIST:
+ # No warning for things on the whitelist.
+ pass
+ elif import_mod == 'sys' and name in ['stdin', 'stdout']:
+ # Those are redirected regularly.
+ pass
+ elif import_mod == 'os' and real_mod == 'posix':
+ pass
+ elif name == '__class__' and real_mod == '__builtin__':
+ pass
+ elif (import_mod == 'types' or
+ import_mod.startswith('xml.dom')) and real_mod == '__builtin__':
+ # No warning for the types from the type module that
+ # really originate from builtins
+ pass
+ elif import_mod.split('.') == real_mod.split('.')[:-1]:
+ # Don't warn if the package of a module re-exports a
+ # symbol.
+ pass
+ elif real_mod in sys.modules and hasattr(sys.modules[real_mod], '__file__') and not sys.modules[real_mod].__file__.endswith('.py'):
+ # The real module seems to be a C-Module which is
+ # regularly masked using another Python module in front
+ # of it.
+ pass
+ elif real_mod.split('.')[-1] == '_' + import_mod.split('.')[-1]:
+ # Looks like an internal module, prefixed with _ was
+ # exported to the same module name without an _
+ pass
+ elif real_mod.startswith('_') and real_mod == import_mod.split('.')[-1]:
+ # Looks like a C-module which doesn't declare its
+ # package path correctly.
+ pass
+ else:
+ attr_type = type(attr).__name__
+ print ("WARNING: indirect import of %s `%s.%s` (originally defined at `%s`)"
+ % (attr_type, import_mod, name, real_mod))
+ frame = sys._getframe(1)
+ file = frame.f_code.co_filename
+ line = frame.f_lineno
+ print "caused at %s:%s" % (file, line)
+ return attr
+
+
+class IndirectImportWarner(ihooks.ModuleImporter):
+
+ def import_module(self, name, globals=None, locals=None,
+ fromlist=None):
+ result = ihooks.ModuleImporter.import_module(
+ self, name, globals=globals, locals=locals, fromlist=fromlist)
+ checker = IndirectAttributeAccessChecker(result)
+ if not hasattr(result, '__all__'):
+ checker.__all__ = [x for x in dir(result) if not
+ x.startswith('_')]
+ return checker
+
+ def import_it(self, partname, fqname, parent, force_load=0):
+ result = ihooks.ModuleImporter.import_it(self, partname, fqname,
+ parent, force_load)
+ if result is not None:
+ if hasattr(result, '__file__') and not '.' in os.path.basename(result.__file__):
+ # Smells like a package which didn't get __init__.py
+ # attached to its path.
+ result.__file__ = os.path.join(result.__file__, '__init__.py')
+ return result
+
+
+class ImportChecker(zope.testing.testrunner.feature.Feature):
+ """Monitor indirect imports and warn about them."""
+
+ active = True
+
+ def global_setup(self):
+ ihooks.install(IndirectImportWarner())
+
+ def global_teardown(self):
+ ihooks.uninstall()
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importcheck.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,107 @@
+=============
+Import checks
+=============
+
+Import checks are intended to help finding imports of classes and
+functions that have been moved and are still referred to by their old
+location. We usually re-import them at their old location to keep code
+working, but for cleaning up, it's nice to get a heads-up where we still
+refer to the old locations.
+
+Note: Only objects that support `__module__` can be checked for indirect
+reports. This is especially *not* true for instances.
+
+Test-runner integration
+=======================
+
+This test (as it is a test for the test runner) assumes that the test
+runner has the import check enabled. We extend Python's search path to
+include some example modules that trigger the import checker:
+
+>>> import sys, os.path
+>>> import zope.testing.testrunner
+>>> sys.path.insert(0, os.path.join(os.path.dirname(
+... zope.testing.testrunner.__file__), 'importchecker-fixtures'))
+
+When importing a function or a class from a module that isn't its
+original place of definition, a warning will be printed:
+
+>>> import impcheck0
+>>> import impcheck0a
+>>> import impcheck0b
+WARNING: indirect import of type `impcheck0a.X` (originally defined at `impcheck0`)
+caused at ...impcheck0b.py:NNN
+WARNING: indirect import of function `impcheck0a.y` (originally defined at `impcheck0`)
+caused at ...impcheck0b.py:NNN
+
+
+Heuristics for ignoring re-imports
+==================================
+
+Indirect imports in the standard library
+----------------------------------------
+
+The standard library uses reimports to hide some lower level modules which is
+ok for us.
+
+The types in the `type` module come originally from `__builtin__`. This is ok
+for us and we ignore that:
+
+>>> from types import NoneType
+>>> NoneType.__module__
+'__builtin__'
+
+The `os` module tends to reimport from locations like `posix`:
+
+>>> from os import unlink
+>>> unlink.__module__
+'posix'
+
+The `re` module imports its content from the `sre` module:
+
+>>> from re import match
+
+
+Indirect imports due to the testing environment
+-----------------------------------------------
+
+The test runner and `doctest` redirect the stdout temporarily, so sys.stdout
+can be ignored as well:
+
+>>> from sys import stdout
+>>> stdout.__module__
+'zope.testing.doctest'
+
+
+Indirect imports from packages and their sub-modules
+----------------------------------------------------
+
+We allow indirect imports for the sake of providing an API within a package
+that hides its (sub-)modules. E.g. the following structure is ok, as X is
+defined in module `impcheck1.sub1` and re-imported from `impcheck1`:
+
+>>> from impcheck1 import X
+>>> from impcheck1.sub1 import X
+
+
+However, we do not allow re-imports from sibling modules:
+
+>>> from impcheck1.sub2 import X
+WARNING: indirect import of type `impcheck1.sub2.X` (originally defined at `impcheck1.sub1`)
+caused at <doctest importcheck.txt[15]>:1
+
+
+Also, we only allow one level of re-importing via modules:
+
+>>> from impcheck1 import Y
+WARNING: indirect import of type `impcheck1.Y` (originally defined at `impcheck1.sub3.sub3a`)
+caused at <doctest importcheck.txt[16]>:1
+
+
+Indirect imports from C-modules
+-------------------------------
+
+Often there are indirect imports from modules that are written in C to provide
+a nicer API for it. We ignore symbols that are re-imported from C:
+
+>>> from curses import setupterm
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,6 @@
+
+def y():
+ pass
+
+class X(object):
+ pass
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,3 @@
+# This import is ok, because X originated in impcheck0
+from impcheck0 import X, y
+
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,2 @@
+# This import gives a warning as X originally came from impcheck0
+from impcheck0a import X, y
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,3 @@
+# Make this a Python package
+from impcheck1.sub1 import X
+from impcheck1.sub3.sub3a import Y
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,2 @@
+class X(object):
+ pass
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1 @@
+from impcheck1.sub1 import X
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1 @@
+# Make this a Python package
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,2 @@
+class Y(object):
+ pass
Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py
___________________________________________________________________
Added: svn:eol-style
+ native
Modified: zope.testing/trunk/src/zope/testing/testrunner/options.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/options.py 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/options.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -517,6 +517,7 @@
args = sys.argv
options, positional = parser.parse_args(args[1:], defaults)
+
options.original_testrunner_args = args
if options.color:
Modified: zope.testing/trunk/src/zope/testing/testrunner/runner.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/runner.py 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/runner.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -46,7 +46,9 @@
import zope.testing.testrunner.subprocess
import zope.testing.testrunner.interfaces
import zope.testing.testrunner.debug
+import zope.testing.testrunner.importcheck
+
PYREFCOUNT_PATTERN = re.compile('\[[0-9]+ refs\]')
is_jython = sys.platform.startswith('java')
@@ -191,6 +193,7 @@
self.features.append(zope.testing.testrunner.filter.Filter(self))
self.features.append(zope.testing.testrunner.listing.Listing(self))
self.features.append(zope.testing.testrunner.statistics.Statistics(self))
+ self.features.append(zope.testing.testrunner.importcheck.ImportChecker(self))
# Remove all features that aren't activated
self.features = [f for f in self.features if f.active]
Modified: zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test 2009-01-28 13:55:39 UTC (rev 95313)
@@ -101,7 +101,6 @@
... zope.testing.testrunner.run(
... ['--path', dir, '-Dvv', '--tests-pattern', 'tests2'])
... finally: sys.stdin = real_stdin
- ... # doctest: +ELLIPSIS +REPORT_NDIFF
Running tests at level 1
Running tests2.Layer1 tests:
Set up tests2.Layer1 in 0.000 seconds.
Modified: zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt 2009-01-28 13:55:39 UTC (rev 95313)
@@ -28,6 +28,8 @@
<BLANKLINE>
Module: sampletestsf
<BLANKLINE>
+ Traceback (most recent call last):
+ ...
ImportError: No module named sampletestsf
...
Modified: zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt 2009-01-28 13:55:39 UTC (rev 95313)
@@ -703,11 +703,12 @@
>>> sys.argv = ('test --tests-pattern ^sampletests(f|_i)?$ --layer 1 '
... ).split()
>>> testrunner.run(defaults)
- ... # doctest: +NORMALIZE_WHITESPACE
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Test-module import failures:
<BLANKLINE>
Module: sample2.sampletests_i
<BLANKLINE>
+ ...
File "testrunner-ex/sample2/sampletests_i.py", line 1
importx unittest
^
@@ -717,9 +718,11 @@
Module: sample2.sample21.sampletests_i
<BLANKLINE>
Traceback (most recent call last):
+ ...
File "testrunner-ex/sample2/sample21/sampletests_i.py", line 15, in ?
import zope.testing.huh
- ImportError: No module named huh
+ ...
+ ImportError: No module named zope.testing.huh
<BLANKLINE>
<BLANKLINE>
Module: sample2.sample22.sampletests_i
@@ -730,6 +733,7 @@
Module: sample2.sample23.sampletests_i
<BLANKLINE>
Traceback (most recent call last):
+ ...
File "testrunner-ex/sample2/sample23/sampletests_i.py", line 18, in ?
class Test(unittest.TestCase):
File "testrunner-ex/sample2/sample23/sampletests_i.py", line 23, in Test
Modified: zope.testing/trunk/src/zope/testing/testrunner/tests.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/tests.py 2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/tests.py 2009-01-28 13:55:39 UTC (rev 95313)
@@ -104,6 +104,7 @@
'testrunner-repeat.txt',
'testrunner-gc.txt',
'testrunner-knit.txt',
+ 'importcheck.txt',
setUp=setUp, tearDown=tearDown,
optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE,
checker=checker),