From 0e4f0a742b9bc1f88091a74d658a9cc87fd148c9 Mon Sep 17 00:00:00 2001 From: vaultah Date: Tue, 29 Aug 2017 10:10:07 +0100 Subject: [PATCH 1/3] Implementation of ignore_modules argument for traceback functions --- Doc/library/traceback.rst | 33 +++++++----- Lib/test/test_traceback.py | 69 ++++++++++++++++++++++++- Lib/traceback.py | 101 ++++++++++++++++++++++++++----------- Misc/NEWS | 4 ++ 4 files changed, 165 insertions(+), 42 deletions(-) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 55d331c996a187..5020eddb4ca360 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -36,7 +36,7 @@ The module defines the following functions: Added negative *limit* support. -.. function:: print_exception(etype, value, tb, limit=None, file=None, chain=True) +.. function:: print_exception(etype, value, tb, limit=None, file=None, ignore_modules=(), chain=True) Print exception information and stack trace entries from traceback object *tb* to *file*. This differs from :func:`print_tb` in the following @@ -53,23 +53,27 @@ The module defines the following functions: If *chain* is true (the default), then chained exceptions (the :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be printed as well, like the interpreter itself does when printing an unhandled - exception. + exception. *ignore_modules* should be an iterable containing absolute + module names. Stack trace entries from modules listed in *ignore_modules* will + not be displayed. .. versionchanged:: 3.5 The *etype* argument is ignored and inferred from the type of *value*. + .. versionadded:: 3.7 + *ignore_modules* argument. -.. function:: print_exc(limit=None, file=None, chain=True) +.. function:: print_exc(limit=None, file=None, ignore_modules=(), chain=True) This is a shorthand for ``print_exception(*sys.exc_info(), limit, file, - chain)``. + ignore_modules, chain)``. -.. function:: print_last(limit=None, file=None, chain=True) +.. function:: print_last(limit=None, file=None, ignore_modules=(), chain=True) This is a shorthand for ``print_exception(sys.last_type, sys.last_value, - sys.last_traceback, limit, file, chain)``. In general it will work only - after an exception has reached an interactive prompt (see + sys.last_traceback, limit, file, ignore_modules, chain)``. In general it will + work only after an exception has reached an interactive prompt (see :data:`sys.last_type`). @@ -126,7 +130,7 @@ The module defines the following functions: which exception occurred is the always last string in the list. -.. function:: format_exception(etype, value, tb, limit=None, chain=True) +.. function:: format_exception(etype, value, tb, limit=None, ignore_modules=(), chain=True) Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments to :func:`print_exception`. The @@ -138,7 +142,7 @@ The module defines the following functions: The *etype* argument is ignored and inferred from the type of *value*. -.. function:: format_exc(limit=None, chain=True) +.. function:: format_exc(limit=None, ignore_modules=(), chain=True) This is like ``print_exc(limit)`` but returns a string instead of printing to a file. @@ -239,12 +243,14 @@ capture data for later printing in a lightweight fashion. Note that when locals are captured, they are also shown in the traceback. - .. method:: format(*, chain=True) + .. method:: format(*, ignore_modules=(), chain=True) Format the exception. - If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not - be formatted. + If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not be + formatted. *ignore_modules* should be an iterable containing absolute + module names. Stack trace entries from modules listed in *ignore_modules* + will not be returned. The return value is a generator of strings, each ending in a newline and some containing internal newlines. :func:`~traceback.print_exception` @@ -253,6 +259,9 @@ capture data for later printing in a lightweight fashion. The message indicating which exception occurred is always the last string in the output. + .. versionadded:: 3.7 + *ignore_modules* keyword argument. + .. method:: format_exception_only() Format the exception part of the traceback. diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index e4833535890d15..da453b84286540 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -8,7 +8,7 @@ import re from test import support from test.support import TESTFN, Error, captured_output, unlink, cpython_only -from test.support.script_helper import assert_python_ok +from test.support.script_helper import assert_python_ok, make_script import textwrap import traceback @@ -1047,6 +1047,73 @@ def test_traceback_header(self): self.assertEqual(list(exc.format()), ["Exception: haven\n"]) +class IgnoredModulesTest(unittest.TestCase): + + def test_namespace_package(self): + with support.temp_dir() as pkg_container, \ + support.temp_dir(f'{pkg_container}/pkg') as pkg, \ + support.temp_dir(f'{pkg}/subpkg') as subpkg, \ + support.change_cwd(path=pkg_container): + module_name = make_script(subpkg, 'module', '1/0') + try: + from pkg.subpkg import module + except Exception as e: + tb_exc = traceback.TracebackException.from_exception(e) + + for mod in ('pkg', 'pkg.subpkg', 'pkg.subpkg.module'): + with self.subTest(ignored=mod): + header, *stack, error = tb_exc.format(ignore_modules=(mod,)) + self.assertEqual(len(stack), 1) + self.assertNotIn(module_name, stack[0]) + self.assertIn('ZeroDivisionError', error) + + # The first part of the name isn't the top-level package + for mod in ('pk', 'subpkg.module'): + with self.subTest(ignored=mod): + header, *stack, error = tb_exc.format(ignore_modules=(mod,)) + self.assertEqual(len(stack), 2) + self.assertIn(__file__, stack[0]) + self.assertIn(module_name, stack[1]) + self.assertIn('ZeroDivisionError', error) + + def test_single_file_module(self): + with support.temp_dir() as script_dir: + import runpy + script_name = make_script(script_dir, 'script', '1/0') + try: + runpy.run_path(script_name) + except Exception as e: + tb_exc = traceback.TracebackException.from_exception(e) + + header, *stack, error = tb_exc.format(ignore_modules=('runpy',)) + self.assertEqual(len(stack), 2) + self.assertIn(__file__, stack[0]) + self.assertIn(script_name, stack[1]) + self.assertIn('ZeroDivisionError', error) + + def test_frozen_module(self): + with support.temp_dir() as script_dir, \ + support.change_cwd(path=script_dir): + module_name = make_script(script_dir, 'module', '1/0') + import _frozen_importlib + try: + _frozen_importlib.__import__('module') + except Exception as e: + tb_exc = traceback.TracebackException.from_exception(e) + + # Ignoring these frozen modules and their parent package must + # produce the same traceback + ignored = [('_frozen_importlib', '_frozen_importlib_external'), + ('importlib',)] + for mod in ignored: + with self.subTest(ignored=mod): + header, *stack, error = tb_exc.format(ignore_modules=mod) + self.assertEqual(len(stack), 2) + self.assertIn(__file__, stack[0]) + self.assertIn(module_name, stack[1]) + self.assertIn('ZeroDivisionError', error) + + class MiscTest(unittest.TestCase): def test_all(self): diff --git a/Lib/traceback.py b/Lib/traceback.py index fb3bce12a131f7..47bc7644b828d7 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -2,6 +2,7 @@ import collections import itertools +import importlib.util import linecache import sys @@ -80,7 +81,8 @@ def extract_tb(tb, limit=None): "another exception occurred:\n\n") -def print_exception(etype, value, tb, limit=None, file=None, chain=True): +def print_exception(etype, value, tb, limit=None, file=None, + ignore_modules=(), chain=True): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -96,12 +98,12 @@ def print_exception(etype, value, tb, limit=None, file=None, chain=True): # ignore it here (rather than in the new TracebackException API). if file is None: file = sys.stderr - for line in TracebackException( - type(value), value, tb, limit=limit).format(chain=chain): + tb_exc = TracebackException(type(value), value, tb, limit=limit) + for line in tb_exc.format(ignore_modules=ignore_modules, chain=chain): print(line, file=file, end="") -def format_exception(etype, value, tb, limit=None, chain=True): +def format_exception(etype, value, tb, limit=None, ignore_modules=(), chain=True): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -113,8 +115,8 @@ def format_exception(etype, value, tb, limit=None, chain=True): # format_exception has ignored etype for some time, and code such as cgitb # passes in bogus values as a result. For compatibility with such code we # ignore it here (rather than in the new TracebackException API). - return list(TracebackException( - type(value), value, tb, limit=limit).format(chain=chain)) + tb_exc = TracebackException(type(value), value, tb, limit=limit) + return list(tb_exc.format(ignore_modules=ignore_modules, chain=chain)) def format_exception_only(etype, value): @@ -154,21 +156,24 @@ def _some_str(value): # -- -def print_exc(limit=None, file=None, chain=True): +def print_exc(limit=None, file=None, ignore_modules=(), chain=True): """Shorthand for 'print_exception(*sys.exc_info(), limit, file)'.""" - print_exception(*sys.exc_info(), limit=limit, file=file, chain=chain) + print_exception(*sys.exc_info(), limit=limit, file=file, + ignore_modules=ignore_modules, chain=chain) -def format_exc(limit=None, chain=True): +def format_exc(limit=None, ignore_modules=(), chain=True): """Like print_exc() but return a string.""" - return "".join(format_exception(*sys.exc_info(), limit=limit, chain=chain)) + return "".join(format_exception( + *sys.exc_info(), limit=limit, + ignore_modules=ignore_modules, chain=chain)) -def print_last(limit=None, file=None, chain=True): +def print_last(limit=None, file=None, ignore_modules=(), chain=True): """This is a shorthand for 'print_exception(sys.last_type, sys.last_value, sys.last_traceback, limit, file)'.""" if not hasattr(sys, "last_type"): raise ValueError("no last exception") print_exception(sys.last_type, sys.last_value, sys.last_traceback, - limit, file, chain) + limit, file, ignore_modules, chain) # # Printing and Extracting Stacks. @@ -306,6 +311,20 @@ def walk_tb(tb): tb = tb.tb_next +def _walk_with_limit(frame_gen, limit=None): + if limit is None: + limit = getattr(sys, 'tracebacklimit', None) + if limit is not None and limit < 0: + limit = 0 + if limit is not None: + if limit >= 0: + return itertools.islice(frame_gen, limit) + else: + return collections.deque(frame_gen, maxlen=-limit) + + return frame_gen + + class StackSummary(list): """A stack of frames.""" @@ -323,19 +342,10 @@ def extract(klass, frame_gen, *, limit=None, lookup_lines=True, :param capture_locals: If True, the local variables from each frame will be captured as object representations into the FrameSummary. """ - if limit is None: - limit = getattr(sys, 'tracebacklimit', None) - if limit is not None and limit < 0: - limit = 0 - if limit is not None: - if limit >= 0: - frame_gen = itertools.islice(frame_gen, limit) - else: - frame_gen = collections.deque(frame_gen, maxlen=-limit) - result = klass() fnames = set() - for f, lineno in frame_gen: + + for f, lineno in _walk_with_limit(frame_gen, limit): co = f.f_code filename = co.co_filename name = co.co_name @@ -491,9 +501,11 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.__suppress_context__ = \ exc_value.__suppress_context__ if exc_value else False # TODO: locals. + # Keep actual frame objects to access module attributes + self._raw_stack = list(_walk_with_limit(walk_tb(exc_traceback), limit=limit)) self.stack = StackSummary.extract( - walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines, - capture_locals=capture_locals) + self._raw_stack, limit=limit, + lookup_lines=lookup_lines, capture_locals=capture_locals) self.exc_type = exc_type # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line @@ -513,6 +525,23 @@ def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) + def _check_frame_source(self, frame, with_dots): + # Non-public helper method. Module names are expected to be + # absolute and have trailing dots. Since __spec__.name may differ + # from __name__, both are considered. + name = frame.f_globals.get('__name__') + if name is not None: + if f'{name}.'.startswith(with_dots): + return True + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): + pass + else: + if spec and f'{spec.name}.'.startswith(with_dots): + return True + return False + def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: @@ -573,11 +602,15 @@ def format_exception_only(self): msg = self.msg or "" yield "{}: {}\n".format(stype, msg) - def format(self, *, chain=True): + def format(self, *, ignore_modules=(), chain=True): """Format the exception. If chain is not *True*, *__cause__* and *__context__* will not be formatted. + ignore_modules should be an iterable containing absolute names of + modules. Stack trace entries from modules listed in ignore_modules will + not be returned. + The return value is a generator of strings, each ending in a newline and some containing internal newlines. `print_exception` is a wrapper around this method which just prints the lines to a file. @@ -587,13 +620,23 @@ def format(self, *, chain=True): """ if chain: if self.__cause__ is not None: - yield from self.__cause__.format(chain=chain) + yield from self.__cause__.format( + ignore_modules=ignore_modules, chain=chain) yield _cause_message elif (self.__context__ is not None and not self.__suppress_context__): - yield from self.__context__.format(chain=chain) + yield from self.__context__.format( + ignore_modules=ignore_modules, chain=chain) yield _context_message if self.exc_traceback is not None: yield 'Traceback (most recent call last):\n' - yield from self.stack.format() + + if ignore_modules: + ignored_with_dot = tuple([f'{mod}.' for mod in ignore_modules]) + for (frame, _), formatted in zip(self._raw_stack, self.stack.format()): + if not self._check_frame_source(frame, ignored_with_dot): + yield formatted + else: + yield from self.stack.format() + yield from self.format_exception_only() diff --git a/Misc/NEWS b/Misc/NEWS index 9927e3490f90c6..90156ac9eb9d59 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -379,6 +379,10 @@ Extension Modules Library ------- +- bpo-31299: ``traceback.TracebackException.format`` and :mod:`traceback` + functions that use it now accept the ``ignore_modules`` argument: a list of + modules, stack trace entries from which should be hidden. + - bpo-30119: ftplib.FTP.putline() now throws ValueError on commands that contains CR or LF. Patch by Dong-hee Na. From af416551ba902746d62cedbb6d43cdd6858ee2d0 Mon Sep 17 00:00:00 2001 From: vaultah Date: Tue, 29 Aug 2017 11:02:20 +0100 Subject: [PATCH 2/3] Removed the news entry from Misc/NEWS, readded it to Misc/NEWS.d --- Misc/NEWS | 4 ---- .../next/Library/2017-08-29-10-52-12.bpo-31299.-a7atn.rst | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-08-29-10-52-12.bpo-31299.-a7atn.rst diff --git a/Misc/NEWS b/Misc/NEWS index 90156ac9eb9d59..9927e3490f90c6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -379,10 +379,6 @@ Extension Modules Library ------- -- bpo-31299: ``traceback.TracebackException.format`` and :mod:`traceback` - functions that use it now accept the ``ignore_modules`` argument: a list of - modules, stack trace entries from which should be hidden. - - bpo-30119: ftplib.FTP.putline() now throws ValueError on commands that contains CR or LF. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2017-08-29-10-52-12.bpo-31299.-a7atn.rst b/Misc/NEWS.d/next/Library/2017-08-29-10-52-12.bpo-31299.-a7atn.rst new file mode 100644 index 00000000000000..d587eece81a8f1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-29-10-52-12.bpo-31299.-a7atn.rst @@ -0,0 +1,3 @@ +``traceback.TracebackException.format`` and :mod:`traceback` functions that +use it now accept the ``ignore_modules`` argument: a list of modules, stack +trace entries from which should be hidden. From 5a7dfbe804a2b5c86c1d90cac90724c2cb335f2f Mon Sep 17 00:00:00 2001 From: vaultah Date: Tue, 29 Aug 2017 13:37:12 +0100 Subject: [PATCH 3/3] More robust tests (this should also fix the AppVeyor build) --- Lib/test/test_traceback.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index da453b84286540..eac236b22bed82 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1054,26 +1054,26 @@ def test_namespace_package(self): support.temp_dir(f'{pkg_container}/pkg') as pkg, \ support.temp_dir(f'{pkg}/subpkg') as subpkg, \ support.change_cwd(path=pkg_container): - module_name = make_script(subpkg, 'module', '1/0') + make_script(subpkg, 'module', '1/0') try: from pkg.subpkg import module except Exception as e: tb_exc = traceback.TracebackException.from_exception(e) + _, *original_stack, _ = tb_exc.format() + for mod in ('pkg', 'pkg.subpkg', 'pkg.subpkg.module'): with self.subTest(ignored=mod): - header, *stack, error = tb_exc.format(ignore_modules=(mod,)) - self.assertEqual(len(stack), 1) - self.assertNotIn(module_name, stack[0]) + _, *clean_stack, error = tb_exc.format(ignore_modules=(mod,)) + self.assertEqual(clean_stack, original_stack[:1]) self.assertIn('ZeroDivisionError', error) - # The first part of the name isn't the top-level package + # The first part of the name isn't our top-level package, nothing + # should be removed for mod in ('pk', 'subpkg.module'): with self.subTest(ignored=mod): - header, *stack, error = tb_exc.format(ignore_modules=(mod,)) - self.assertEqual(len(stack), 2) - self.assertIn(__file__, stack[0]) - self.assertIn(module_name, stack[1]) + _, *clean_stack, error = tb_exc.format(ignore_modules=(mod,)) + self.assertEqual(clean_stack, original_stack) self.assertIn('ZeroDivisionError', error) def test_single_file_module(self): @@ -1085,32 +1085,30 @@ def test_single_file_module(self): except Exception as e: tb_exc = traceback.TracebackException.from_exception(e) - header, *stack, error = tb_exc.format(ignore_modules=('runpy',)) - self.assertEqual(len(stack), 2) - self.assertIn(__file__, stack[0]) - self.assertIn(script_name, stack[1]) + _, *original_stack, _ = tb_exc.format() + _, *clean_stack, error = tb_exc.format(ignore_modules=('runpy',)) + self.assertEqual(clean_stack, [original_stack[0], original_stack[-1]]) self.assertIn('ZeroDivisionError', error) def test_frozen_module(self): with support.temp_dir() as script_dir, \ support.change_cwd(path=script_dir): - module_name = make_script(script_dir, 'module', '1/0') + make_script(script_dir, 'module', '1/0') import _frozen_importlib try: _frozen_importlib.__import__('module') except Exception as e: tb_exc = traceback.TracebackException.from_exception(e) + _, *original_stack, _ = tb_exc.format() # Ignoring these frozen modules and their parent package must # produce the same traceback ignored = [('_frozen_importlib', '_frozen_importlib_external'), ('importlib',)] for mod in ignored: with self.subTest(ignored=mod): - header, *stack, error = tb_exc.format(ignore_modules=mod) - self.assertEqual(len(stack), 2) - self.assertIn(__file__, stack[0]) - self.assertIn(module_name, stack[1]) + _, *clean_stack, error = tb_exc.format(ignore_modules=mod) + self.assertEqual(clean_stack, [original_stack[0], original_stack[-1]]) self.assertIn('ZeroDivisionError', error)