diff --git a/.coveragerc b/.coveragerc index a0a7a02f2..9ef955843 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,4 @@ [run] source = _pytest,testing parallel = 1 +branch = 1 diff --git a/.gitignore b/.gitignore index afb6bf9fd..2ae5ea752 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ env/ .ropeproject .idea .hypothesis +.pydevproject +.project +.settings diff --git a/AUTHORS b/AUTHORS index 144eb19db..2d59a1b0f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -73,6 +73,7 @@ Endre Galaczi Eric Hunsberger Eric Siegerman Erik M. Bray +Fabio Zadrozny Feng Ma Florian Bruhin Floris Bruynooghe diff --git a/appveyor.yml b/appveyor.yml index 40a171021..11e57f0c8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,7 +15,7 @@ environment: - TOXENV: "py35" - TOXENV: "py36" - TOXENV: "py37" -# - TOXENV: "pypy" reenable when we are able to provide a scandir wheel or build scandir + - TOXENV: "pypy" - TOXENV: "py27-pexpect" - TOXENV: "py27-xdist" - TOXENV: "py27-trial" diff --git a/changelog/3566.doc.rst b/changelog/3566.doc.rst new file mode 100644 index 000000000..d8eda4241 --- /dev/null +++ b/changelog/3566.doc.rst @@ -0,0 +1 @@ +Added a blurb in usage.rst for the usage of -r flag which is used to show an extra test summary info. diff --git a/changelog/3911.bugfix.rst b/changelog/3911.bugfix.rst new file mode 100644 index 000000000..8839fe7d9 --- /dev/null +++ b/changelog/3911.bugfix.rst @@ -0,0 +1 @@ +Terminal writer now takes into account unicode character width when writing out progress. diff --git a/changelog/3918.bugfix.rst b/changelog/3918.bugfix.rst new file mode 100644 index 000000000..7ba811916 --- /dev/null +++ b/changelog/3918.bugfix.rst @@ -0,0 +1 @@ +Improve performance of assertion rewriting. diff --git a/doc/en/conf.py b/doc/en/conf.py index 5941716ab..3e881ea01 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -329,7 +329,7 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"python": ("http://docs.python.org/3", None)} +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} def setup(app): diff --git a/doc/en/talks.rst b/doc/en/talks.rst index 79534ea6e..c4310b522 100644 --- a/doc/en/talks.rst +++ b/doc/en/talks.rst @@ -14,6 +14,9 @@ Talks and Tutorials Books --------------------------------------------- +- `pytest Quick Start Guide, by Bruno Oliveira (2018) + `_. + - `Python Testing with pytest, by Brian Okken (2017) `_. diff --git a/doc/en/usage.rst b/doc/en/usage.rst index a5418df22..f1f0c079e 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -140,6 +140,48 @@ will be shown (because KeyboardInterrupt is caught by pytest). By using this option you make sure a trace is shown. +.. _`pytest.detailed_failed_tests_usage`: + +Detailed summary report +----------------------- + +.. versionadded:: 2.9 + +The ``-r`` flag can be used to display test results summary at the end of the test session, +making it easy in large test suites to get a clear picture of all failures, skips, xfails, etc. + +Example:: + + $ pytest -ra + ======================== test session starts ======================== + ... + ====================== short test summary info ====================== + FAIL summary\test_foo.py::test_1 + SKIP [1] summary\test_foo.py:12: not supported in this platform + XPASS summary\test_bar.py::test_4 flaky + + ===== 1 failed, 1 passed, 1 skipped, 1 xpassed in 0.08 seconds ====== + + +The ``-r`` options accepts a number of characters after it, with ``a`` used above meaning "all except passes". + +Here is the full list of available characters that can be used: + + - ``f`` - failed + - ``E`` - error + - ``s`` - skipped + - ``x`` - xfailed + - ``X`` - xpassed + - ``p`` - passed + - ``P`` - passed with output + - ``a`` - all except ``pP`` + +More than one character can be used, so for example to only see failed and skipped tests, you can execute:: + + $ pytest -rfs + + + .. _pdb-option: Dropping to PDB_ (Python Debugger) on failures diff --git a/setup.py b/setup.py index 6207ad09b..4c12fbfcc 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ def get_environment_marker_support_level(): def main(): extras_require = {} install_requires = [ - "py>=1.5.0", + "py>=1.5.0", # if py gets upgrade to >=1.6, remove _width_of_current_line in terminal.py "six>=1.10.0", "setuptools", "attrs>=17.4.0", diff --git a/src/_pytest/assertion/rewrite.py b/src/_pytest/assertion/rewrite.py index 1f485eb89..738d63396 100644 --- a/src/_pytest/assertion/rewrite.py +++ b/src/_pytest/assertion/rewrite.py @@ -67,14 +67,24 @@ class AssertionRewritingHook(object): # flag to guard against trying to rewrite a pyc file while we are already writing another pyc file, # which might result in infinite recursion (#3506) self._writing_pyc = False + self._basenames_to_check_rewrite = {"conftest"} + self._marked_for_rewrite_cache = {} + self._session_paths_checked = False def set_session(self, session): self.session = session + self._session_paths_checked = False + + def _imp_find_module(self, name, path=None): + """Indirection so we can mock calls to find_module originated from the hook during testing""" + return imp.find_module(name, path) def find_module(self, name, path=None): if self._writing_pyc: return None state = self.config._assertstate + if self._early_rewrite_bailout(name, state): + return None state.trace("find_module called for: %s" % name) names = name.rsplit(".", 1) lastname = names[-1] @@ -87,7 +97,7 @@ class AssertionRewritingHook(object): pth = path[0] if pth is None: try: - fd, fn, desc = imp.find_module(lastname, path) + fd, fn, desc = self._imp_find_module(lastname, path) except ImportError: return None if fd is not None: @@ -166,6 +176,44 @@ class AssertionRewritingHook(object): self.modules[name] = co, pyc return self + def _early_rewrite_bailout(self, name, state): + """ + This is a fast way to get out of rewriting modules. Profiling has + shown that the call to imp.find_module (inside of the find_module + from this class) is a major slowdown, so, this method tries to + filter what we're sure won't be rewritten before getting to it. + """ + if self.session is not None and not self._session_paths_checked: + self._session_paths_checked = True + for path in self.session._initialpaths: + # Make something as c:/projects/my_project/path.py -> + # ['c:', 'projects', 'my_project', 'path.py'] + parts = str(path).split(os.path.sep) + # add 'path' to basenames to be checked. + self._basenames_to_check_rewrite.add(os.path.splitext(parts[-1])[0]) + + # Note: conftest already by default in _basenames_to_check_rewrite. + parts = name.split(".") + if parts[-1] in self._basenames_to_check_rewrite: + return False + + # For matching the name it must be as if it was a filename. + parts[-1] = parts[-1] + ".py" + fn_pypath = py.path.local(os.path.sep.join(parts)) + for pat in self.fnpats: + # if the pattern contains subdirectories ("tests/**.py" for example) we can't bail out based + # on the name alone because we need to match against the full path + if os.path.dirname(pat): + return False + if fn_pypath.fnmatch(pat): + return False + + if self._is_marked_for_rewrite(name, state): + return False + + state.trace("early skip of rewriting module: %s" % (name,)) + return True + def _should_rewrite(self, name, fn_pypath, state): # always rewrite conftest files fn = str(fn_pypath) @@ -185,12 +233,20 @@ class AssertionRewritingHook(object): state.trace("matched test file %r" % (fn,)) return True - for marked in self._must_rewrite: - if name == marked or name.startswith(marked + "."): - state.trace("matched marked file %r (from %r)" % (name, marked)) - return True + return self._is_marked_for_rewrite(name, state) - return False + def _is_marked_for_rewrite(self, name, state): + try: + return self._marked_for_rewrite_cache[name] + except KeyError: + for marked in self._must_rewrite: + if name == marked or name.startswith(marked + "."): + state.trace("matched marked file %r (from %r)" % (name, marked)) + self._marked_for_rewrite_cache[name] = True + return True + + self._marked_for_rewrite_cache[name] = False + return False def mark_rewrite(self, *names): """Mark import names as needing to be rewritten. @@ -207,6 +263,7 @@ class AssertionRewritingHook(object): ): self._warn_already_imported(name) self._must_rewrite.update(names) + self._marked_for_rewrite_cache.clear() def _warn_already_imported(self, name): from _pytest.warning_types import PytestWarning @@ -245,7 +302,7 @@ class AssertionRewritingHook(object): def is_package(self, name): try: - fd, fn, desc = imp.find_module(name) + fd, fn, desc = self._imp_find_module(name) except ImportError: return False if fd is not None: diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 993e091f2..bc45d65a9 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -51,6 +51,8 @@ def main(args=None, plugins=None): :arg plugins: list of plugin objects to be auto-registered during initialization. """ + from _pytest.main import EXIT_USAGEERROR + try: try: config = _prepareconfig(args, plugins) @@ -69,7 +71,7 @@ def main(args=None, plugins=None): tw = py.io.TerminalWriter(sys.stderr) for msg in e.args: tw.line("ERROR: {}\n".format(msg), red=True) - return 4 + return EXIT_USAGEERROR class cmdline(object): # compatibility namespace diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 947c6aa4b..f5078b9e7 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -383,6 +383,7 @@ class Session(nodes.FSCollector): self.trace = config.trace.root.get("collection") self._norecursepatterns = config.getini("norecursedirs") self.startdir = py.path.local() + self._initialpaths = frozenset() # Keep track of any collected nodes in here, so we don't duplicate fixtures self._node_cache = {} @@ -441,13 +442,14 @@ class Session(nodes.FSCollector): self.trace("perform_collect", self, args) self.trace.root.indent += 1 self._notfound = [] - self._initialpaths = set() + initialpaths = [] self._initialparts = [] self.items = items = [] for arg in args: parts = self._parsearg(arg) self._initialparts.append(parts) - self._initialpaths.add(parts[0]) + initialpaths.append(parts[0]) + self._initialpaths = frozenset(initialpaths) rep = collect_one_node(self) self.ihook.pytest_collectreport(report=rep) self.trace.root.indent -= 1 @@ -564,7 +566,6 @@ class Session(nodes.FSCollector): """Convert a dotted module name to path. """ - try: with _patched_find_module(): loader = pkgutil.find_loader(x) diff --git a/src/_pytest/outcomes.py b/src/_pytest/outcomes.py index 0a66fcab4..f6093ef76 100644 --- a/src/_pytest/outcomes.py +++ b/src/_pytest/outcomes.py @@ -67,13 +67,19 @@ exit.Exception = Exit def skip(msg="", **kwargs): - """ skip an executing test with the given message. Note: it's usually - better to use the pytest.mark.skipif marker to declare a test to be - skipped under certain conditions like mismatching platforms or - dependencies. See the pytest_skipping plugin for details. + """ + Skip an executing test with the given message. + + This function should be called only during testing (setup, call or teardown) or + during collection by using the ``allow_module_level`` flag. :kwarg bool allow_module_level: allows this function to be called at module level, skipping the rest of the module. Default to False. + + .. note:: + It is better to use the :ref:`pytest.mark.skipif ref` marker when possible to declare a test to be + skipped under certain conditions like mismatching platforms or + dependencies. """ __tracebackhide__ = True allow_module_level = kwargs.pop("allow_module_level", False) @@ -87,10 +93,12 @@ skip.Exception = Skipped def fail(msg="", pytrace=True): - """ explicitly fail a currently-executing test with the given Message. + """ + Explicitly fail an executing test with the given message. - :arg pytrace: if false the msg represents the full failure information - and no python traceback will be reported. + :param str msg: the message to show the user as reason for the failure. + :param bool pytrace: if false the msg represents the full failure information and no + python traceback will be reported. """ __tracebackhide__ = True raise Failed(msg=msg, pytrace=pytrace) @@ -104,7 +112,15 @@ class XFailed(fail.Exception): def xfail(reason=""): - """ xfail an executing test or setup functions with the given reason.""" + """ + Imperatively xfail an executing test or setup functions with the given reason. + + This function should be called only during testing (setup, call or teardown). + + .. note:: + It is better to use the :ref:`pytest.mark.xfail ref` marker when possible to declare a test to be + xfailed under certain conditions like known bugs or missing features. + """ __tracebackhide__ = True raise XFailed(reason) diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index 4f6f09537..49a9a33fb 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -435,10 +435,8 @@ class TerminalReporter(object): if last_item: self._write_progress_information_filling_space() else: - past_edge = ( - self._tw.chars_on_current_line + progress_length + 1 - >= self._screen_width - ) + w = self._width_of_current_line + past_edge = w + progress_length + 1 >= self._screen_width if past_edge: msg = self._get_progress_information_message() self._tw.write(msg + "\n", cyan=True) @@ -462,10 +460,18 @@ class TerminalReporter(object): def _write_progress_information_filling_space(self): msg = self._get_progress_information_message() - fill = " " * ( - self._tw.fullwidth - self._tw.chars_on_current_line - len(msg) - 1 - ) - self.write(fill + msg, cyan=True) + w = self._width_of_current_line + fill = self._tw.fullwidth - w - 1 + self.write(msg.rjust(fill), cyan=True) + + @property + def _width_of_current_line(self): + """Return the width of current line, using the superior implementation of py-1.6 when available""" + try: + return self._tw.width_of_current_line + except AttributeError: + # py < 1.6.0 + return self._tw.chars_on_current_line def pytest_collection(self): if not self.isatty and self.config.option.verbose >= 1: diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index c4cafcaab..aaf3e4785 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -1106,22 +1106,21 @@ class TestIssue925(object): class TestIssue2121: - def test_simple(self, testdir): - testdir.tmpdir.join("tests/file.py").ensure().write( - """ -def test_simple_failure(): - assert 1 + 1 == 3 -""" - ) - testdir.tmpdir.join("pytest.ini").write( - textwrap.dedent( + def test_rewrite_python_files_contain_subdirs(self, testdir): + testdir.makepyfile( + **{ + "tests/file.py": """ + def test_simple_failure(): + assert 1 + 1 == 3 """ - [pytest] - python_files = tests/**.py - """ - ) + } + ) + testdir.makeini( + """ + [pytest] + python_files = tests/**.py + """ ) - result = testdir.runpytest() result.stdout.fnmatch_lines("*E*assert (1 + 1) == 3") @@ -1153,3 +1152,83 @@ def test_rewrite_infinite_recursion(testdir, pytestconfig, monkeypatch): hook = AssertionRewritingHook(pytestconfig) assert hook.find_module("test_foo") is not None assert len(write_pyc_called) == 1 + + +class TestEarlyRewriteBailout(object): + @pytest.fixture + def hook(self, pytestconfig, monkeypatch, testdir): + """Returns a patched AssertionRewritingHook instance so we can configure its initial paths and track + if imp.find_module has been called. + """ + import imp + + self.find_module_calls = [] + self.initial_paths = set() + + class StubSession(object): + _initialpaths = self.initial_paths + + def isinitpath(self, p): + return p in self._initialpaths + + def spy_imp_find_module(name, path): + self.find_module_calls.append(name) + return imp.find_module(name, path) + + hook = AssertionRewritingHook(pytestconfig) + # use default patterns, otherwise we inherit pytest's testing config + hook.fnpats[:] = ["test_*.py", "*_test.py"] + monkeypatch.setattr(hook, "_imp_find_module", spy_imp_find_module) + hook.set_session(StubSession()) + testdir.syspathinsert() + return hook + + def test_basic(self, testdir, hook): + """ + Ensure we avoid calling imp.find_module when we know for sure a certain module will not be rewritten + to optimize assertion rewriting (#3918). + """ + testdir.makeconftest( + """ + import pytest + @pytest.fixture + def fix(): return 1 + """ + ) + testdir.makepyfile(test_foo="def test_foo(): pass") + testdir.makepyfile(bar="def bar(): pass") + foobar_path = testdir.makepyfile(foobar="def foobar(): pass") + self.initial_paths.add(foobar_path) + + # conftest files should always be rewritten + assert hook.find_module("conftest") is not None + assert self.find_module_calls == ["conftest"] + + # files matching "python_files" mask should always be rewritten + assert hook.find_module("test_foo") is not None + assert self.find_module_calls == ["conftest", "test_foo"] + + # file does not match "python_files": early bailout + assert hook.find_module("bar") is None + assert self.find_module_calls == ["conftest", "test_foo"] + + # file is an initial path (passed on the command-line): should be rewritten + assert hook.find_module("foobar") is not None + assert self.find_module_calls == ["conftest", "test_foo", "foobar"] + + def test_pattern_contains_subdirectories(self, testdir, hook): + """If one of the python_files patterns contain subdirectories ("tests/**.py") we can't bailout early + because we need to match with the full path, which can only be found by calling imp.find_module. + """ + p = testdir.makepyfile( + **{ + "tests/file.py": """ + def test_simple_failure(): + assert 1 + 1 == 3 + """ + } + ) + testdir.syspathinsert(p.dirpath()) + hook.fnpats[:] = ["tests/**.py"] + assert hook.find_module("file") is not None + assert self.find_module_calls == ["file"]