diff --git a/doc/en/changelog.rst b/doc/en/changelog.rst index 2806fb6a3..40b5ee690 100644 --- a/doc/en/changelog.rst +++ b/doc/en/changelog.rst @@ -287,7 +287,7 @@ Bug Fixes - `#6646 `_: Assertion rewriting hooks are (re)stored for the current item, which fixes them being still used after e.g. pytester's :func:`testdir.runpytest <_pytest.pytester.Testdir.runpytest>` etc. -- `#6660 `_: :func:`pytest.exit() <_pytest.outcomes.exit>` is handled when emitted from the :func:`pytest_sessionfinish <_pytest.hookspec.pytest_sessionfinish>` hook. This includes quitting from a debugger. +- `#6660 `_: :py:func:`pytest.exit` is handled when emitted from the :func:`pytest_sessionfinish <_pytest.hookspec.pytest_sessionfinish>` hook. This includes quitting from a debugger. - `#6752 `_: When :py:func:`pytest.raises` is used as a function (as opposed to a context manager), @@ -399,7 +399,7 @@ Improvements - `#6231 `_: Improve check for misspelling of :ref:`pytest.mark.parametrize ref`. -- `#6257 `_: Handle :py:func:`_pytest.outcomes.exit` being used via :py:func:`~_pytest.hookspec.pytest_internalerror`, e.g. when quitting pdb from post mortem. +- `#6257 `_: Handle :py:func:`pytest.exit` being used via :py:func:`~_pytest.hookspec.pytest_internalerror`, e.g. when quitting pdb from post mortem. diff --git a/doc/en/reference.rst b/doc/en/reference.rst index d5580ad65..94a247095 100644 --- a/doc/en/reference.rst +++ b/doc/en/reference.rst @@ -15,41 +15,41 @@ Functions pytest.approx ~~~~~~~~~~~~~ -.. autofunction:: _pytest.python_api.approx +.. autofunction:: pytest.approx pytest.fail ~~~~~~~~~~~ **Tutorial**: :ref:`skipping` -.. autofunction:: _pytest.outcomes.fail +.. autofunction:: pytest.fail pytest.skip ~~~~~~~~~~~ -.. autofunction:: _pytest.outcomes.skip(msg, [allow_module_level=False]) +.. autofunction:: pytest.skip(msg, [allow_module_level=False]) .. _`pytest.importorskip ref`: pytest.importorskip ~~~~~~~~~~~~~~~~~~~ -.. autofunction:: _pytest.outcomes.importorskip +.. autofunction:: pytest.importorskip pytest.xfail ~~~~~~~~~~~~ -.. autofunction:: _pytest.outcomes.xfail +.. autofunction:: pytest.xfail pytest.exit ~~~~~~~~~~~ -.. autofunction:: _pytest.outcomes.exit +.. autofunction:: pytest.exit pytest.main ~~~~~~~~~~~ -.. autofunction:: _pytest.config.main +.. autofunction:: pytest.main pytest.param ~~~~~~~~~~~~ @@ -644,31 +644,6 @@ Initialization hooks called for plugins and ``conftest.py`` files. .. autofunction:: pytest_plugin_registered -Test running hooks -~~~~~~~~~~~~~~~~~~ - -All runtest related hooks receive a :py:class:`pytest.Item <_pytest.main.Item>` object. - -.. autofunction:: pytest_runtestloop -.. autofunction:: pytest_runtest_protocol -.. autofunction:: pytest_runtest_logstart -.. autofunction:: pytest_runtest_logfinish -.. autofunction:: pytest_runtest_setup -.. autofunction:: pytest_runtest_call -.. autofunction:: pytest_runtest_teardown -.. autofunction:: pytest_runtest_makereport - -For deeper understanding you may look at the default implementation of -these hooks in :py:mod:`_pytest.runner` and maybe also -in :py:mod:`_pytest.pdb` which interacts with :py:mod:`_pytest.capture` -and its input/output capturing in order to immediately drop -into interactive debugging when a test failure occurs. - -The :py:mod:`_pytest.terminal` reported specifically uses -the reporting hook to print information about a test run. - -.. autofunction:: pytest_pyfunc_call - Collection hooks ~~~~~~~~~~~~~~~~ @@ -694,6 +669,28 @@ items, delete or otherwise amend the test items: .. autofunction:: pytest_collection_finish +Test running (runtest) hooks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All runtest related hooks receive a :py:class:`pytest.Item <_pytest.main.Item>` object. + +.. autofunction:: pytest_runtestloop +.. autofunction:: pytest_runtest_protocol +.. autofunction:: pytest_runtest_logstart +.. autofunction:: pytest_runtest_logfinish +.. autofunction:: pytest_runtest_setup +.. autofunction:: pytest_runtest_call +.. autofunction:: pytest_runtest_teardown +.. autofunction:: pytest_runtest_makereport + +For deeper understanding you may look at the default implementation of +these hooks in :py:mod:`_pytest.runner` and maybe also +in :py:mod:`_pytest.pdb` which interacts with :py:mod:`_pytest.capture` +and its input/output capturing in order to immediately drop +into interactive debugging when a test failure occurs. + +.. autofunction:: pytest_pyfunc_call + Reporting hooks ~~~~~~~~~~~~~~~ @@ -762,6 +759,14 @@ Collector :members: :show-inheritance: +CollectReport +~~~~~~~~~~~~~ + +.. autoclass:: _pytest.reports.CollectReport() + :members: + :show-inheritance: + :inherited-members: + Config ~~~~~~ @@ -881,7 +886,7 @@ Session TestReport ~~~~~~~~~~ -.. autoclass:: _pytest.runner.TestReport() +.. autoclass:: _pytest.reports.TestReport() :members: :show-inheritance: :inherited-members: diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 1c1726d53..eba6f5ba9 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -361,18 +361,28 @@ def pytest_make_parametrize_id( # ------------------------------------------------------------------------- -# generic runtest related hooks +# runtest related hooks # ------------------------------------------------------------------------- @hookspec(firstresult=True) def pytest_runtestloop(session: "Session") -> Optional[object]: - """ called for performing the main runtest loop - (after collection finished). + """Performs the main runtest loop (after collection finished). - Stops at first non-None result, see :ref:`firstresult` + The default hook implementation performs the runtest protocol for all items + collected in the session (``session.items``), unless the collection failed + or the ``collectonly`` pytest option is set. - :param _pytest.main.Session session: the pytest session object + If at any point :py:func:`pytest.exit` is called, the loop is + terminated immediately. + + If at any point ``session.shouldfail`` or ``session.shouldstop`` are set, the + loop is terminated after the runtest protocol for the current item is finished. + + :param _pytest.main.Session session: The pytest session object. + + Stops at first non-None result, see :ref:`firstresult`. + The return value is not used, but only stops further processing. """ @@ -380,56 +390,91 @@ def pytest_runtestloop(session: "Session") -> Optional[object]: def pytest_runtest_protocol( item: "Item", nextitem: "Optional[Item]" ) -> Optional[object]: - """ implements the runtest_setup/call/teardown protocol for - the given test item, including capturing exceptions and calling - reporting hooks. + """Performs the runtest protocol for a single test item. - :arg item: test item for which the runtest protocol is performed. + The default runtest protocol is this (see individual hooks for full details): - :arg nextitem: the scheduled-to-be-next test item (or None if this - is the end my friend). This argument is passed on to - :py:func:`pytest_runtest_teardown`. + - ``pytest_runtest_logstart(nodeid, location)`` - :return boolean: True if no further hook implementations should be invoked. + - Setup phase: + - ``call = pytest_runtest_setup(item)`` (wrapped in ``CallInfo(when="setup")``) + - ``report = pytest_runtest_makereport(item, call)`` + - ``pytest_runtest_logreport(report)`` + - ``pytest_exception_interact(call, report)`` if an interactive exception occurred + - Call phase, if the the setup passed and the ``setuponly`` pytest option is not set: + - ``call = pytest_runtest_call(item)`` (wrapped in ``CallInfo(when="call")``) + - ``report = pytest_runtest_makereport(item, call)`` + - ``pytest_runtest_logreport(report)`` + - ``pytest_exception_interact(call, report)`` if an interactive exception occurred - Stops at first non-None result, see :ref:`firstresult` """ + - Teardown phase: + - ``call = pytest_runtest_teardown(item, nextitem)`` (wrapped in ``CallInfo(when="teardown")``) + - ``report = pytest_runtest_makereport(item, call)`` + - ``pytest_runtest_logreport(report)`` + - ``pytest_exception_interact(call, report)`` if an interactive exception occurred + - ``pytest_runtest_logfinish(nodeid, location)`` -def pytest_runtest_logstart(nodeid, location): - """ signal the start of running a single test item. + :arg item: Test item for which the runtest protocol is performed. - This hook will be called **before** :func:`pytest_runtest_setup`, :func:`pytest_runtest_call` and - :func:`pytest_runtest_teardown` hooks. + :arg nextitem: The scheduled-to-be-next test item (or None if this is the end my friend). - :param str nodeid: full id of the item - :param location: a triple of ``(filename, linenum, testname)`` + Stops at first non-None result, see :ref:`firstresult`. + The return value is not used, but only stops further processing. """ -def pytest_runtest_logfinish(nodeid, location): - """ signal the complete finish of running a single test item. +def pytest_runtest_logstart( + nodeid: str, location: Tuple[str, Optional[int], str] +) -> None: + """Called at the start of running the runtest protocol for a single item. - This hook will be called **after** :func:`pytest_runtest_setup`, :func:`pytest_runtest_call` and - :func:`pytest_runtest_teardown` hooks. + See :func:`pytest_runtest_protocol` for a description of the runtest protocol. - :param str nodeid: full id of the item - :param location: a triple of ``(filename, linenum, testname)`` + :param str nodeid: Full node ID of the item. + :param location: A triple of ``(filename, lineno, testname)``. + """ + + +def pytest_runtest_logfinish( + nodeid: str, location: Tuple[str, Optional[int], str] +) -> None: + """Called at the end of running the runtest protocol for a single item. + + See :func:`pytest_runtest_protocol` for a description of the runtest protocol. + + :param str nodeid: Full node ID of the item. + :param location: A triple of ``(filename, lineno, testname)``. """ def pytest_runtest_setup(item: "Item") -> None: - """ called before ``pytest_runtest_call(item)``. """ + """Called to perform the setup phase for a test item. + + The default implementation runs ``setup()`` on ``item`` and all of its + parents (which haven't been setup yet). This includes obtaining the + values of fixtures required by the item (which haven't been obtained + yet). + """ def pytest_runtest_call(item: "Item") -> None: - """ called to execute the test ``item``. """ + """Called to run the test for test item (the call phase). + + The default implementation calls ``item.runtest()``. + """ def pytest_runtest_teardown(item: "Item", nextitem: "Optional[Item]") -> None: - """ called after ``pytest_runtest_call``. + """Called to perform the teardown phase for a test item. - :arg nextitem: the scheduled-to-be-next test item (None if no further + The default implementation runs the finalizers and calls ``teardown()`` + on ``item`` and all of its parents (which need to be torn down). This + includes running the teardown phase of fixtures required by the item (if + they go out of scope). + + :arg nextitem: The scheduled-to-be-next test item (None if no further test item is scheduled). This argument can be used to perform exact teardowns, i.e. calling just enough finalizers so that nextitem only needs to call setup-functions. @@ -437,17 +482,26 @@ def pytest_runtest_teardown(item: "Item", nextitem: "Optional[Item]") -> None: @hookspec(firstresult=True) -def pytest_runtest_makereport(item: "Item", call: "CallInfo[None]") -> Optional[object]: - """ return a :py:class:`_pytest.runner.TestReport` object - for the given :py:class:`pytest.Item <_pytest.main.Item>` and - :py:class:`_pytest.runner.CallInfo`. +def pytest_runtest_makereport( + item: "Item", call: "CallInfo[None]" +) -> Optional["TestReport"]: + """Called to create a :py:class:`_pytest.reports.TestReport` for each of + the setup, call and teardown runtest phases of a test item. - Stops at first non-None result, see :ref:`firstresult` """ + See :func:`pytest_runtest_protocol` for a description of the runtest protocol. + + :param CallInfo[None] call: The ``CallInfo`` for the phase. + + Stops at first non-None result, see :ref:`firstresult`. + """ def pytest_runtest_logreport(report: "TestReport") -> None: - """ process a test setup/call/teardown report relating to - the respective phase of executing a test. """ + """Process the :py:class:`_pytest.reports.TestReport` produced for each + of the setup, call and teardown runtest phases of an item. + + See :func:`pytest_runtest_protocol` for a description of the runtest protocol. + """ @hookspec(firstresult=True) @@ -779,11 +833,17 @@ def pytest_keyboard_interrupt( def pytest_exception_interact( node: "Node", call: "CallInfo[object]", report: "Union[CollectReport, TestReport]" ) -> None: - """called when an exception was raised which can potentially be + """Called when an exception was raised which can potentially be interactively handled. - This hook is only called if an exception was raised - that is not an internal exception like ``skip.Exception``. + May be called during collection (see :py:func:`pytest_make_collect_report`), + in which case ``report`` is a :py:class:`_pytest.reports.CollectReport`. + + May be called during runtest of an item (see :py:func:`pytest_runtest_protocol`), + in which case ``report`` is a :py:class:`_pytest.reports.TestReport`. + + This hook is not called if the exception that was raised is an internal + exception like ``skip.Exception``. """ diff --git a/src/_pytest/logging.py b/src/_pytest/logging.py index 04bf74b6c..8755e5611 100644 --- a/src/_pytest/logging.py +++ b/src/_pytest/logging.py @@ -653,12 +653,12 @@ class LoggingPlugin: yield # run all the tests @pytest.hookimpl - def pytest_runtest_logstart(self): + def pytest_runtest_logstart(self) -> None: self.log_cli_handler.reset() self.log_cli_handler.set_when("start") @pytest.hookimpl - def pytest_runtest_logreport(self): + def pytest_runtest_logreport(self) -> None: self.log_cli_handler.set_when("logreport") def _runtest_for(self, item: nodes.Item, when: str) -> Generator[None, None, None]: diff --git a/src/_pytest/reports.py b/src/_pytest/reports.py index 6a408354b..8b213ed13 100644 --- a/src/_pytest/reports.py +++ b/src/_pytest/reports.py @@ -335,6 +335,8 @@ class TestReport(BaseReport): class CollectReport(BaseReport): + """Collection report object.""" + when = "collect" def __init__( @@ -346,11 +348,24 @@ class CollectReport(BaseReport): sections: Iterable[Tuple[str, str]] = (), **extra ) -> None: + #: normalized collection node id self.nodeid = nodeid + + #: test outcome, always one of "passed", "failed", "skipped". self.outcome = outcome + + #: None or a failure representation. self.longrepr = longrepr + + #: The collected items and collection nodes. self.result = result or [] + + #: list of pairs ``(str, str)`` of extra information which needs to + #: marshallable. Used by pytest to add captured text + #: from ``stdout`` and ``stderr``, but may be used by other plugins + #: to add arbitrary information to reports. self.sections = list(sections) + self.__dict__.update(extra) @property diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index f98c9b8ab..6a58260e9 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -502,7 +502,9 @@ class TerminalReporter: def pytest_deselected(self, items) -> None: self._add_stats("deselected", items) - def pytest_runtest_logstart(self, nodeid, location) -> None: + def pytest_runtest_logstart( + self, nodeid: str, location: Tuple[str, Optional[int], str] + ) -> None: # ensure that the path is printed before the # 1st test of a module starts running if self.showlongtestinfo: @@ -569,7 +571,7 @@ class TerminalReporter: assert self._session is not None return len(self._progress_nodeids_reported) == self._session.testscollected - def pytest_runtest_logfinish(self, nodeid) -> None: + def pytest_runtest_logfinish(self, nodeid: str) -> None: assert self._session if self.verbosity <= 0 and self._show_progress_info: if self._show_progress_info == "count": diff --git a/src/pytest/collect.py b/src/pytest/collect.py index 73c9d35a0..ec9c2d8b4 100644 --- a/src/pytest/collect.py +++ b/src/pytest/collect.py @@ -1,6 +1,8 @@ import sys import warnings from types import ModuleType +from typing import Any +from typing import List import pytest from _pytest.deprecated import PYTEST_COLLECT_MODULE @@ -20,15 +22,15 @@ COLLECT_FAKEMODULE_ATTRIBUTES = [ class FakeCollectModule(ModuleType): - def __init__(self): + def __init__(self) -> None: super().__init__("pytest.collect") self.__all__ = list(COLLECT_FAKEMODULE_ATTRIBUTES) self.__pytest = pytest - def __dir__(self): + def __dir__(self) -> List[str]: return dir(super()) + self.__all__ - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: if name not in self.__all__: raise AttributeError(name) warnings.warn(PYTEST_COLLECT_MODULE.format(name=name), stacklevel=2)