Merge master into features (#6312)

Merge master into features

.gitignore

@@ -25,6 +25,7 @@ src/_pytest/_version.py
 
 doc/*/_build
 doc/*/.doctrees
+doc/*/_changelog_towncrier_draft.rst
 build/
 dist/
 *.egg-info

.pre-commit-config.yaml

@@ -1,4 +1,3 @@
-exclude: doc/en/example/py2py3/test_py2.py
 repos:
 -   repo: https://github.com/psf/black
     rev: 19.10b0
@@ -48,7 +47,7 @@ repos:
     -   id: rst
         name: rst
         entry: rst-lint --encoding utf-8
-        files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|TIDELIFT.rst|changelog/.*)$
+        files: ^(HOWTORELEASE.rst|README.rst|TIDELIFT.rst)$
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
     -   id: changelogs-rst

changelog/4639.feature.rst

@@ -0,0 +1,3 @@
+Revert "A warning is now issued when assertions are made for ``None``".
+The warning proved to be less useful than initially expected and had quite a
+few false positive cases.

changelog/5914.bugfix.rst

@@ -1 +0,0 @@
-pytester: fix ``no_fnmatch_line`` when used after positive matching.

changelog/6231.improvement.rst

@@ -1 +0,0 @@
-Improve check for misspelling of ``pytest.mark.parametrize``.

changelog/6255.bugfix.rst

@@ -1,3 +0,0 @@
-Clear the ``sys.last_traceback``, ``sys.last_type`` and ``sys.last_value``
-attributes by deleting them instead of setting them to ``None``. This better
-matches the behaviour of the Python standard library.

changelog/6290.bugfix.rst

@@ -0,0 +1 @@
+The supporting files in the ``.pytest_cache`` directory are kept with ``--cache-clear``, which only clears cached values now.

changelog/README.rst

@@ -31,6 +31,7 @@ changelog using that instead.
 If you are not sure what issue type to use, don't hesitate to ask in your PR.
 
 ``towncrier`` preserves multiple paragraphs and formatting (code blocks, lists, and so on), but for entries
-other than ``features`` it is usually better to stick to a single paragraph to keep it concise. You can install
+other than ``features`` it is usually better to stick to a single paragraph to keep it concise.
-``towncrier`` and then run ``towncrier --draft``
+
-if you want to get a preview of how your change will look in the final release notes.
+You can also run ``tox -e docs`` to build the documentation
+with the draft changelog (``doc/en/_build/changelog.html``) if you want to get a preview of how your change will look in the final release notes.

doc/en/announce/index.rst

@@ -6,6 +6,7 @@ Release announcements
    :maxdepth: 2
 
 
+   release-5.3.1
    release-5.3.0
    release-5.2.4
    release-5.2.3

doc/en/announce/release-5.3.1.rst

@@ -0,0 +1,26 @@
+pytest-5.3.1
+=======================================
+
+pytest 5.3.1 has just been released to PyPI.
+
+This is a bug-fix release, being a drop-in replacement. To upgrade::
+
+  pip install --upgrade pytest
+
+The full changelog is available at https://docs.pytest.org/en/latest/changelog.html.
+
+Thanks to all who contributed to this release, among them:
+
+* Anthony Sottile
+* Bruno Oliveira
+* Daniel Hahler
+* Felix Yan
+* Florian Bruhin
+* Mark Dickinson
+* Nikolay Kondratyev
+* Steffen Schroeder
+* Zac Hatfield-Dodds
+
+
+Happy testing,
+The pytest Development Team

doc/en/conf.py

@@ -20,6 +20,10 @@ import sys
 
 from _pytest import __version__ as version
 
+if False:  # TYPE_CHECKING
+    import sphinx.application
+
+
 release = ".".join(version.split(".")[:2])
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -342,7 +346,30 @@ texinfo_documents = [
 intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
 
 
-def setup(app):
+def configure_logging(app: "sphinx.application.Sphinx") -> None:
+    """Configure Sphinx's WarningHandler to handle (expected) missing include."""
+    import sphinx.util.logging
+    import logging
+
+    class WarnLogFilter(logging.Filter):
+        def filter(self, record: logging.LogRecord) -> bool:
+            """Ignore warnings about missing include with "only" directive.
+
+            Ref: https://github.com/sphinx-doc/sphinx/issues/2150."""
+            if (
+                record.msg.startswith('Problems with "include" directive path:')
+                and "_changelog_towncrier_draft.rst" in record.msg
+            ):
+                return False
+            return True
+
+    logger = logging.getLogger(sphinx.util.logging.NAMESPACE)
+    warn_handler = [x for x in logger.handlers if x.level == logging.WARNING]
+    assert len(warn_handler) == 1, warn_handler
+    warn_handler[0].filters.insert(0, WarnLogFilter())
+
+
+def setup(app: "sphinx.application.Sphinx") -> None:
     # from sphinx.ext.autodoc import cut_lines
     # app.connect('autodoc-process-docstring', cut_lines(4, what=['module']))
     app.add_object_type(
@@ -351,3 +378,4 @@ def setup(app):
         objname="configuration value",
         indextemplate="pair: %s; configuration value",
     )
+    configure_logging(app)

doc/en/example/parametrize.rst

@@ -475,10 +475,10 @@ Running it results in some skips if we don't have all the python interpreters in
 .. code-block:: pytest
 
    . $ pytest -rs -q multipython.py
-   ssssssssssss...ssssssssssss                                          [100%]
+   ssssssssssssssssssssssss...                                          [100%]
    ========================= short test summary info ==========================
    SKIPPED [12] $REGENDOC_TMPDIR/CWD/multipython.py:29: 'python3.5' not found
-   SKIPPED [12] $REGENDOC_TMPDIR/CWD/multipython.py:29: 'python3.7' not found
+   SKIPPED [12] $REGENDOC_TMPDIR/CWD/multipython.py:29: 'python3.6' not found
    3 passed, 24 skipped in 0.12s
 
 Indirect parametrization of optional implementations/imports

doc/en/example/py2py3/conftest.py

@@ -1,16 +0,0 @@
-import sys
-
-import pytest
-
-py3 = sys.version_info[0] >= 3
-
-
-class DummyCollector(pytest.collect.File):
-    def collect(self):
-        return []
-
-
-def pytest_pycollect_makemodule(path, parent):
-    bn = path.basename
-    if "py3" in bn and not py3 or ("py2" in bn and py3):
-        return DummyCollector.from_parent(parent, fspath=path)

doc/en/example/py2py3/test_py2.py

@@ -1,5 +0,0 @@
-def test_exception_syntax():
-    try:
-        0 / 0
-    except ZeroDivisionError, e:
-        assert e

doc/en/example/py2py3/test_py3.py

@@ -1,5 +0,0 @@
-def test_exception_syntax():
-    try:
-        0 / 0
-    except ZeroDivisionError as e:
-        assert e

doc/en/example/reportingdemo.rst

@@ -436,7 +436,7 @@ Here is a nice run of several failures and how ``pytest`` presents things:
             items = [1, 2, 3]
             print("items is {!r}".format(items))
     >       a, b = items.pop()
-    E       TypeError: 'int' object is not iterable
+    E       TypeError: cannot unpack non-iterable int object
 
     failure_demo.py:181: TypeError
     --------------------------- Captured stdout call ---------------------------
@@ -516,7 +516,7 @@ Here is a nice run of several failures and how ``pytest`` presents things:
         def test_z2_type_error(self):
             items = 3
     >       a, b = items
-    E       TypeError: 'int' object is not iterable
+    E       TypeError: cannot unpack non-iterable int object
 
     failure_demo.py:222: TypeError
     ______________________ TestMoreErrors.test_startswith ______________________

doc/en/example/simple.rst

@@ -442,7 +442,7 @@ Now we can profile which test functions execute the slowest:
 
     ========================= slowest 3 test durations =========================
     0.30s call     test_some_are_slow.py::test_funcslow2
-    0.20s call     test_some_are_slow.py::test_funcslow1
+    0.21s call     test_some_are_slow.py::test_funcslow1
     0.11s call     test_some_are_slow.py::test_funcfast
     ============================ 3 passed in 0.12s =============================
 

doc/en/getting-started.rst

@@ -28,7 +28,7 @@ Install ``pytest``
 .. code-block:: bash
 
     $ pytest --version
-    This is pytest version 5.x.y, imported from $PYTHON_PREFIX/lib/python3.6/site-packages/pytest.py
+    This is pytest version 5.x.y, imported from $PYTHON_PREFIX/lib/python3.7/site-packages/pytest.py
 
 .. _`simpletest`:
 

doc/en/reference.rst

@@ -360,7 +360,7 @@ capfd
 
         def test_system_echo(capfd):
             os.system('echo "hello"')
-            captured = capsys.readouterr()
+            captured = capfd.readouterr()
             assert captured.out == "hello\n"
 
 
@@ -463,6 +463,8 @@ monkeypatch
 .. autoclass:: _pytest.monkeypatch.MonkeyPatch
     :members:
 
+.. _testdir:
+
 testdir
 ~~~~~~~
 
@@ -1215,6 +1217,15 @@ passed multiple times. The expected format is ``name=value``. For example::
     a specific entry in the log. ``extra`` kwarg overrides the value specified
     on the command line or in the config.
 
+.. confval:: log_cli
+
+    Enable log display during test run (also known as :ref:`"live logging" <live_logs>`).
+    The default is ``False``.
+
+    .. code-block:: ini
+
+        [pytest]
+        log_cli = True
 
 .. confval:: log_cli_date_format
 

pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 [tool.towncrier]
 package = "pytest"
 package_dir = "src"
-filename = "CHANGELOG.rst"
+filename = "doc/en/changelog.rst"
 directory = "changelog/"
 title_format = "pytest {version} ({project_date})"
 template = "changelog/_template.rst"

scripts/release.py

@@ -85,12 +85,13 @@ def check_links():
     check_call(["tox", "-e", "docs-checklinks"])
 
 
-def pre_release(version):
+def pre_release(version, *, skip_check_links):
     """Generates new docs, release announcements and creates a local tag."""
     announce(version)
     regen()
     changelog(version, write_out=True)
     fix_formatting()
+    if not skip_check_links:
         check_links()
 
     msg = "Preparing release version {}".format(version)
@@ -114,8 +115,9 @@ def main():
     init(autoreset=True)
     parser = argparse.ArgumentParser()
     parser.add_argument("version", help="Release version")
+    parser.add_argument("--skip-check-links", action="store_true", default=False)
     options = parser.parse_args()
-    pre_release(options.version)
+    pre_release(options.version, skip_check_links=options.skip_check_links)
 
 
 if __name__ == "__main__":

setup.cfg

@@ -38,8 +38,8 @@ packages =
     _pytest.assertion
     _pytest.config
     _pytest.mark
+    pytest
 
-py_modules = pytest
 python_requires = >=3.5
 
 [options.entry_points]

src/_pytest/assertion/rewrite.py

@@ -799,13 +799,6 @@ class AssertionRewriter(ast.NodeVisitor):
         self.push_format_context()
         # Rewrite assert into a bunch of statements.
         top_condition, explanation = self.visit(assert_.test)
-        # If in a test module, check if directly asserting None, in order to warn [Issue #3191]
-        if self.module_path is not None:
-            self.statements.append(
-                self.warn_about_none_ast(
-                    top_condition, module_path=self.module_path, lineno=assert_.lineno
-                )
-            )
 
         negation = ast.UnaryOp(ast.Not(), top_condition)
 
@@ -887,30 +880,6 @@ class AssertionRewriter(ast.NodeVisitor):
             set_location(stmt, assert_.lineno, assert_.col_offset)
         return self.statements
 
-    def warn_about_none_ast(self, node, module_path, lineno):
-        """
-        Returns an AST issuing a warning if the value of node is `None`.
-        This is used to warn the user when asserting a function that asserts
-        internally already.
-        See issue #3191 for more details.
-        """
-        val_is_none = ast.Compare(node, [ast.Is()], [ast.NameConstant(None)])
-        send_warning = ast.parse(
-            """\
-from _pytest.warning_types import PytestAssertRewriteWarning
-from warnings import warn_explicit
-warn_explicit(
-    PytestAssertRewriteWarning('asserting the value None, please use "assert is None"'),
-    category=None,
-    filename={filename!r},
-    lineno={lineno},
-)
-            """.format(
-                filename=fspath(module_path), lineno=lineno
-            )
-        ).body
-        return ast.If(val_is_none, send_warning, [])
-
     def visit_Name(self, name):
         # Display the repr of the name if it's a local variable or
         # _should_repr_global_name() thinks it's acceptable.

src/_pytest/cacheprovider.py

@@ -44,14 +44,27 @@ class Cache:
     _cachedir = attr.ib(repr=False)
     _config = attr.ib(repr=False)
 
+    # sub-directory under cache-dir for directories created by "makedir"
+    _CACHE_PREFIX_DIRS = "d"
+
+    # sub-directory under cache-dir for values created by "set"
+    _CACHE_PREFIX_VALUES = "v"
+
     @classmethod
     def for_config(cls, config):
         cachedir = cls.cache_dir_from_config(config)
-        if config.getoption("cacheclear") and cachedir.exists():
+        if config.getoption("cacheclear") and cachedir.is_dir():
-            rm_rf(cachedir)
+            cls.clear_cache(cachedir)
-            cachedir.mkdir()
         return cls(cachedir, config)
 
+    @classmethod
+    def clear_cache(cls, cachedir: Path):
+        """Clears the sub-directories used to hold cached directories and values."""
+        for prefix in (cls._CACHE_PREFIX_DIRS, cls._CACHE_PREFIX_VALUES):
+            d = cachedir / prefix
+            if d.is_dir():
+                rm_rf(d)
+
     @staticmethod
     def cache_dir_from_config(config):
         return resolve_from_str(config.getini("cache_dir"), config.rootdir)
@@ -79,12 +92,12 @@ class Cache:
         name = Path(name)
         if len(name.parts) > 1:
             raise ValueError("name is not allowed to contain path separators")
-        res = self._cachedir.joinpath("d", name)
+        res = self._cachedir.joinpath(self._CACHE_PREFIX_DIRS, name)
         res.mkdir(exist_ok=True, parents=True)
         return py.path.local(res)
 
     def _getvaluepath(self, key):
-        return self._cachedir.joinpath("v", Path(key))
+        return self._cachedir.joinpath(self._CACHE_PREFIX_VALUES, Path(key))
 
     def get(self, key, default):
         """ return cached value for the given key.  If no value
@@ -417,7 +430,7 @@ def cacheshow(config, session):
 
     dummy = object()
     basedir = config.cache._cachedir
-    vdir = basedir / "v"
+    vdir = basedir / Cache._CACHE_PREFIX_VALUES
     tw.sep("-", "cache values for %r" % glob)
     for valpath in sorted(x for x in vdir.rglob(glob) if x.is_file()):
         key = valpath.relative_to(vdir)
@@ -429,7 +442,7 @@ def cacheshow(config, session):
             for line in pformat(val).splitlines():
                 tw.line("  " + line)
 
-    ddir = basedir / "d"
+    ddir = basedir / Cache._CACHE_PREFIX_DIRS
     if ddir.is_dir():
         contents = sorted(ddir.rglob(glob))
         tw.sep("-", "cache directories for %r" % glob)

src/_pytest/config/__init__.py

@@ -780,7 +780,7 @@ class Config:
 
     @classmethod
     def fromdictargs(cls, option_dict, args):
-        """ constructor useable for subprocesses. """
+        """ constructor usable for subprocesses. """
         config = get_config(args)
         config.option.__dict__.update(option_dict)
         config.parse(args, addopts=False)

src/_pytest/doctest.py

@@ -441,6 +441,16 @@ class DoctestModule(pytest.Module):
             https://bugs.python.org/issue25532
             """
 
+            def _find_lineno(self, obj, source_lines):
+                """
+                Doctest code does not take into account `@property`, this is a hackish way to fix it.
+
+                https://bugs.python.org/issue17446
+                """
+                if isinstance(obj, property):
+                    obj = getattr(obj, "fget", obj)
+                return doctest.DocTestFinder._find_lineno(self, obj, source_lines)
+
             def _find(self, tests, obj, name, module, source_lines, globs, seen):
                 if _is_mocked(obj):
                     return

src/_pytest/main.py

@@ -213,9 +213,15 @@ def wrap_session(config, doit):
             config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
             session.exitstatus = exitstatus
         except:  # noqa
-            excinfo = _pytest._code.ExceptionInfo.from_current()
-            config.notify_exception(excinfo, config.option)
             session.exitstatus = ExitCode.INTERNAL_ERROR
+            excinfo = _pytest._code.ExceptionInfo.from_current()
+            try:
+                config.notify_exception(excinfo, config.option)
+            except exit.Exception as exc:
+                if exc.returncode is not None:
+                    session.exitstatus = exc.returncode
+                sys.stderr.write("{}: {}\n".format(type(exc).__name__, exc))
+            else:
                 if excinfo.errisinstance(SystemExit):
                     sys.stderr.write("mainloop: caught unexpected SystemExit!\n")
 

src/_pytest/terminal.py

@@ -1099,7 +1099,7 @@ def _get_main_color(stats) -> Tuple[str, List[str]]:
         "failed passed skipped deselected xfailed xpassed warnings error".split()
     )
     unknown_type_seen = False
-    for found_type in stats:
+    for found_type in stats.keys():
         if found_type not in known_types:
             if found_type:  # setup/teardown reports have an empty key, ignore them
                 known_types.append(found_type)

src/pytest/__init__.py

@@ -4,6 +4,7 @@ pytest: unit and functional testing with Python.
 """
 from _pytest import __version__
 from _pytest.assertion import register_assert_rewrite
+from _pytest.compat import _setup_collect_fakemodule
 from _pytest.config import cmdline
 from _pytest.config import hookimpl
 from _pytest.config import hookspec
@@ -93,14 +94,6 @@ __all__ = [
     "yield_fixture",
 ]
 
-if __name__ == "__main__":
-    # if run as a script or by 'python -m pytest'
-    # we trigger the below "else" condition by the following import
-    import pytest
 
-    raise SystemExit(pytest.main())
+_setup_collect_fakemodule()
-else:
+del _setup_collect_fakemodule
-
-    from _pytest.compat import _setup_collect_fakemodule
-
-    _setup_collect_fakemodule()

src/pytest/__main__.py

@@ -0,0 +1,7 @@
+"""
+pytest entry point
+"""
+import pytest
+
+if __name__ == "__main__":
+    raise SystemExit(pytest.main())

testing/test_cacheprovider.py

@@ -270,6 +270,7 @@ class TestLastFailed:
         )
         result = testdir.runpytest(str(p), "--lf", "--cache-clear")
         result.stdout.fnmatch_lines(["*1 failed*2 passed*"])
+        assert testdir.tmpdir.join(".pytest_cache", "README.md").isfile()
 
         # Run this again to make sure clear-cache is robust
         if os.path.isdir(".pytest_cache"):

testing/test_doctest.py

@@ -287,13 +287,68 @@ class TestDoctests:
             )
         )
         result = testdir.runpytest("--doctest-modules")
+        result.stdout.fnmatch_lines(
+            ["*hello*", "006*>>> 1/0*", "*UNEXPECTED*ZeroDivision*", "*1 failed*"]
+        )
+
+    def test_doctest_linedata_on_property(self, testdir):
+        testdir.makepyfile(
+            """
+            class Sample(object):
+                @property
+                def some_property(self):
+                    '''
+                    >>> Sample().some_property
+                    'another thing'
+                    '''
+                    return 'something'
+            """
+        )
+        result = testdir.runpytest("--doctest-modules")
         result.stdout.fnmatch_lines(
             [
-                "*hello*",
+                "*= FAILURES =*",
-                "*EXAMPLE LOCATION UNKNOWN, not showing all tests of that example*",
+                "*_ [[]doctest[]] test_doctest_linedata_on_property.Sample.some_property _*",
-                "*1/0*",
+                "004 ",
-                "*UNEXPECTED*ZeroDivision*",
+                "005         >>> Sample().some_property",
-                "*1 failed*",
+                "Expected:",
+                "    'another thing'",
+                "Got:",
+                "    'something'",
+                "",
+                "*/test_doctest_linedata_on_property.py:5: DocTestFailure",
+                "*= 1 failed in *",
+            ]
+        )
+
+    def test_doctest_no_linedata_on_overriden_property(self, testdir):
+        testdir.makepyfile(
+            """
+            class Sample(object):
+                @property
+                def some_property(self):
+                    '''
+                    >>> Sample().some_property
+                    'another thing'
+                    '''
+                    return 'something'
+                some_property = property(some_property.__get__, None, None, some_property.__doc__)
+            """
+        )
+        result = testdir.runpytest("--doctest-modules")
+        result.stdout.fnmatch_lines(
+            [
+                "*= FAILURES =*",
+                "*_ [[]doctest[]] test_doctest_no_linedata_on_overriden_property.Sample.some_property _*",
+                "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example",
+                "[?][?][?] >>> Sample().some_property",
+                "Expected:",
+                "    'another thing'",
+                "Got:",
+                "    'something'",
+                "",
+                "*/test_doctest_no_linedata_on_overriden_property.py:None: DocTestFailure",
+                "*= 1 failed in *",
             ]
         )
 

testing/test_main.py

@@ -0,0 +1,52 @@
+import pytest
+from _pytest.main import ExitCode
+
+
+@pytest.mark.parametrize(
+    "ret_exc",
+    (
+        pytest.param((None, ValueError)),
+        pytest.param((42, SystemExit)),
+        pytest.param((False, SystemExit)),
+    ),
+)
+def test_wrap_session_notify_exception(ret_exc, testdir):
+    returncode, exc = ret_exc
+    c1 = testdir.makeconftest(
+        """
+        import pytest
+
+        def pytest_sessionstart():
+            raise {exc}("boom")
+
+        def pytest_internalerror(excrepr, excinfo):
+            returncode = {returncode!r}
+            if returncode is not False:
+                pytest.exit("exiting after %s..." % excinfo.typename, returncode={returncode!r})
+    """.format(
+            returncode=returncode, exc=exc.__name__
+        )
+    )
+    result = testdir.runpytest()
+    if returncode:
+        assert result.ret == returncode
+    else:
+        assert result.ret == ExitCode.INTERNAL_ERROR
+    assert result.stdout.lines[0] == "INTERNALERROR> Traceback (most recent call last):"
+
+    if exc == SystemExit:
+        assert result.stdout.lines[-3:] == [
+            'INTERNALERROR>   File "{}", line 4, in pytest_sessionstart'.format(c1),
+            'INTERNALERROR>     raise SystemExit("boom")',
+            "INTERNALERROR> SystemExit: boom",
+        ]
+    else:
+        assert result.stdout.lines[-3:] == [
+            'INTERNALERROR>   File "{}", line 4, in pytest_sessionstart'.format(c1),
+            'INTERNALERROR>     raise ValueError("boom")',
+            "INTERNALERROR> ValueError: boom",
+        ]
+    if returncode is False:
+        assert result.stderr.lines == ["mainloop: caught unexpected SystemExit!"]
+    else:
+        assert result.stderr.lines == ["Exit: exiting after {}...".format(exc.__name__)]

testing/test_warnings.py

@@ -544,7 +544,7 @@ class TestAssertionWarnings:
 
     def test_tuple_warning(self, testdir):
         testdir.makepyfile(
-            """
+            """\
             def test_foo():
                 assert (1,2)
             """
@@ -554,48 +554,6 @@ class TestAssertionWarnings:
             result, "assertion is always true, perhaps remove parentheses?"
         )
 
-    @staticmethod
-    def create_file(testdir, return_none):
-        testdir.makepyfile(
-            """
-            def foo(return_none):
-                if return_none:
-                    return None
-                else:
-                    return False
-
-            def test_foo():
-                assert foo({return_none})
-            """.format(
-                return_none=return_none
-            )
-        )
-
-    def test_none_function_warns(self, testdir):
-        self.create_file(testdir, True)
-        result = testdir.runpytest()
-        self.assert_result_warns(
-            result, 'asserting the value None, please use "assert is None"'
-        )
-
-    def test_assert_is_none_no_warn(self, testdir):
-        testdir.makepyfile(
-            """
-            def foo():
-                return None
-
-            def test_foo():
-                assert foo() is None
-            """
-        )
-        result = testdir.runpytest()
-        result.stdout.fnmatch_lines(["*1 passed in*"])
-
-    def test_false_function_no_warn(self, testdir):
-        self.create_file(testdir, False)
-        result = testdir.runpytest()
-        result.stdout.fnmatch_lines(["*1 failed in*"])
-
 
 def test_warnings_checker_twice():
     """Issue #4617"""

tox.ini

@@ -58,11 +58,16 @@ commands = pre-commit run --all-files --show-diff-on-failure
 [testenv:docs]
 basepython = python3
 usedevelop = True
-changedir = doc/en
+deps =
-deps = -r{toxinidir}/doc/en/requirements.txt
+    -r{toxinidir}/doc/en/requirements.txt
-
+    towncrier
+whitelist_externals = sh
 commands =
-    sphinx-build -W -b html . _build
+    sh -c 'towncrier --draft > doc/en/_changelog_towncrier_draft.rst'
+    # the '-t changelog_towncrier_draft' tags makes sphinx include the draft
+    # changelog in the docs; this does not happen on ReadTheDocs because it uses
+    # the standard sphinx command so the 'changelog_towncrier_draft' is never set there
+    sphinx-build -W --keep-going -b html doc/en doc/en/_build -t changelog_towncrier_draft {posargs:}
 
 [testenv:docs-checklinks]
 basepython = python3
@@ -87,10 +92,10 @@ changedir = doc/en
 skipsdist = True
 basepython = python3
 deps =
-    sphinx
+    dataclasses
     PyYAML
     regendoc>=0.6.1
-    dataclasses
+    sphinx
 whitelist_externals =
     rm
     make
@@ -118,8 +123,8 @@ deps =
     colorama
     gitpython
     pre-commit>=1.11.0
-    towncrier
     wheel
+    towncrier
 commands = python scripts/release.py {posargs}
 
 [testenv:publish_gh_release_notes]