From 0c18e244334d1c04fa2a7942fe5e2eb179ba6915 Mon Sep 17 00:00:00 2001
From: Bruno Oliveira <nicoddemus@gmail.com>
Date: Sat, 5 Oct 2019 12:17:20 -0300
Subject: [PATCH 1/2] Introduce no_fnmatch_line/no_re_match_line in pytester

The current idiom is to use:

  assert re.match(pat, result.stdout.str())

Or

  assert line in result.stdout.str()

But this does not really give good results when it fails.

Those new functions produce similar output to ther other match lines functions.
---
 changelog/5914.feature.rst | 19 ++++++++++++++++
 src/_pytest/pytester.py    | 43 +++++++++++++++++++++++++++++++----
 testing/test_pytester.py   | 46 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 104 insertions(+), 4 deletions(-)
 create mode 100644 changelog/5914.feature.rst

diff --git a/changelog/5914.feature.rst b/changelog/5914.feature.rst
new file mode 100644
index 000000000..68cd66f99
--- /dev/null
+++ b/changelog/5914.feature.rst
@@ -0,0 +1,19 @@
+``pytester`` learned two new functions, `no_fnmatch_line <https://docs.pytest.org/en/latest/reference.html#_pytest.pytester.LineMatcher.no_fnmatch_line>`_ and
+`no_re_match_line <https://docs.pytest.org/en/latest/reference.html#_pytest.pytester.LineMatcher.no_re_match_line>`_.
+
+The functions are used to ensure the captured text *does not* match the given
+pattern.
+
+The previous idiom was to use ``re.match``:
+
+.. code-block:: python
+
+    assert re.match(pat, result.stdout.str()) is None
+
+Or the ``in`` operator:
+
+.. code-block:: python
+
+    assert text in result.stdout.str()
+
+But the new functions produce best output on failure.
diff --git a/src/_pytest/pytester.py b/src/_pytest/pytester.py
index 0f3460741..a050dad09 100644
--- a/src/_pytest/pytester.py
+++ b/src/_pytest/pytester.py
@@ -1318,8 +1318,7 @@ class LineMatcher:
 
         The argument is a list of lines which have to match and can use glob
         wildcards.  If they do not match a pytest.fail() is called.  The
-        matches and non-matches are also printed on stdout.
-
+        matches and non-matches are also shown as part of the error message.
         """
         __tracebackhide__ = True
         self._match_lines(lines2, fnmatch, "fnmatch")
@@ -1330,8 +1329,7 @@ class LineMatcher:
         The argument is a list of lines which have to match using ``re.match``.
         If they do not match a pytest.fail() is called.
 
-        The matches and non-matches are also printed on stdout.
-
+        The matches and non-matches are also shown as part of the error message.
         """
         __tracebackhide__ = True
         self._match_lines(lines2, lambda name, pat: re.match(pat, name), "re.match")
@@ -1374,3 +1372,40 @@ class LineMatcher:
             else:
                 self._log("remains unmatched: {!r}".format(line))
                 pytest.fail(self._log_text)
+
+    def no_fnmatch_line(self, pat):
+        """Ensure captured lines do not match the given pattern, using ``fnmatch.fnmatch``.
+
+        :param str pat: the pattern to match lines.
+        """
+        __tracebackhide__ = True
+        self._no_match_line(pat, fnmatch, "fnmatch")
+
+    def no_re_match_line(self, pat):
+        """Ensure captured lines do not match the given pattern, using ``re.match``.
+
+        :param str pat: the regular expression to match lines.
+        """
+        __tracebackhide__ = True
+        self._no_match_line(pat, lambda name, pat: re.match(pat, name), "re.match")
+
+    def _no_match_line(self, pat, match_func, match_nickname):
+        """Ensure captured lines does not have a the given pattern, using ``fnmatch.fnmatch``
+
+        :param str pat: the pattern to match lines
+        """
+        __tracebackhide__ = True
+        nomatch_printed = False
+        try:
+            for line in self.lines:
+                if match_func(line, pat):
+                    self._log("%s:" % match_nickname, repr(pat))
+                    self._log("   with:", repr(line))
+                    pytest.fail(self._log_text)
+                else:
+                    if not nomatch_printed:
+                        self._log("nomatch:", repr(pat))
+                        nomatch_printed = True
+                    self._log("    and:", repr(line))
+        finally:
+            self._log_output = []
diff --git a/testing/test_pytester.py b/testing/test_pytester.py
index d330ff253..f8b0896c5 100644
--- a/testing/test_pytester.py
+++ b/testing/test_pytester.py
@@ -457,6 +457,52 @@ def test_linematcher_with_nonlist():
     assert lm._getlines(set()) == set()
 
 
+@pytest.mark.parametrize("function", ["no_fnmatch_line", "no_re_match_line"])
+def test_no_matching(function):
+    """"""
+    if function == "no_fnmatch_line":
+        match_func_name = "fnmatch"
+        good_pattern = "*.py OK*"
+        bad_pattern = "*X.py OK*"
+    else:
+        assert function == "no_re_match_line"
+        match_func_name = "re.match"
+        good_pattern = r".*py OK"
+        bad_pattern = r".*Xpy OK"
+
+    lm = LineMatcher(
+        [
+            "cachedir: .pytest_cache",
+            "collecting ... collected 1 item",
+            "",
+            "show_fixtures_per_test.py OK",
+            "=== elapsed 1s ===",
+        ]
+    )
+
+    def check_failure_lines(lines):
+        expected = [
+            "nomatch: '{}'".format(good_pattern),
+            "    and: 'cachedir: .pytest_cache'",
+            "    and: 'collecting ... collected 1 item'",
+            "    and: ''",
+            "{}: '{}'".format(match_func_name, good_pattern),
+            "   with: 'show_fixtures_per_test.py OK'",
+        ]
+        assert lines == expected
+
+    # check the function twice to ensure we don't accumulate the internal buffer
+    for i in range(2):
+        with pytest.raises(pytest.fail.Exception) as e:
+            func = getattr(lm, function)
+            func(good_pattern)
+        obtained = str(e.value).splitlines()
+        check_failure_lines(obtained)
+
+    func = getattr(lm, function)
+    func(bad_pattern)  # bad pattern does not match any line: passes
+
+
 def test_pytester_addopts(request, monkeypatch):
     monkeypatch.setenv("PYTEST_ADDOPTS", "--orig-unused")
 

From 47c2091ecd2f341e10f38f1d505c21fb3323c140 Mon Sep 17 00:00:00 2001
From: Bruno Oliveira <nicoddemus@gmail.com>
Date: Sat, 5 Oct 2019 14:18:51 -0300
Subject: [PATCH 2/2] Use new no-match functions to replace previous idiom

---
 testing/acceptance_test.py               |  6 +++---
 testing/code/test_excinfo.py             |  5 +++--
 testing/logging/test_fixture.py          |  2 +-
 testing/logging/test_reporting.py        | 16 ++++++++--------
 testing/python/collect.py                |  2 +-
 testing/python/fixtures.py               |  6 +++---
 testing/python/setup_only.py             |  2 +-
 testing/python/show_fixtures_per_test.py |  4 ++--
 testing/test_assertion.py                |  4 ++--
 testing/test_assertrewrite.py            |  6 +++---
 testing/test_cacheprovider.py            |  6 +++---
 testing/test_capture.py                  | 14 +++++++-------
 testing/test_collection.py               | 12 ++++++------
 testing/test_conftest.py                 |  4 ++--
 testing/test_doctest.py                  |  8 ++++----
 testing/test_junitxml.py                 |  6 +++---
 testing/test_runner.py                   | 12 ++++++------
 testing/test_runner_xunit.py             |  2 +-
 testing/test_skipping.py                 |  2 +-
 testing/test_terminal.py                 | 24 ++++++++++++------------
 testing/test_unittest.py                 | 10 +++++-----
 21 files changed, 77 insertions(+), 76 deletions(-)

diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py
index ad9c37737..c82699651 100644
--- a/testing/acceptance_test.py
+++ b/testing/acceptance_test.py
@@ -246,7 +246,7 @@ class TestGeneralUsage:
         )
         result = testdir.runpytest()
         assert result.ret == ExitCode.NO_TESTS_COLLECTED
-        assert "should not be seen" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*should not be seen*")
         assert "stderr42" not in result.stderr.str()
 
     def test_conftest_printing_shows_if_error(self, testdir):
@@ -954,7 +954,7 @@ class TestDurations:
         result.stdout.fnmatch_lines(["*Interrupted: 1 errors during collection*"])
         # Collection errors abort test execution, therefore no duration is
         # output
-        assert "duration" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*duration*")
 
     def test_with_not(self, testdir):
         testdir.makepyfile(self.source)
@@ -1008,7 +1008,7 @@ def test_zipimport_hook(testdir, tmpdir):
     result = testdir.runpython(target)
     assert result.ret == 0
     result.stderr.fnmatch_lines(["*not found*foo*"])
-    assert "INTERNALERROR>" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR>*")
 
 
 def test_import_plugin_unicode_name(testdir):
diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py
index 5673b811b..e2f06a0a2 100644
--- a/testing/code/test_excinfo.py
+++ b/testing/code/test_excinfo.py
@@ -399,7 +399,7 @@ def test_match_raises_error(testdir):
     result = testdir.runpytest()
     assert result.ret != 0
     result.stdout.fnmatch_lines(["*AssertionError*Pattern*[123]*not found*"])
-    assert "__tracebackhide__ = True" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*__tracebackhide__ = True*")
 
     result = testdir.runpytest("--fulltrace")
     assert result.ret != 0
@@ -1343,7 +1343,8 @@ def test_cwd_deleted(testdir):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["* 1 failed in *"])
-    assert "INTERNALERROR" not in result.stdout.str() + result.stderr.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR*")
+    result.stderr.no_fnmatch_line("*INTERNALERROR*")
 
 
 @pytest.mark.usefixtures("limited_recursion_depth")
diff --git a/testing/logging/test_fixture.py b/testing/logging/test_fixture.py
index 5d2ff4654..c68866bef 100644
--- a/testing/logging/test_fixture.py
+++ b/testing/logging/test_fixture.py
@@ -46,7 +46,7 @@ def test_change_level_undo(testdir):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["*log from test1*", "*2 failed in *"])
-    assert "log from test2" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*log from test2*")
 
 
 def test_with_statement(caplog):
diff --git a/testing/logging/test_reporting.py b/testing/logging/test_reporting.py
index 1ae0bd783..5b24ef963 100644
--- a/testing/logging/test_reporting.py
+++ b/testing/logging/test_reporting.py
@@ -109,7 +109,7 @@ def test_log_cli_level_log_level_interaction(testdir):
             "=* 1 failed in *=",
         ]
     )
-    assert "DEBUG" not in result.stdout.str()
+    result.stdout.no_re_match_line("DEBUG")
 
 
 def test_setup_logging(testdir):
@@ -282,7 +282,7 @@ def test_log_cli_default_level(testdir):
             "WARNING*test_log_cli_default_level.py* message will be shown*",
         ]
     )
-    assert "INFO message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INFO message won't be shown*")
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
 
@@ -566,7 +566,7 @@ def test_log_cli_level(testdir):
             "PASSED",  # 'PASSED' on its own line because the log message prints a new line
         ]
     )
-    assert "This log message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*This log message won't be shown*")
 
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
@@ -580,7 +580,7 @@ def test_log_cli_level(testdir):
             "PASSED",  # 'PASSED' on its own line because the log message prints a new line
         ]
     )
-    assert "This log message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*This log message won't be shown*")
 
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
@@ -616,7 +616,7 @@ def test_log_cli_ini_level(testdir):
             "PASSED",  # 'PASSED' on its own line because the log message prints a new line
         ]
     )
-    assert "This log message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*This log message won't be shown*")
 
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
@@ -942,7 +942,7 @@ def test_collection_collect_only_live_logging(testdir, verbose):
             ]
         )
     elif verbose == "-q":
-        assert "collected 1 item*" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*collected 1 item**")
         expected_lines.extend(
             [
                 "*test_collection_collect_only_live_logging.py::test_simple*",
@@ -950,7 +950,7 @@ def test_collection_collect_only_live_logging(testdir, verbose):
             ]
         )
     elif verbose == "-qq":
-        assert "collected 1 item*" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*collected 1 item**")
         expected_lines.extend(["*test_collection_collect_only_live_logging.py: 1*"])
 
     result.stdout.fnmatch_lines(expected_lines)
@@ -983,7 +983,7 @@ def test_collection_logging_to_file(testdir):
 
     result = testdir.runpytest()
 
-    assert "--- live log collection ---" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*--- live log collection ---*")
 
     assert result.ret == 0
     assert os.path.isfile(log_file)
diff --git a/testing/python/collect.py b/testing/python/collect.py
index e6dd3e870..8fc882f88 100644
--- a/testing/python/collect.py
+++ b/testing/python/collect.py
@@ -1139,7 +1139,7 @@ def test_unorderable_types(testdir):
     """
     )
     result = testdir.runpytest()
-    assert "TypeError" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*TypeError*")
     assert result.ret == ExitCode.NO_TESTS_COLLECTED
 
 
diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py
index f4dbfdf09..7bacfb6e2 100644
--- a/testing/python/fixtures.py
+++ b/testing/python/fixtures.py
@@ -455,7 +455,7 @@ class TestFillFixtures:
                 "*1 error*",
             ]
         )
-        assert "INTERNAL" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*INTERNAL*")
 
     def test_fixture_excinfo_leak(self, testdir):
         # on python2 sys.excinfo would leak into fixture executions
@@ -2647,7 +2647,7 @@ class TestFixtureMarker:
             *3 passed*
         """
         )
-        assert "error" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*error*")
 
     def test_fixture_finalizer(self, testdir):
         testdir.makeconftest(
@@ -3151,7 +3151,7 @@ class TestShowFixtures:
             *hello world*
         """
         )
-        assert "arg0" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*arg0*")
 
     @pytest.mark.parametrize("testmod", [True, False])
     def test_show_fixtures_conftest(self, testdir, testmod):
diff --git a/testing/python/setup_only.py b/testing/python/setup_only.py
index 7c871a9ee..6343991ae 100644
--- a/testing/python/setup_only.py
+++ b/testing/python/setup_only.py
@@ -27,7 +27,7 @@ def test_show_only_active_fixtures(testdir, mode, dummy_yaml_custom_test):
     result.stdout.fnmatch_lines(
         ["*SETUP    F arg1*", "*test_arg1 (fixtures used: arg1)*", "*TEARDOWN F arg1*"]
     )
-    assert "_arg0" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*_arg0*")
 
 
 def test_show_different_scopes(testdir, mode):
diff --git a/testing/python/show_fixtures_per_test.py b/testing/python/show_fixtures_per_test.py
index aff8aa0e5..ef841819d 100644
--- a/testing/python/show_fixtures_per_test.py
+++ b/testing/python/show_fixtures_per_test.py
@@ -1,6 +1,6 @@
 def test_no_items_should_not_show_output(testdir):
     result = testdir.runpytest("--fixtures-per-test")
-    assert "fixtures used by" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*fixtures used by*")
     assert result.ret == 0
 
 
@@ -30,7 +30,7 @@ def test_fixtures_in_module(testdir):
             "    arg1 docstring",
         ]
     )
-    assert "_arg0" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*_arg0*")
 
 
 def test_fixtures_in_conftest(testdir):
diff --git a/testing/test_assertion.py b/testing/test_assertion.py
index 8fce5e279..56729d28a 100644
--- a/testing/test_assertion.py
+++ b/testing/test_assertion.py
@@ -1034,7 +1034,7 @@ def test_assertion_options(testdir):
     result = testdir.runpytest()
     assert "3 == 4" in result.stdout.str()
     result = testdir.runpytest_subprocess("--assert=plain")
-    assert "3 == 4" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*3 == 4*")
 
 
 def test_triple_quoted_string_issue113(testdir):
@@ -1046,7 +1046,7 @@ def test_triple_quoted_string_issue113(testdir):
     )
     result = testdir.runpytest("--fulltrace")
     result.stdout.fnmatch_lines(["*1 failed*"])
-    assert "SyntaxError" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*SyntaxError*")
 
 
 def test_traceback_failure(testdir):
diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py
index 89b23a72c..470c54145 100644
--- a/testing/test_assertrewrite.py
+++ b/testing/test_assertrewrite.py
@@ -914,7 +914,7 @@ def test_rewritten():
         testdir.chdir()
         result = testdir.runpytest_subprocess()
         result.stdout.fnmatch_lines(["*= 1 passed in *=*"])
-        assert "pytest-warning summary" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*pytest-warning summary*")
 
     def test_rewrite_warning_using_pytest_plugins_env_var(self, testdir, monkeypatch):
         monkeypatch.setenv("PYTEST_PLUGINS", "plugin")
@@ -932,7 +932,7 @@ def test_rewritten():
         testdir.chdir()
         result = testdir.runpytest_subprocess()
         result.stdout.fnmatch_lines(["*= 1 passed in *=*"])
-        assert "pytest-warning summary" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*pytest-warning summary*")
 
 
 class TestAssertionRewriteHookDetails:
@@ -1124,7 +1124,7 @@ def test_issue731(testdir):
     """
     )
     result = testdir.runpytest()
-    assert "unbalanced braces" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*unbalanced braces*")
 
 
 class TestIssue925:
diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py
index cbba27e5f..e2fd5a4ca 100644
--- a/testing/test_cacheprovider.py
+++ b/testing/test_cacheprovider.py
@@ -327,7 +327,7 @@ class TestLastFailed:
         result = testdir.runpytest("--lf", "--ff")
         # Test order will be failing tests firs
         result.stdout.fnmatch_lines(["test_b.py*"])
-        assert "test_a.py" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*test_a.py*")
 
     def test_lastfailed_difference_invocations(self, testdir, monkeypatch):
         monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", "1")
@@ -660,11 +660,11 @@ class TestLastFailed:
         if quiet:
             args.append("-q")
         result = testdir.runpytest(*args)
-        assert "run all" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*run all*")
 
         result = testdir.runpytest(*args)
         if quiet:
-            assert "run all" not in result.stdout.str()
+            result.stdout.no_fnmatch_line("*run all*")
         else:
             assert "rerun previous" in result.stdout.str()
 
diff --git a/testing/test_capture.py b/testing/test_capture.py
index f5b193597..0f7db4b8e 100644
--- a/testing/test_capture.py
+++ b/testing/test_capture.py
@@ -609,12 +609,12 @@ class TestCaptureFixture:
             *while capture is disabled*
         """
         )
-        assert "captured before" not in result.stdout.str()
-        assert "captured after" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*captured before*")
+        result.stdout.no_fnmatch_line("*captured after*")
         if no_capture:
             assert "test_normal executed" in result.stdout.str()
         else:
-            assert "test_normal executed" not in result.stdout.str()
+            result.stdout.no_fnmatch_line("*test_normal executed*")
 
     @pytest.mark.parametrize("fixture", ["capsys", "capfd"])
     def test_fixture_use_by_other_fixtures(self, testdir, fixture):
@@ -650,8 +650,8 @@ class TestCaptureFixture:
         )
         result = testdir.runpytest_subprocess()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        assert "stdout contents begin" not in result.stdout.str()
-        assert "stderr contents begin" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*stdout contents begin*")
+        result.stdout.no_fnmatch_line("*stderr contents begin*")
 
     @pytest.mark.parametrize("cap", ["capsys", "capfd"])
     def test_fixture_use_by_other_fixtures_teardown(self, testdir, cap):
@@ -721,7 +721,7 @@ def test_capture_conftest_runtest_setup(testdir):
     testdir.makepyfile("def test_func(): pass")
     result = testdir.runpytest()
     assert result.ret == 0
-    assert "hello19" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*hello19*")
 
 
 def test_capture_badoutput_issue412(testdir):
@@ -1388,7 +1388,7 @@ def test_crash_on_closing_tmpfile_py27(testdir):
     result = testdir.runpytest_subprocess(str(p))
     assert result.ret == 0
     assert result.stderr.str() == ""
-    assert "IOError" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*IOError*")
 
 
 def test_pickling_and_unpickling_encoded_file():
diff --git a/testing/test_collection.py b/testing/test_collection.py
index dee07d5c7..7a5cf795b 100644
--- a/testing/test_collection.py
+++ b/testing/test_collection.py
@@ -139,7 +139,7 @@ class TestCollectFS:
 
         # by default, ignore tests inside a virtualenv
         result = testdir.runpytest()
-        assert "test_invenv" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*test_invenv*")
         # allow test collection if user insists
         result = testdir.runpytest("--collect-in-virtualenv")
         assert "test_invenv" in result.stdout.str()
@@ -165,7 +165,7 @@ class TestCollectFS:
         testfile = testdir.tmpdir.ensure(".virtual", "test_invenv.py")
         testfile.write("def test_hello(): pass")
         result = testdir.runpytest("--collect-in-virtualenv")
-        assert "test_invenv" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*test_invenv*")
         # ...unless the virtualenv is explicitly given on the CLI
         result = testdir.runpytest("--collect-in-virtualenv", ".virtual")
         assert "test_invenv" in result.stdout.str()
@@ -364,7 +364,7 @@ class TestCustomConftests:
         testdir.makepyfile(test_world="def test_hello(): pass")
         result = testdir.runpytest()
         assert result.ret == ExitCode.NO_TESTS_COLLECTED
-        assert "passed" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*passed*")
         result = testdir.runpytest("--XX")
         assert result.ret == 0
         assert "passed" in result.stdout.str()
@@ -857,7 +857,7 @@ def test_exit_on_collection_with_maxfail_smaller_than_n_errors(testdir):
         ["*ERROR collecting test_02_import_error.py*", "*No module named *asdfa*"]
     )
 
-    assert "test_03" not in res.stdout.str()
+    res.stdout.no_fnmatch_line("*test_03*")
 
 
 def test_exit_on_collection_with_maxfail_bigger_than_n_errors(testdir):
@@ -996,12 +996,12 @@ def test_collect_init_tests(testdir):
     result.stdout.fnmatch_lines(
         ["<Package */tests>", "  <Module test_foo.py>", "    <Function test_foo>"]
     )
-    assert "test_init" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*test_init*")
     result = testdir.runpytest("./tests/__init__.py", "--collect-only")
     result.stdout.fnmatch_lines(
         ["<Package */tests>", "  <Module __init__.py>", "    <Function test_init>"]
     )
-    assert "test_foo" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*test_foo*")
 
 
 def test_collect_invalid_signature_message(testdir):
diff --git a/testing/test_conftest.py b/testing/test_conftest.py
index 3f08ee381..0374db0b3 100644
--- a/testing/test_conftest.py
+++ b/testing/test_conftest.py
@@ -187,7 +187,7 @@ def test_conftest_confcutdir(testdir):
     )
     result = testdir.runpytest("-h", "--confcutdir=%s" % x, x)
     result.stdout.fnmatch_lines(["*--xyz*"])
-    assert "warning: could not load initial" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*warning: could not load initial*")
 
 
 @pytest.mark.skipif(
@@ -648,5 +648,5 @@ def test_required_option_help(testdir):
         )
     )
     result = testdir.runpytest("-h", x)
-    assert "argument --xyz is required" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*argument --xyz is required*")
     assert "general:" in result.stdout.str()
diff --git a/testing/test_doctest.py b/testing/test_doctest.py
index 4aac5432d..755f26286 100644
--- a/testing/test_doctest.py
+++ b/testing/test_doctest.py
@@ -239,8 +239,8 @@ class TestDoctests:
             ]
         )
         # lines below should be trimmed out
-        assert "text-line-2" not in result.stdout.str()
-        assert "text-line-after" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*text-line-2*")
+        result.stdout.no_fnmatch_line("*text-line-after*")
 
     def test_docstring_full_context_around_error(self, testdir):
         """Test that we show the whole context before the actual line of a failing
@@ -1177,7 +1177,7 @@ class TestDoctestAutoUseFixtures:
             """
             )
         result = testdir.runpytest("--doctest-modules")
-        assert "FAILURES" not in str(result.stdout.str())
+        result.stdout.no_fnmatch_line("*FAILURES*")
         result.stdout.fnmatch_lines(["*=== 1 passed in *"])
 
     @pytest.mark.parametrize("scope", SCOPES)
@@ -1209,7 +1209,7 @@ class TestDoctestAutoUseFixtures:
         """
         )
         result = testdir.runpytest("--doctest-modules")
-        assert "FAILURES" not in str(result.stdout.str())
+        str(result.stdout.no_fnmatch_line("*FAILURES*"))
         result.stdout.fnmatch_lines(["*=== 1 passed in *"])
 
 
diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py
index d4a1f6cc3..06a033489 100644
--- a/testing/test_junitxml.py
+++ b/testing/test_junitxml.py
@@ -1216,7 +1216,7 @@ def test_runs_twice(testdir, run_and_parse):
     )
 
     result, dom = run_and_parse(f, f)
-    assert "INTERNALERROR" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR*")
     first, second = [x["classname"] for x in dom.find_by_tag("testcase")]
     assert first == second
 
@@ -1231,7 +1231,7 @@ def test_runs_twice_xdist(testdir, run_and_parse):
     )
 
     result, dom = run_and_parse(f, "--dist", "each", "--tx", "2*popen")
-    assert "INTERNALERROR" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR*")
     first, second = [x["classname"] for x in dom.find_by_tag("testcase")]
     assert first == second
 
@@ -1271,7 +1271,7 @@ def test_fancy_items_regression(testdir, run_and_parse):
 
     result, dom = run_and_parse()
 
-    assert "INTERNALERROR" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR*")
 
     items = sorted("%(classname)s %(name)s" % x for x in dom.find_by_tag("testcase"))
     import pprint
diff --git a/testing/test_runner.py b/testing/test_runner.py
index 82e413518..9920d2b3e 100644
--- a/testing/test_runner.py
+++ b/testing/test_runner.py
@@ -615,7 +615,7 @@ def test_pytest_fail_notrace_runtest(testdir):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["world", "hello"])
-    assert "def teardown_function" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*def teardown_function*")
 
 
 def test_pytest_fail_notrace_collection(testdir):
@@ -630,7 +630,7 @@ def test_pytest_fail_notrace_collection(testdir):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["hello"])
-    assert "def some_internal_function()" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*def some_internal_function()*")
 
 
 def test_pytest_fail_notrace_non_ascii(testdir):
@@ -648,7 +648,7 @@ def test_pytest_fail_notrace_non_ascii(testdir):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["*test_hello*", "oh oh: ☺"])
-    assert "def test_hello" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*def test_hello*")
 
 
 def test_pytest_no_tests_collected_exit_status(testdir):
@@ -813,7 +813,7 @@ def test_failure_in_setup(testdir):
     """
     )
     result = testdir.runpytest("--tb=line")
-    assert "def setup_module" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*def setup_module*")
 
 
 def test_makereport_getsource(testdir):
@@ -825,7 +825,7 @@ def test_makereport_getsource(testdir):
     """
     )
     result = testdir.runpytest()
-    assert "INTERNALERROR" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR*")
     result.stdout.fnmatch_lines(["*else: assert False*"])
 
 
@@ -856,7 +856,7 @@ def test_makereport_getsource_dynamic_code(testdir, monkeypatch):
     """
     )
     result = testdir.runpytest("-vv")
-    assert "INTERNALERROR" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR*")
     result.stdout.fnmatch_lines(["*test_fix*", "*fixture*'missing'*not found*"])
 
 
diff --git a/testing/test_runner_xunit.py b/testing/test_runner_xunit.py
index 34a086551..1e63bbf49 100644
--- a/testing/test_runner_xunit.py
+++ b/testing/test_runner_xunit.py
@@ -237,7 +237,7 @@ def test_setup_funcarg_setup_when_outer_scope_fails(testdir):
             "*2 error*",
         ]
     )
-    assert "xyz43" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*xyz43*")
 
 
 @pytest.mark.parametrize("arg", ["", "arg"])
diff --git a/testing/test_skipping.py b/testing/test_skipping.py
index 8bba479f1..51b1bbdd6 100644
--- a/testing/test_skipping.py
+++ b/testing/test_skipping.py
@@ -949,7 +949,7 @@ def test_xfail_test_setup_exception(testdir):
     result = testdir.runpytest(p)
     assert result.ret == 0
     assert "xfailed" in result.stdout.str()
-    assert "xpassed" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*xpassed*")
 
 
 def test_imperativeskip_on_xfail_test(testdir):
diff --git a/testing/test_terminal.py b/testing/test_terminal.py
index 88f96f894..3bdabc5de 100644
--- a/testing/test_terminal.py
+++ b/testing/test_terminal.py
@@ -204,7 +204,7 @@ class TestTerminal:
         result = testdir.runpytest("-vv")
         assert result.ret == 0
         result.stdout.fnmatch_lines(["*a123/test_hello123.py*PASS*"])
-        assert " <- " not in result.stdout.str()
+        result.stdout.no_fnmatch_line("* <- *")
 
     def test_keyboard_interrupt(self, testdir, option):
         testdir.makepyfile(
@@ -559,7 +559,7 @@ class TestTerminalFunctional:
                 "*= 2 passed, 1 deselected in * =*",
             ]
         )
-        assert "= 1 deselected =" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*= 1 deselected =*")
         assert result.ret == 0
 
     def test_no_skip_summary_if_failure(self, testdir):
@@ -759,7 +759,7 @@ def test_fail_extra_reporting(testdir, monkeypatch):
     monkeypatch.setenv("COLUMNS", "80")
     testdir.makepyfile("def test_this(): assert 0, 'this_failed' * 100")
     result = testdir.runpytest()
-    assert "short test summary" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*short test summary*")
     result = testdir.runpytest("-rf")
     result.stdout.fnmatch_lines(
         [
@@ -772,13 +772,13 @@ def test_fail_extra_reporting(testdir, monkeypatch):
 def test_fail_reporting_on_pass(testdir):
     testdir.makepyfile("def test_this(): assert 1")
     result = testdir.runpytest("-rf")
-    assert "short test summary" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*short test summary*")
 
 
 def test_pass_extra_reporting(testdir):
     testdir.makepyfile("def test_this(): assert 1")
     result = testdir.runpytest()
-    assert "short test summary" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*short test summary*")
     result = testdir.runpytest("-rp")
     result.stdout.fnmatch_lines(["*test summary*", "PASS*test_pass_extra_reporting*"])
 
@@ -786,7 +786,7 @@ def test_pass_extra_reporting(testdir):
 def test_pass_reporting_on_fail(testdir):
     testdir.makepyfile("def test_this(): assert 0")
     result = testdir.runpytest("-rp")
-    assert "short test summary" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*short test summary*")
 
 
 def test_pass_output_reporting(testdir):
@@ -829,7 +829,7 @@ def test_color_no(testdir):
     testdir.makepyfile("def test_this(): assert 1")
     result = testdir.runpytest("--color=no")
     assert "test session starts" in result.stdout.str()
-    assert "\x1b[1m" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*\x1b[1m*")
 
 
 @pytest.mark.parametrize("verbose", [True, False])
@@ -851,7 +851,7 @@ def test_color_yes_collection_on_non_atty(testdir, verbose):
     result = testdir.runpytest(*args)
     assert "test session starts" in result.stdout.str()
     assert "\x1b[1m" in result.stdout.str()
-    assert "collecting 10 items" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*collecting 10 items*")
     if verbose:
         assert "collecting ..." in result.stdout.str()
     assert "collected 10 items" in result.stdout.str()
@@ -1214,7 +1214,7 @@ def test_terminal_summary_warnings_are_displayed(testdir):
             "*== 1 failed, 2 warnings in *",
         ]
     )
-    assert "None" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*None*")
     stdout = result.stdout.str()
     assert stdout.count("warning_from_test") == 1
     assert stdout.count("=== warnings summary ") == 2
@@ -1239,7 +1239,7 @@ def test_terminal_summary_warnings_header_once(testdir):
             "*== 1 failed, 1 warnings in *",
         ]
     )
-    assert "None" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*None*")
     stdout = result.stdout.str()
     assert stdout.count("warning_from_test") == 1
     assert stdout.count("=== warnings summary ") == 1
@@ -1402,7 +1402,7 @@ class TestProgressOutputStyle:
         """
         )
         output = testdir.runpytest()
-        assert "ZeroDivisionError" not in output.stdout.str()
+        output.stdout.no_fnmatch_line("*ZeroDivisionError*")
         output.stdout.fnmatch_lines(["=* 2 passed in *="])
 
     def test_normal(self, many_tests_files, testdir):
@@ -1494,7 +1494,7 @@ class TestProgressOutputStyle:
         )
 
         output = testdir.runpytest("--capture=no")
-        assert "%]" not in output.stdout.str()
+        output.stdout.no_fnmatch_line("*%]*")
 
 
 class TestProgressWithTeardown:
diff --git a/testing/test_unittest.py b/testing/test_unittest.py
index 9b1b688ff..281c85281 100644
--- a/testing/test_unittest.py
+++ b/testing/test_unittest.py
@@ -270,7 +270,7 @@ def test_setup_failure_is_shown(testdir):
     result = testdir.runpytest("-s")
     assert result.ret == 1
     result.stdout.fnmatch_lines(["*setUp*", "*assert 0*down1*", "*1 failed*"])
-    assert "never42" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*never42*")
 
 
 def test_setup_setUpClass(testdir):
@@ -342,7 +342,7 @@ def test_testcase_adderrorandfailure_defers(testdir, type):
         % (type, type)
     )
     result = testdir.runpytest()
-    assert "should not raise" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*should not raise*")
 
 
 @pytest.mark.parametrize("type", ["Error", "Failure"])
@@ -684,7 +684,7 @@ def test_unittest_not_shown_in_traceback(testdir):
     """
     )
     res = testdir.runpytest()
-    assert "failUnlessEqual" not in res.stdout.str()
+    res.stdout.no_fnmatch_line("*failUnlessEqual*")
 
 
 def test_unorderable_types(testdir):
@@ -703,7 +703,7 @@ def test_unorderable_types(testdir):
     """
     )
     result = testdir.runpytest()
-    assert "TypeError" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*TypeError*")
     assert result.ret == ExitCode.NO_TESTS_COLLECTED
 
 
@@ -1020,7 +1020,7 @@ def test_testcase_handles_init_exceptions(testdir):
     )
     result = testdir.runpytest()
     assert "should raise this exception" in result.stdout.str()
-    assert "ERROR at teardown of MyTestCase.test_hello" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*ERROR at teardown of MyTestCase.test_hello*")
 
 
 def test_error_message_with_parametrized_fixtures(testdir):