diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py
index 8d1b82f70..3bc77857d 100644
--- a/testing/test_stepwise.py
+++ b/testing/test_stepwise.py
@@ -167,11 +167,13 @@ def test_stop_on_collection_errors(broken_testdir, broken_first):
     result.stdout.fnmatch_lines("*error during collection*")
 
 
-def test_xfail_handling(testdir):
+def test_xfail_handling(testdir, monkeypatch):
     """Ensure normal xfail is ignored, and strict xfail interrupts the session in sw mode
 
     (#5547)
     """
+    monkeypatch.setattr("sys.dont_write_bytecode", True)
+
     contents = """
         import pytest
         def test_a(): pass
@@ -205,10 +207,6 @@ def test_xfail_handling(testdir):
         ]
     )
 
-    # because we are writing to the same file, mtime might not be affected enough to
-    # invalidate the cache, making this next run flaky
-    if testdir.tmpdir.join("__pycache__").exists():
-        testdir.tmpdir.join("__pycache__").remove()
     testdir.makepyfile(contents.format(assert_value="0", strict="True"))
     result = testdir.runpytest("--sw", "-v")
     result.stdout.fnmatch_lines(