diff --git a/changelog/5664.trivial.rst b/changelog/5664.trivial.rst
new file mode 100644
index 000000000..3928454ef
--- /dev/null
+++ b/changelog/5664.trivial.rst
@@ -0,0 +1,2 @@
+When invoking pytest's own testsuite with ``PYTHONDONTWRITEBYTECODE=1``,
+the ``test_xfail_handling`` test no longer fails.
diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py
index 591d67b6c..f61425b6b 100644
--- a/testing/test_stepwise.py
+++ b/testing/test_stepwise.py
@@ -207,7 +207,8 @@ def test_xfail_handling(testdir):
 
     # because we are writing to the same file, mtime might not be affected enough to
     # invalidate the cache, making this next run flaky
-    testdir.tmpdir.join("__pycache__").remove()
+    if testdir.tmpdir.join("__pycache__").exists():
+        testdir.tmpdir.join("__pycache__").remove()
     testdir.makepyfile(contents.format(assert_value="0", strict="True"))
     result = testdir.runpytest("--sw", "-v")
     result.stdout.fnmatch_lines(