Merge pull request #7541 from bluetech/py-visit

pathlib: stop using py.path.local.visit(), use os.scandir

src/_pytest/main.py

@@ -32,6 +32,7 @@ from _pytest.config.argparsing import Parser
 from _pytest.fixtures import FixtureManager
 from _pytest.outcomes import exit
 from _pytest.pathlib import Path
+from _pytest.pathlib import visit
 from _pytest.reports import CollectReport
 from _pytest.reports import TestReport
 from _pytest.runner import collect_one_node
@@ -617,10 +618,13 @@ class Session(nodes.FSCollector):
             assert not names, "invalid arg {!r}".format((argpath, names))
 
             seen_dirs = set()  # type: Set[py.path.local]
-            for path in argpath.visit(
+            for direntry in visit(str(argpath), self._recurse):
-                fil=self._visit_filter, rec=self._recurse, bf=True, sort=True
+                if not direntry.is_file():
-            ):
+                    continue
+
+                path = py.path.local(direntry.path)
                 dirpath = path.dirpath()
+
                 if dirpath not in seen_dirs:
                     # Collect packages first.
                     seen_dirs.add(dirpath)
@@ -668,11 +672,6 @@ class Session(nodes.FSCollector):
                 return
             yield from m
 
-    @staticmethod
-    def _visit_filter(f: py.path.local) -> bool:
-        # TODO: Remove type: ignore once `py` is typed.
-        return f.check(file=1)  # type: ignore
-
     def _tryconvertpyarg(self, x: str) -> str:
         """Convert a dotted module name to path."""
         try:

src/_pytest/nodes.py

@@ -562,17 +562,18 @@ class FSCollector(Collector):
     def gethookproxy(self, fspath: py.path.local):
         raise NotImplementedError()
 
-    def _recurse(self, dirpath: py.path.local) -> bool:
+    def _recurse(self, direntry: "os.DirEntry[str]") -> bool:
-        if dirpath.basename == "__pycache__":
+        if direntry.name == "__pycache__":
             return False
-        ihook = self._gethookproxy(dirpath.dirpath())
+        path = py.path.local(direntry.path)
-        if ihook.pytest_ignore_collect(path=dirpath, config=self.config):
+        ihook = self._gethookproxy(path.dirpath())
+        if ihook.pytest_ignore_collect(path=path, config=self.config):
             return False
         for pat in self._norecursepatterns:
-            if dirpath.check(fnmatch=pat):
+            if path.check(fnmatch=pat):
                 return False
-        ihook = self._gethookproxy(dirpath)
+        ihook = self._gethookproxy(path)
-        ihook.pytest_collect_directory(path=dirpath, parent=self)
+        ihook.pytest_collect_directory(path=path, parent=self)
         return True
 
     def isinitpath(self, path: py.path.local) -> bool:

src/_pytest/pathlib.py

@@ -16,6 +16,7 @@ from os.path import isabs
 from os.path import sep
 from posixpath import sep as posix_sep
 from types import ModuleType
+from typing import Callable
 from typing import Iterable
 from typing import Iterator
 from typing import Optional
@@ -556,3 +557,17 @@ def resolve_package_path(path: Path) -> Optional[Path]:
                 break
             result = parent
     return result
+
+
+def visit(
+    path: str, recurse: Callable[["os.DirEntry[str]"], bool]
+) -> Iterator["os.DirEntry[str]"]:
+    """Walk a directory recursively, in breadth-first order.
+
+    Entries at each directory level are sorted.
+    """
+    entries = sorted(os.scandir(path), key=lambda entry: entry.name)
+    yield from entries
+    for entry in entries:
+        if entry.is_dir(follow_symlinks=False) and recurse(entry):
+            yield from visit(entry.path, recurse)

src/_pytest/python.py

@@ -65,6 +65,7 @@ from _pytest.outcomes import skip
 from _pytest.pathlib import import_path
 from _pytest.pathlib import ImportPathMismatchError
 from _pytest.pathlib import parts
+from _pytest.pathlib import visit
 from _pytest.reports import TerminalRepr
 from _pytest.warning_types import PytestCollectionWarning
 from _pytest.warning_types import PytestUnhandledCoroutineWarning
@@ -641,23 +642,24 @@ class Package(Module):
         ):
             yield Module.from_parent(self, fspath=init_module)
         pkg_prefixes = set()  # type: Set[py.path.local]
-        for path in this_path.visit(rec=self._recurse, bf=True, sort=True):
+        for direntry in visit(str(this_path), recurse=self._recurse):
+            path = py.path.local(direntry.path)
+
             # We will visit our own __init__.py file, in which case we skip it.
-            is_file = path.isfile()
+            if direntry.is_file():
-            if is_file:
+                if direntry.name == "__init__.py" and path.dirpath() == this_path:
-                if path.basename == "__init__.py" and path.dirpath() == this_path:
                     continue
 
-            parts_ = parts(path.strpath)
+            parts_ = parts(direntry.path)
             if any(
                 str(pkg_prefix) in parts_ and pkg_prefix.join("__init__.py") != path
                 for pkg_prefix in pkg_prefixes
             ):
                 continue
 
-            if is_file:
+            if direntry.is_file():
                 yield from self._collectfile(path)
-            elif not path.isdir():
+            elif not direntry.is_dir():
                 # Broken symlink or invalid/missing file.
                 continue
             elif path.join("__init__.py").check(file=1):

testing/python/fixtures.py

@@ -142,14 +142,14 @@ class TestFillFixtures:
         p = testdir.copy_example()
         result = testdir.runpytest()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        result = testdir.runpytest(next(p.visit("test_*.py")))
+        result = testdir.runpytest(str(next(Path(str(p)).rglob("test_*.py"))))
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_extend_fixture_conftest_conftest(self, testdir):
         p = testdir.copy_example()
         result = testdir.runpytest()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        result = testdir.runpytest(next(p.visit("test_*.py")))
+        result = testdir.runpytest(str(next(Path(str(p)).rglob("test_*.py"))))
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_extend_fixture_conftest_plugin(self, testdir):

testing/test_collection.py

@@ -7,6 +7,7 @@ import pytest
 from _pytest.config import ExitCode
 from _pytest.main import _in_venv
 from _pytest.main import Session
+from _pytest.pathlib import Path
 from _pytest.pathlib import symlink_or_skip
 from _pytest.pytester import Testdir
 
@@ -115,8 +116,8 @@ class TestCollectFS:
         tmpdir.ensure(".whatever", "test_notfound.py")
         tmpdir.ensure(".bzr", "test_notfound.py")
         tmpdir.ensure("normal", "test_found.py")
-        for x in tmpdir.visit("test_*.py"):
+        for x in Path(str(tmpdir)).rglob("test_*.py"):
-            x.write("def test_hello(): pass")
+            x.write_text("def test_hello(): pass", "utf-8")
 
         result = testdir.runpytest("--collect-only")
         s = result.stdout.str()

testing/test_conftest.py

@@ -477,8 +477,9 @@ class TestConftestVisibility:
             )
         )
         print("created directory structure:")
-        for x in testdir.tmpdir.visit():
+        tmppath = Path(str(testdir.tmpdir))
-            print("   " + x.relto(testdir.tmpdir))
+        for x in tmppath.rglob(""):
+            print("   " + str(x.relative_to(tmppath)))
 
         return {"runner": runner, "package": package, "swc": swc, "snc": snc}