Handle dirs only once

Time: 5.73s/5.88s => 5.36s

(Before rebase: 4.86s => 4.45s)

src/_pytest/main.py

@@ -529,10 +529,14 @@ class Session(nodes.FSCollector):
                 def filter_(f):
                     return f.check(file=1)
 
+            seen_dirs = set()
             for path in argpath.visit(
                 fil=filter_, rec=self._recurse, bf=True, sort=True
             ):
-                pkginit = path.dirpath().join("__init__.py")
+                dirpath = path.dirpath()
+                if dirpath not in seen_dirs:
+                    seen_dirs.add(dirpath)
+                    pkginit = dirpath.join("__init__.py")
                     if pkginit.exists() and not any(
                         x in parts(pkginit.strpath) for x in paths
                     ):