From 5e0441d96e21c9655e38ac564ef1577f418acdcb Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 4 Nov 2018 09:02:28 -0800 Subject: [PATCH 01/13] Update pypi.python.org reference to pypi.org --- doc/en/plugins.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/en/plugins.rst b/doc/en/plugins.rst index 62456e7dd..3513e79fb 100644 --- a/doc/en/plugins.rst +++ b/doc/en/plugins.rst @@ -59,9 +59,9 @@ To see a complete list of all plugins with their latest testing status against different pytest and Python versions, please visit `plugincompat `_. -You may also discover more plugins through a `pytest- pypi.python.org search`_. +You may also discover more plugins through a `pytest- pypi.org search`_. -.. _`pytest- pypi.python.org search`: https://pypi.org/search/?q=pytest- +.. _`pytest- pypi.org search`: https://pypi.org/search/?q=pytest- .. _`available installable plugins`: From 34152445cff55ca0eb66a896d23b2fdf2677e9da Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 4 Nov 2018 18:25:16 +0100 Subject: [PATCH 02/13] doc: add lost changelog entry Closes https://github.com/pytest-dev/pytest/issues/4300. [ci skip] --- CHANGELOG.rst | 2 ++ doc/4266.bugfix.rst | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 doc/4266.bugfix.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eef3b42e9..b604cf1f5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -60,6 +60,8 @@ Bug Fixes - `#611 `_: Naming a fixture ``request`` will now raise a warning: the ``request`` fixture is internal and should not be overwritten as it will lead to internal errors. +- `#4266 `_: Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class. + Improved Documentation diff --git a/doc/4266.bugfix.rst b/doc/4266.bugfix.rst deleted file mode 100644 index f19a7cc1f..000000000 --- a/doc/4266.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class. From 7cb271b46fb4089fae048a144d1576cd6fbe9432 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 4 Nov 2018 21:59:28 +0100 Subject: [PATCH 03/13] replace byte/unicode helpers in test_capture with python level syntax --- changelog/4305.trivial.rst | 1 + testing/test_capture.py | 47 ++++++++++++-------------------------- 2 files changed, 16 insertions(+), 32 deletions(-) create mode 100644 changelog/4305.trivial.rst diff --git a/changelog/4305.trivial.rst b/changelog/4305.trivial.rst new file mode 100644 index 000000000..2430a5f91 --- /dev/null +++ b/changelog/4305.trivial.rst @@ -0,0 +1 @@ +Replace byte/unicode helpers in test_capture with python level syntax. diff --git a/testing/test_capture.py b/testing/test_capture.py index 3dc422efe..73f0bed59 100644 --- a/testing/test_capture.py +++ b/testing/test_capture.py @@ -23,24 +23,6 @@ needsosdup = pytest.mark.skipif( ) -def tobytes(obj): - if isinstance(obj, text_type): - obj = obj.encode("UTF-8") - assert isinstance(obj, bytes) - return obj - - -def totext(obj): - if isinstance(obj, bytes): - obj = text_type(obj, "UTF-8") - assert isinstance(obj, text_type) - return obj - - -def oswritebytes(fd, obj): - os.write(fd, tobytes(obj)) - - def StdCaptureFD(out=True, err=True, in_=True): return capture.MultiCapture(out, err, in_, Capture=capture.FDCapture) @@ -832,10 +814,11 @@ class TestCaptureIO(object): def test_bytes_io(): f = py.io.BytesIO() - f.write(tobytes("hello")) - pytest.raises(TypeError, "f.write(totext('hello'))") + f.write(b"hello") + with pytest.raises(TypeError): + f.write(u"hello") s = f.getvalue() - assert s == tobytes("hello") + assert s == b"hello" def test_dontreadfrominput(): @@ -948,7 +931,7 @@ class TestFDCapture(object): def test_simple(self, tmpfile): fd = tmpfile.fileno() cap = capture.FDCapture(fd) - data = tobytes("hello") + data = b"hello" os.write(fd, data) s = cap.snap() cap.done() @@ -988,10 +971,10 @@ class TestFDCapture(object): cap.start() x = os.read(0, 100).strip() cap.done() - assert x == tobytes("") + assert x == b"" def test_writeorg(self, tmpfile): - data1, data2 = tobytes("foo"), tobytes("bar") + data1, data2 = b"foo", b"bar" cap = capture.FDCapture(tmpfile.fileno()) cap.start() tmpfile.write(data1) @@ -999,7 +982,7 @@ class TestFDCapture(object): cap.writeorg(data2) scap = cap.snap() cap.done() - assert scap == totext(data1) + assert scap == data1.decode("ascii") with open(tmpfile.name, "rb") as stmp_file: stmp = stmp_file.read() assert stmp == data2 @@ -1008,17 +991,17 @@ class TestFDCapture(object): with saved_fd(1): cap = capture.FDCapture(1) cap.start() - data = tobytes("hello") + data = b"hello" os.write(1, data) sys.stdout.write("whatever") s = cap.snap() assert s == "hellowhatever" cap.suspend() - os.write(1, tobytes("world")) + os.write(1, b"world") sys.stdout.write("qlwkej") assert not cap.snap() cap.resume() - os.write(1, tobytes("but now")) + os.write(1, b"but now") sys.stdout.write(" yes\n") s = cap.snap() assert s == "but now yes\n" @@ -1189,14 +1172,14 @@ class TestStdCaptureFD(TestStdCapture): def test_intermingling(self): with self.getcapture() as cap: - oswritebytes(1, "1") + os.write(1, b"1") sys.stdout.write(str(2)) sys.stdout.flush() - oswritebytes(1, "3") - oswritebytes(2, "a") + os.write(1, b"3") + os.write(2, b"a") sys.stderr.write("b") sys.stderr.flush() - oswritebytes(2, "c") + os.write(2, b"c") out, err = cap.readouterr() assert out == "123" assert err == "abc" From 85a33338247bc2c06c4711be2b0d5621efae1ef4 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 4 Nov 2018 17:28:35 -0800 Subject: [PATCH 04/13] Don't string-compare version numbers --- changelog/4306.bugfix.rst | 1 + src/_pytest/config/__init__.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 changelog/4306.bugfix.rst diff --git a/changelog/4306.bugfix.rst b/changelog/4306.bugfix.rst new file mode 100644 index 000000000..cb2872d3f --- /dev/null +++ b/changelog/4306.bugfix.rst @@ -0,0 +1 @@ +Parse ``minversion`` as an actual version and not as dot-separated strings. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 6fbf8144a..b42d6f843 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -11,6 +11,7 @@ import shlex import sys import types import warnings +from distutils.version import LooseVersion import py import six @@ -816,9 +817,7 @@ class Config(object): minver = self.inicfg.get("minversion", None) if minver: - ver = minver.split(".") - myver = pytest.__version__.split(".") - if myver < ver: + if LooseVersion(minver) > LooseVersion(pytest.__version__): raise pytest.UsageError( "%s:%d: requires pytest-%s, actual pytest-%s'" % ( From a4819844a422994729e9cbc33765a02c720fa2f2 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 4 Nov 2018 17:43:24 -0800 Subject: [PATCH 05/13] Use unicode/bytes literals instead of calls --- src/_pytest/assertion/util.py | 76 ++++++++++++++++------------------- src/_pytest/capture.py | 4 +- testing/test_assertion.py | 7 +--- 3 files changed, 38 insertions(+), 49 deletions(-) diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 063341095..451e45495 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -11,8 +11,6 @@ import six import _pytest._code from ..compat import Sequence -u = six.text_type - # The _reprcompare attribute on the util module is used by the new assertion # interpretation code and assertion rewriter to detect this plugin was # loaded and in turn call the hooks defined here as part of the @@ -23,9 +21,9 @@ _reprcompare = None # the re-encoding is needed for python2 repr # with non-ascii characters (see issue 877 and 1379) def ecu(s): - try: - return u(s, "utf-8", "replace") - except TypeError: + if isinstance(s, bytes): + return s.decode("UTF-8", "replace") + else: return s @@ -42,7 +40,7 @@ def format_explanation(explanation): explanation = ecu(explanation) lines = _split_explanation(explanation) result = _format_lines(lines) - return u("\n").join(result) + return u"\n".join(result) def _split_explanation(explanation): @@ -52,7 +50,7 @@ def _split_explanation(explanation): Any other newlines will be escaped and appear in the line as the literal '\n' characters. """ - raw_lines = (explanation or u("")).split("\n") + raw_lines = (explanation or u"").split("\n") lines = [raw_lines[0]] for values in raw_lines[1:]: if values and values[0] in ["{", "}", "~", ">"]: @@ -77,13 +75,13 @@ def _format_lines(lines): for line in lines[1:]: if line.startswith("{"): if stackcnt[-1]: - s = u("and ") + s = u"and " else: - s = u("where ") + s = u"where " stack.append(len(result)) stackcnt[-1] += 1 stackcnt.append(0) - result.append(u(" +") + u(" ") * (len(stack) - 1) + s + line[1:]) + result.append(u" +" + u" " * (len(stack) - 1) + s + line[1:]) elif line.startswith("}"): stack.pop() stackcnt.pop() @@ -92,7 +90,7 @@ def _format_lines(lines): assert line[0] in ["~", ">"] stack[-1] += 1 indent = len(stack) if line.startswith("~") else len(stack) - 1 - result.append(u(" ") * indent + line[1:]) + result.append(u" " * indent + line[1:]) assert len(stack) == 1 return result @@ -110,7 +108,7 @@ def assertrepr_compare(config, op, left, right): left_repr = py.io.saferepr(left, maxsize=int(width // 2)) right_repr = py.io.saferepr(right, maxsize=width - len(left_repr)) - summary = u("%s %s %s") % (ecu(left_repr), op, ecu(right_repr)) + summary = u"%s %s %s" % (ecu(left_repr), op, ecu(right_repr)) def issequence(x): return isinstance(x, Sequence) and not isinstance(x, basestring) @@ -155,11 +153,9 @@ def assertrepr_compare(config, op, left, right): explanation = _notin_text(left, right, verbose) except Exception: explanation = [ - u( - "(pytest_assertion plugin: representation of details failed. " - "Probably an object has a faulty __repr__.)" - ), - u(_pytest._code.ExceptionInfo()), + u"(pytest_assertion plugin: representation of details failed. " + u"Probably an object has a faulty __repr__.)", + six.text_type(_pytest._code.ExceptionInfo()), ] if not explanation: @@ -203,8 +199,7 @@ def _diff_text(left, right, verbose=False): if i > 42: i -= 10 # Provide some context explanation = [ - u("Skipping %s identical leading characters in diff, use -v to show") - % i + u"Skipping %s identical leading characters in diff, use -v to show" % i ] left = left[i:] right = right[i:] @@ -215,11 +210,8 @@ def _diff_text(left, right, verbose=False): if i > 42: i -= 10 # Provide some context explanation += [ - u( - "Skipping %s identical trailing " - "characters in diff, use -v to show" - ) - % i + u"Skipping {} identical trailing " + u"characters in diff, use -v to show".format(i) ] left = left[:-i] right = right[:-i] @@ -237,21 +229,21 @@ def _diff_text(left, right, verbose=False): def _compare_eq_iterable(left, right, verbose=False): if not verbose: - return [u("Use -v to get the full diff")] + return [u"Use -v to get the full diff"] # dynamic import to speedup pytest import difflib try: left_formatting = pprint.pformat(left).splitlines() right_formatting = pprint.pformat(right).splitlines() - explanation = [u("Full diff:")] + explanation = [u"Full diff:"] except Exception: # hack: PrettyPrinter.pformat() in python 2 fails when formatting items that can't be sorted(), ie, calling # sorted() on a list would raise. See issue #718. # As a workaround, the full diff is generated by using the repr() string of each item of each container. left_formatting = sorted(repr(x) for x in left) right_formatting = sorted(repr(x) for x in right) - explanation = [u("Full diff (fallback to calling repr on each item):")] + explanation = [u"Full diff (fallback to calling repr on each item):"] explanation.extend( line.strip() for line in difflib.ndiff(left_formatting, right_formatting) ) @@ -262,16 +254,16 @@ def _compare_eq_sequence(left, right, verbose=False): explanation = [] for i in range(min(len(left), len(right))): if left[i] != right[i]: - explanation += [u("At index %s diff: %r != %r") % (i, left[i], right[i])] + explanation += [u"At index %s diff: %r != %r" % (i, left[i], right[i])] break if len(left) > len(right): explanation += [ - u("Left contains more items, first extra item: %s") + u"Left contains more items, first extra item: %s" % py.io.saferepr(left[len(right)]) ] elif len(left) < len(right): explanation += [ - u("Right contains more items, first extra item: %s") + u"Right contains more items, first extra item: %s" % py.io.saferepr(right[len(left)]) ] return explanation @@ -282,11 +274,11 @@ def _compare_eq_set(left, right, verbose=False): diff_left = left - right diff_right = right - left if diff_left: - explanation.append(u("Extra items in the left set:")) + explanation.append(u"Extra items in the left set:") for item in diff_left: explanation.append(py.io.saferepr(item)) if diff_right: - explanation.append(u("Extra items in the right set:")) + explanation.append(u"Extra items in the right set:") for item in diff_right: explanation.append(py.io.saferepr(item)) return explanation @@ -297,26 +289,26 @@ def _compare_eq_dict(left, right, verbose=False): common = set(left).intersection(set(right)) same = {k: left[k] for k in common if left[k] == right[k]} if same and verbose < 2: - explanation += [u("Omitting %s identical items, use -vv to show") % len(same)] + explanation += [u"Omitting %s identical items, use -vv to show" % len(same)] elif same: - explanation += [u("Common items:")] + explanation += [u"Common items:"] explanation += pprint.pformat(same).splitlines() diff = {k for k in common if left[k] != right[k]} if diff: - explanation += [u("Differing items:")] + explanation += [u"Differing items:"] for k in diff: explanation += [ py.io.saferepr({k: left[k]}) + " != " + py.io.saferepr({k: right[k]}) ] extra_left = set(left) - set(right) if extra_left: - explanation.append(u("Left contains more items:")) + explanation.append(u"Left contains more items:") explanation.extend( pprint.pformat({k: left[k] for k in extra_left}).splitlines() ) extra_right = set(right) - set(left) if extra_right: - explanation.append(u("Right contains more items:")) + explanation.append(u"Right contains more items:") explanation.extend( pprint.pformat({k: right[k] for k in extra_right}).splitlines() ) @@ -329,14 +321,14 @@ def _notin_text(term, text, verbose=False): tail = text[index + len(term) :] correct_text = head + tail diff = _diff_text(correct_text, text, verbose) - newdiff = [u("%s is contained here:") % py.io.saferepr(term, maxsize=42)] + newdiff = [u"%s is contained here:" % py.io.saferepr(term, maxsize=42)] for line in diff: - if line.startswith(u("Skipping")): + if line.startswith(u"Skipping"): continue - if line.startswith(u("- ")): + if line.startswith(u"- "): continue - if line.startswith(u("+ ")): - newdiff.append(u(" ") + line[2:]) + if line.startswith(u"+ "): + newdiff.append(u" " + line[2:]) else: newdiff.append(line) return newdiff diff --git a/src/_pytest/capture.py b/src/_pytest/capture.py index 99e95a442..ec72ae3ec 100644 --- a/src/_pytest/capture.py +++ b/src/_pytest/capture.py @@ -504,7 +504,7 @@ class FDCaptureBinary(object): snap() produces `bytes` """ - EMPTY_BUFFER = bytes() + EMPTY_BUFFER = b"" def __init__(self, targetfd, tmpfile=None): self.targetfd = targetfd @@ -630,7 +630,7 @@ class SysCapture(object): class SysCaptureBinary(SysCapture): - EMPTY_BUFFER = bytes() + EMPTY_BUFFER = b"" def snap(self): res = self.tmpfile.buffer.getvalue() diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 94fe9d272..b6c31aba2 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -539,11 +539,8 @@ class TestAssert_reprcompare(object): def test_mojibake(self): # issue 429 - left = "e" - right = "\xc3\xa9" - if not isinstance(left, bytes): - left = bytes(left, "utf-8") - right = bytes(right, "utf-8") + left = b"e" + right = b"\xc3\xa9" expl = callequal(left, right) for line in expl: assert isinstance(line, six.text_type) From d42c490bc1cbd0d95e59cc0372683b311b6baaf7 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 5 Nov 2018 15:39:35 +0100 Subject: [PATCH 06/13] Add missing `-` in front of the new option `--sw` --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b604cf1f5..64a955198 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,7 +30,7 @@ Features existing ``pytest_enter_pdb`` hook. -- `#4147 `_: Add ``-sw``, ``--stepwise`` as an alternative to ``--lf -x`` for stopping at the first failure, but starting the next test invocation from that test. See `the documentation `__ for more info. +- `#4147 `_: Add ``--sw``, ``--stepwise`` as an alternative to ``--lf -x`` for stopping at the first failure, but starting the next test invocation from that test. See `the documentation `__ for more info. - `#4188 `_: Make ``--color`` emit colorful dots when not running in verbose mode. Earlier, it would only colorize the test-by-test output if ``--verbose`` was also passed. From fa35f650b53821baa85c78caf116fd9e4522c408 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 6 Nov 2018 18:47:19 +0100 Subject: [PATCH 07/13] Fix handling of duplicate args with regard to Python packages Fixes https://github.com/pytest-dev/pytest/issues/4310. --- changelog/4310.bugfix.rst | 1 + src/_pytest/main.py | 29 +++++++++++++++-------------- src/_pytest/python.py | 15 ++++++++++++++- testing/test_collection.py | 32 ++++++++++++++++++++++++-------- 4 files changed, 54 insertions(+), 23 deletions(-) create mode 100644 changelog/4310.bugfix.rst diff --git a/changelog/4310.bugfix.rst b/changelog/4310.bugfix.rst new file mode 100644 index 000000000..24e177c2e --- /dev/null +++ b/changelog/4310.bugfix.rst @@ -0,0 +1 @@ +Fix duplicate collection due to multiple args matching the same packages. diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 3c908ec4c..46228f824 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -387,6 +387,7 @@ class Session(nodes.FSCollector): self._initialpaths = frozenset() # Keep track of any collected nodes in here, so we don't duplicate fixtures self._node_cache = {} + self._collect_seen_pkgdirs = set() self.config.pluginmanager.register(self, name="session") @@ -496,18 +497,19 @@ class Session(nodes.FSCollector): # and stack all Packages found on the way. # No point in finding packages when collecting doctests if not self.config.option.doctestmodules: + pm = self.config.pluginmanager for parent in argpath.parts(): - pm = self.config.pluginmanager if pm._confcutdir and pm._confcutdir.relto(parent): continue if parent.isdir(): pkginit = parent.join("__init__.py") if pkginit.isfile(): + self._collect_seen_pkgdirs.add(parent) if pkginit in self._node_cache: root = self._node_cache[pkginit][0] else: - col = root._collectfile(pkginit) + col = root._collectfile(pkginit, handle_dupes=False) if col: if isinstance(col[0], Package): root = col[0] @@ -529,13 +531,12 @@ class Session(nodes.FSCollector): def filter_(f): return f.check(file=1) - seen_dirs = set() for path in argpath.visit( fil=filter_, rec=self._recurse, bf=True, sort=True ): dirpath = path.dirpath() - if dirpath not in seen_dirs: - seen_dirs.add(dirpath) + if dirpath not in self._collect_seen_pkgdirs: + self._collect_seen_pkgdirs.add(dirpath) pkginit = dirpath.join("__init__.py") if pkginit.exists() and parts(pkginit.strpath).isdisjoint(paths): for x in root._collectfile(pkginit): @@ -570,20 +571,20 @@ class Session(nodes.FSCollector): for y in m: yield y - def _collectfile(self, path): + def _collectfile(self, path, handle_dupes=True): ihook = self.gethookproxy(path) if not self.isinitpath(path): if ihook.pytest_ignore_collect(path=path, config=self.config): return () - # Skip duplicate paths. - keepduplicates = self.config.getoption("keepduplicates") - if not keepduplicates: - duplicate_paths = self.config.pluginmanager._duplicatepaths - if path in duplicate_paths: - return () - else: - duplicate_paths.add(path) + if handle_dupes: + keepduplicates = self.config.getoption("keepduplicates") + if not keepduplicates: + duplicate_paths = self.config.pluginmanager._duplicatepaths + if path in duplicate_paths: + return () + else: + duplicate_paths.add(path) return ihook.pytest_collect_file(path=path, parent=self) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 03a9fe031..d360e2c8f 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -545,11 +545,24 @@ class Package(Module): proxy = self.config.hook return proxy - def _collectfile(self, path): + def _collectfile(self, path, handle_dupes=True): ihook = self.gethookproxy(path) if not self.isinitpath(path): if ihook.pytest_ignore_collect(path=path, config=self.config): return () + + if handle_dupes: + keepduplicates = self.config.getoption("keepduplicates") + if not keepduplicates: + duplicate_paths = self.config.pluginmanager._duplicatepaths + if path in duplicate_paths: + return () + else: + duplicate_paths.add(path) + + if self.fspath == path: # __init__.py + return [self] + return ihook.pytest_collect_file(path=path, parent=self) def isinitpath(self, path): diff --git a/testing/test_collection.py b/testing/test_collection.py index 3860cf9f9..dd8ecb1af 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -951,19 +951,35 @@ def test_collect_init_tests(testdir): result = testdir.runpytest(p, "--collect-only") result.stdout.fnmatch_lines( [ - "*", - "*", - "*", - "*", + "collected 2 items", + "", + " ", + " ", + " ", ] ) result = testdir.runpytest("./tests", "--collect-only") result.stdout.fnmatch_lines( [ - "*", - "*", - "*", - "*", + "collected 2 items", + "", + " ", + " ", + " ", + ] + ) + # Ignores duplicates with "." and pkginit (#4310). + result = testdir.runpytest("./tests", ".", "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + " ", + " ", ] ) result = testdir.runpytest("./tests/test_foo.py", "--collect-only") From 134b103605ffbe50ea5175cfeecb8d033458fbd0 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 7 Nov 2018 11:01:39 +0100 Subject: [PATCH 08/13] XXX: revert _collect_seen_pkgdirs --- src/_pytest/main.py | 7 +++---- testing/test_collection.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 46228f824..a2b27d9fa 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -387,7 +387,6 @@ class Session(nodes.FSCollector): self._initialpaths = frozenset() # Keep track of any collected nodes in here, so we don't duplicate fixtures self._node_cache = {} - self._collect_seen_pkgdirs = set() self.config.pluginmanager.register(self, name="session") @@ -505,7 +504,6 @@ class Session(nodes.FSCollector): if parent.isdir(): pkginit = parent.join("__init__.py") if pkginit.isfile(): - self._collect_seen_pkgdirs.add(parent) if pkginit in self._node_cache: root = self._node_cache[pkginit][0] else: @@ -531,12 +529,13 @@ class Session(nodes.FSCollector): def filter_(f): return f.check(file=1) + seen_dirs = set() for path in argpath.visit( fil=filter_, rec=self._recurse, bf=True, sort=True ): dirpath = path.dirpath() - if dirpath not in self._collect_seen_pkgdirs: - self._collect_seen_pkgdirs.add(dirpath) + if dirpath not in seen_dirs: + seen_dirs.add(dirpath) pkginit = dirpath.join("__init__.py") if pkginit.exists() and parts(pkginit.strpath).isdisjoint(paths): for x in root._collectfile(pkginit): diff --git a/testing/test_collection.py b/testing/test_collection.py index dd8ecb1af..c7cde090e 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -978,6 +978,19 @@ def test_collect_init_tests(testdir): "", " ", + "", + " ", + ] + ) + # XXX: Same as before, but different order. + result = testdir.runpytest(".", "tests", "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + "", " ", ] From f840521854d15288b57315c56b620001dd534d12 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 7 Nov 2018 19:29:55 +0100 Subject: [PATCH 09/13] harden test_collect_init_tests --- testing/test_collection.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/testing/test_collection.py b/testing/test_collection.py index c7cde090e..18033b9c0 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -975,31 +975,34 @@ def test_collect_init_tests(testdir): result.stdout.fnmatch_lines( [ "collected 2 items", - "", " ", " ", - "", - " ", + " ", + " ", ] ) - # XXX: Same as before, but different order. + # Same as before, but different order. result = testdir.runpytest(".", "tests", "--collect-only") result.stdout.fnmatch_lines( [ "collected 2 items", - "", " ", " ", - "", " ", ] ) result = testdir.runpytest("./tests/test_foo.py", "--collect-only") - result.stdout.fnmatch_lines(["*", "*"]) + result.stdout.fnmatch_lines( + ["", " ", " "] + ) assert "test_init" not in result.stdout.str() result = testdir.runpytest("./tests/__init__.py", "--collect-only") - result.stdout.fnmatch_lines(["*", "*"]) + result.stdout.fnmatch_lines( + ["", " ", " "] + ) assert "test_foo" not in result.stdout.str() From f8b944dee0b59bdc483ce505738ea1cb3a57a5b4 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 7 Nov 2018 19:33:22 +0100 Subject: [PATCH 10/13] pkg_roots --- src/_pytest/main.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index a2b27d9fa..59e2b6d4d 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -490,6 +490,7 @@ class Session(nodes.FSCollector): names = self._parsearg(arg) argpath = names.pop(0).realpath() paths = set() + pkg_roots = {} root = self # Start with a Session root, and delve to argpath item (dir or file) @@ -510,9 +511,9 @@ class Session(nodes.FSCollector): col = root._collectfile(pkginit, handle_dupes=False) if col: if isinstance(col[0], Package): - root = col[0] + pkg_roots[parent] = col[0] # always store a list in the cache, matchnodes expects it - self._node_cache[root.fspath] = [root] + self._node_cache[col[0].fspath] = [col[0]] # If it's a directory argument, recurse and look for any Subpackages. # Let the Package collector deal with subnodes, don't collect here. @@ -534,16 +535,19 @@ class Session(nodes.FSCollector): fil=filter_, rec=self._recurse, bf=True, sort=True ): dirpath = path.dirpath() + collect_root = pkg_roots.get(dirpath, root) if dirpath not in seen_dirs: seen_dirs.add(dirpath) pkginit = dirpath.join("__init__.py") if pkginit.exists() and parts(pkginit.strpath).isdisjoint(paths): - for x in root._collectfile(pkginit): + for x in collect_root._collectfile(pkginit): yield x + if isinstance(x, Package): + pkg_roots[dirpath] = x paths.add(x.fspath.dirpath()) - if parts(path.strpath).isdisjoint(paths): - for x in root._collectfile(path): + if True or parts(path.strpath).isdisjoint(paths): + for x in collect_root._collectfile(path): key = (type(x), x.fspath) if key in self._node_cache: yield self._node_cache[key] @@ -556,7 +560,8 @@ class Session(nodes.FSCollector): if argpath in self._node_cache: col = self._node_cache[argpath] else: - col = root._collectfile(argpath) + collect_root = pkg_roots.get(argpath.dirname, root) + col = collect_root._collectfile(argpath) if col: self._node_cache[argpath] = col m = self.matchnodes(col, names) From bbb9d72c1315e73d9acc6a9b8a9488479db76f0b Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 7 Nov 2018 19:36:19 +0100 Subject: [PATCH 11/13] remove paths/parts --- src/_pytest/main.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 59e2b6d4d..50bd10979 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -18,7 +18,6 @@ from _pytest.config import directory_arg from _pytest.config import hookimpl from _pytest.config import UsageError from _pytest.outcomes import exit -from _pytest.pathlib import parts from _pytest.runner import collect_one_node @@ -489,7 +488,6 @@ class Session(nodes.FSCollector): names = self._parsearg(arg) argpath = names.pop(0).realpath() - paths = set() pkg_roots = {} root = self @@ -539,21 +537,19 @@ class Session(nodes.FSCollector): if dirpath not in seen_dirs: seen_dirs.add(dirpath) pkginit = dirpath.join("__init__.py") - if pkginit.exists() and parts(pkginit.strpath).isdisjoint(paths): + if pkginit.exists(): for x in collect_root._collectfile(pkginit): yield x if isinstance(x, Package): pkg_roots[dirpath] = x - paths.add(x.fspath.dirpath()) - if True or parts(path.strpath).isdisjoint(paths): - for x in collect_root._collectfile(path): - key = (type(x), x.fspath) - if key in self._node_cache: - yield self._node_cache[key] - else: - self._node_cache[key] = x - yield x + for x in collect_root._collectfile(path): + key = (type(x), x.fspath) + if key in self._node_cache: + yield self._node_cache[key] + else: + self._node_cache[key] = x + yield x else: assert argpath.check(file=1) From 6fce1f0ac7a9a569429efa740f5cdd04805b4a6c Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 7 Nov 2018 20:06:35 +0100 Subject: [PATCH 12/13] pkg_roots per session --- src/_pytest/main.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 50bd10979..5d5d02a23 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -386,6 +386,7 @@ class Session(nodes.FSCollector): self._initialpaths = frozenset() # Keep track of any collected nodes in here, so we don't duplicate fixtures self._node_cache = {} + self._pkg_roots = {} self.config.pluginmanager.register(self, name="session") @@ -488,7 +489,6 @@ class Session(nodes.FSCollector): names = self._parsearg(arg) argpath = names.pop(0).realpath() - pkg_roots = {} root = self # Start with a Session root, and delve to argpath item (dir or file) @@ -509,7 +509,7 @@ class Session(nodes.FSCollector): col = root._collectfile(pkginit, handle_dupes=False) if col: if isinstance(col[0], Package): - pkg_roots[parent] = col[0] + self._pkg_roots[parent] = col[0] # always store a list in the cache, matchnodes expects it self._node_cache[col[0].fspath] = [col[0]] @@ -533,15 +533,25 @@ class Session(nodes.FSCollector): fil=filter_, rec=self._recurse, bf=True, sort=True ): dirpath = path.dirpath() - collect_root = pkg_roots.get(dirpath, root) + collect_root = self._pkg_roots.get(dirpath, root) if dirpath not in seen_dirs: + # Collect packages first. seen_dirs.add(dirpath) pkginit = dirpath.join("__init__.py") if pkginit.exists(): + got_pkg = False for x in collect_root._collectfile(pkginit): yield x if isinstance(x, Package): - pkg_roots[dirpath] = x + self._pkg_roots[dirpath] = x + got_pkg = True + if got_pkg: + continue + if path.basename == "__init__.py": + continue + + if dirpath in self._pkg_roots: + continue for x in collect_root._collectfile(path): key = (type(x), x.fspath) @@ -556,7 +566,7 @@ class Session(nodes.FSCollector): if argpath in self._node_cache: col = self._node_cache[argpath] else: - collect_root = pkg_roots.get(argpath.dirname, root) + collect_root = self._pkg_roots.get(argpath.dirname, root) col = collect_root._collectfile(argpath) if col: self._node_cache[argpath] = col From 827573c0494f58e7a2454633fe3d1584a8416325 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 7 Nov 2018 20:14:07 +0100 Subject: [PATCH 13/13] cleanup, TODO: use _node_cache --- src/_pytest/main.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 5d5d02a23..1de5f656f 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -490,7 +490,6 @@ class Session(nodes.FSCollector): names = self._parsearg(arg) argpath = names.pop(0).realpath() - root = self # Start with a Session root, and delve to argpath item (dir or file) # and stack all Packages found on the way. # No point in finding packages when collecting doctests @@ -503,10 +502,8 @@ class Session(nodes.FSCollector): if parent.isdir(): pkginit = parent.join("__init__.py") if pkginit.isfile(): - if pkginit in self._node_cache: - root = self._node_cache[pkginit][0] - else: - col = root._collectfile(pkginit, handle_dupes=False) + if pkginit not in self._node_cache: + col = self._collectfile(pkginit, handle_dupes=False) if col: if isinstance(col[0], Package): self._pkg_roots[parent] = col[0] @@ -533,27 +530,21 @@ class Session(nodes.FSCollector): fil=filter_, rec=self._recurse, bf=True, sort=True ): dirpath = path.dirpath() - collect_root = self._pkg_roots.get(dirpath, root) if dirpath not in seen_dirs: # Collect packages first. seen_dirs.add(dirpath) pkginit = dirpath.join("__init__.py") if pkginit.exists(): - got_pkg = False + collect_root = self._pkg_roots.get(dirpath, self) for x in collect_root._collectfile(pkginit): yield x if isinstance(x, Package): self._pkg_roots[dirpath] = x - got_pkg = True - if got_pkg: - continue - if path.basename == "__init__.py": - continue - if dirpath in self._pkg_roots: + # Do not collect packages here. continue - for x in collect_root._collectfile(path): + for x in self._collectfile(path): key = (type(x), x.fspath) if key in self._node_cache: yield self._node_cache[key] @@ -566,7 +557,7 @@ class Session(nodes.FSCollector): if argpath in self._node_cache: col = self._node_cache[argpath] else: - collect_root = self._pkg_roots.get(argpath.dirname, root) + collect_root = self._pkg_roots.get(argpath.dirname, self) col = collect_root._collectfile(argpath) if col: self._node_cache[argpath] = col