From b0c0911ba30dc69581fcbc012776401c1107d9e3 Mon Sep 17 00:00:00 2001 From: Bernardo Gomes Date: Sat, 27 Oct 2018 14:31:50 -0300 Subject: [PATCH 01/27] changed address to pytest-data-dir --- doc/en/fixture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/en/fixture.rst b/doc/en/fixture.rst index 65664c0b2..7d9af6fa8 100644 --- a/doc/en/fixture.rst +++ b/doc/en/fixture.rst @@ -153,7 +153,7 @@ This makes use of the automatic caching mechanisms of pytest. Another good approach is by adding the data files in the ``tests`` folder. There are also community plugins available to help managing this aspect of -testing, e.g. `pytest-datadir `__ +testing, e.g. `pytest-datadir `__ and `pytest-datafiles `__. .. _smtpshared: From c3acf049bd0424c455e7fcbecc3d6bf0b5d31c6a Mon Sep 17 00:00:00 2001 From: Palash Chatterjee Date: Sun, 28 Oct 2018 10:42:12 +0530 Subject: [PATCH 02/27] Fixes #4255 by adding to the documentation that module names are not regex-escaped --- changelog/4255.doc.rst | 1 + doc/en/reference.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog/4255.doc.rst diff --git a/changelog/4255.doc.rst b/changelog/4255.doc.rst new file mode 100644 index 000000000..673027cf5 --- /dev/null +++ b/changelog/4255.doc.rst @@ -0,0 +1 @@ +Added missing documentation about the fact that module names passed to filter warnings are not regex-escaped. diff --git a/doc/en/reference.rst b/doc/en/reference.rst index 632bb4e36..da53e7fea 100644 --- a/doc/en/reference.rst +++ b/doc/en/reference.rst @@ -117,6 +117,7 @@ Add warning filters to marked test items. A *warning specification string*, which is composed of contents of the tuple ``(action, message, category, module, lineno)`` as specified in `The Warnings filter `_ section of the Python documentation, separated by ``":"``. Optional fields can be omitted. + Module names passed for filtering are not regex-escaped. For example: From 8c475a45bbf345022237755a62517384741e289f Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 28 Oct 2018 16:43:17 -0700 Subject: [PATCH 03/27] Unrelated cleanups of source.py --- src/_pytest/_code/source.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 072ddb1b8..fa0ecaa69 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -14,8 +14,6 @@ from bisect import bisect_right import py import six -cpy_compile = compile - class Source(object): """ an immutable object holding a source code fragment, @@ -161,7 +159,7 @@ class Source(object): filename = base + "%r %s:%d>" % (filename, fn, lineno) source = "\n".join(self.lines) + "\n" try: - co = cpy_compile(source, filename, mode, flag) + co = compile(source, filename, mode, flag) except SyntaxError: ex = sys.exc_info()[1] # re-represent syntax errors from parsing python strings @@ -195,7 +193,7 @@ def compile_(source, filename=None, mode="exec", flags=0, dont_inherit=0): """ if isinstance(source, ast.AST): # XXX should Source support having AST? - return cpy_compile(source, filename, mode, flags, dont_inherit) + return compile(source, filename, mode, flags, dont_inherit) _genframe = sys._getframe(1) # the caller s = Source(source) co = s.compile(filename, mode, flags, _genframe=_genframe) @@ -290,7 +288,7 @@ def get_statement_startend2(lineno, node): def getstatementrange_ast(lineno, source, assertion=False, astnode=None): if astnode is None: content = str(source) - astnode = compile(content, "source", "exec", 1024) # 1024 for AST + astnode = compile(content, "source", "exec", _AST_FLAG) start, end = get_statement_startend2(lineno, astnode) # we need to correct the end: From 0d1f142b1cbc5b8379217895df098f2f21ea8fad Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 28 Oct 2018 16:44:34 -0700 Subject: [PATCH 04/27] Swallow warnings during anonymous compilation of source --- changelog/4260.bugfix.rst | 1 + src/_pytest/_code/source.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changelog/4260.bugfix.rst diff --git a/changelog/4260.bugfix.rst b/changelog/4260.bugfix.rst new file mode 100644 index 000000000..e1e1a009f --- /dev/null +++ b/changelog/4260.bugfix.rst @@ -0,0 +1 @@ +Swallow warnings during anonymous compilation of source. diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index fa0ecaa69..b74ecf88e 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -8,6 +8,7 @@ import linecache import sys import textwrap import tokenize +import warnings from ast import PyCF_ONLY_AST as _AST_FLAG from bisect import bisect_right @@ -288,7 +289,11 @@ def get_statement_startend2(lineno, node): def getstatementrange_ast(lineno, source, assertion=False, astnode=None): if astnode is None: content = str(source) - astnode = compile(content, "source", "exec", _AST_FLAG) + # See #4260: + # don't produce duplicate warnings when compiling source to find ast + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + astnode = compile(content, "source", "exec", _AST_FLAG) start, end = get_statement_startend2(lineno, astnode) # we need to correct the end: From 22ab737243387282acac2ebb735802ebe2042876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Mon, 29 Oct 2018 23:45:45 +0200 Subject: [PATCH 05/27] Spelling and grammar fixes --- CHANGELOG.rst | 2 +- doc/en/proposals/parametrize_with_fixtures.rst | 2 +- doc/en/writing_plugins.rst | 2 +- src/_pytest/capture.py | 2 +- src/_pytest/config/argparsing.py | 2 +- src/_pytest/fixtures.py | 2 +- src/_pytest/hookspec.py | 4 ++-- src/_pytest/mark/structures.py | 2 +- src/_pytest/pathlib.py | 10 +++++----- src/_pytest/python.py | 4 ++-- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 06ea24e23..6b06cbfb5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -541,7 +541,7 @@ Bug Fixes - `#3473 `_: Raise immediately if ``approx()`` is given an expected value of a type it doesn't understand (e.g. strings, nested dicts, etc.). -- `#3712 `_: Correctly represent the dimensions of an numpy array when calling ``repr()`` on ``approx()``. +- `#3712 `_: Correctly represent the dimensions of a numpy array when calling ``repr()`` on ``approx()``. - `#3742 `_: Fix incompatibility with third party plugins during collection, which produced the error ``object has no attribute '_collectfile'``. diff --git a/doc/en/proposals/parametrize_with_fixtures.rst b/doc/en/proposals/parametrize_with_fixtures.rst index 92e7993f3..b7295f27a 100644 --- a/doc/en/proposals/parametrize_with_fixtures.rst +++ b/doc/en/proposals/parametrize_with_fixtures.rst @@ -75,7 +75,7 @@ Issues ------ * By using ``request.getfuncargvalue()`` we rely on actual fixture function - execution to know what fixtures are involved, due to it's dynamic nature + execution to know what fixtures are involved, due to its dynamic nature * More importantly, ``request.getfuncargvalue()`` cannot be combined with parametrized fixtures, such as ``extra_context`` * This is very inconvenient if you wish to extend an existing test suite by diff --git a/doc/en/writing_plugins.rst b/doc/en/writing_plugins.rst index 027a087b4..464f8eb00 100644 --- a/doc/en/writing_plugins.rst +++ b/doc/en/writing_plugins.rst @@ -386,7 +386,7 @@ return a result object, with which we can assert the tests' outcomes. result.assert_outcomes(passed=4) -additionally it is possible to copy examples for a example folder before running pytest on it +additionally it is possible to copy examples for an example folder before running pytest on it .. code:: ini diff --git a/src/_pytest/capture.py b/src/_pytest/capture.py index bc50ccc3f..38f4292f9 100644 --- a/src/_pytest/capture.py +++ b/src/_pytest/capture.py @@ -124,7 +124,7 @@ class CaptureManager(object): def read_global_capture(self): return self._global_capturing.readouterr() - # Fixture Control (its just forwarding, think about removing this later) + # Fixture Control (it's just forwarding, think about removing this later) def activate_fixture(self, item): """If the current item is using ``capsys`` or ``capfd``, activate them so they take precedence over diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index 5012456b9..5b8306dda 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -152,7 +152,7 @@ class ArgumentError(Exception): class Argument(object): """class that mimics the necessary behaviour of optparse.Option - its currently a least effort implementation + it's currently a least effort implementation and ignoring choices and integer prefixes https://docs.python.org/3/library/optparse.html#optparse-standard-option-types """ diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 49c3402dc..3d57d9cf2 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -618,7 +618,7 @@ class FixtureRequest(FuncargnamesCompatAttr): subrequest._check_scope(argname, self.scope, scope) # clear sys.exc_info before invoking the fixture (python bug?) - # if its not explicitly cleared it will leak into the call + # if it's not explicitly cleared it will leak into the call exc_clear() try: # call the fixture function diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 27e55f0ea..18f23a50a 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -41,10 +41,10 @@ def pytest_namespace(): Plugins whose users depend on the current namespace functionality should prepare to migrate to a namespace they actually own. - To support the migration its suggested to trigger ``DeprecationWarnings`` for objects they put into the + To support the migration it's suggested to trigger ``DeprecationWarnings`` for objects they put into the pytest namespace. - An stopgap measure to avoid the warning is to monkeypatch the ``pytest`` module, but just as the + A stopgap measure to avoid the warning is to monkeypatch the ``pytest`` module, but just as the ``pytest_namespace`` hook this should be seen as a temporary measure to be removed in future versions after an appropriate transition period. """ diff --git a/src/_pytest/mark/structures.py b/src/_pytest/mark/structures.py index 2ca1d830a..b8fa011d1 100644 --- a/src/_pytest/mark/structures.py +++ b/src/_pytest/mark/structures.py @@ -443,7 +443,7 @@ class NodeKeywords(MappingMixin): @attr.s(cmp=False, hash=False) class NodeMarkers(object): """ - internal strucutre for storing marks belongong to a node + internal structure for storing marks belonging to a node ..warning:: diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 7cf3f40b6..5063c4651 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -36,7 +36,7 @@ get_lock_path = operator.methodcaller("joinpath", ".lock") def ensure_reset_dir(path): """ - ensures the given path is a empty directory + ensures the given path is an empty directory """ if path.exists(): rmtree(path, force=True) @@ -106,7 +106,7 @@ else: def _force_symlink(root, target, link_to): """helper to create the current symlink - its full of race conditions that are reasonably ok to ignore + it's full of race conditions that are reasonably ok to ignore for the contex of best effort linking to the latest testrun the presumption being thatin case of much parallelism @@ -124,7 +124,7 @@ def _force_symlink(root, target, link_to): def make_numbered_dir(root, prefix): - """create a directory with a increased number as suffix for the given prefix""" + """create a directory with an increased number as suffix for the given prefix""" for i in range(10): # try up to 10 times to create the folder max_existing = _max(map(parse_num, find_suffixes(root, prefix)), default=-1) @@ -164,7 +164,7 @@ def create_cleanup_lock(p): os.write(fd, spid) os.close(fd) if not lock_path.is_file(): - raise EnvironmentError("lock path got renamed after sucessfull creation") + raise EnvironmentError("lock path got renamed after successful creation") return lock_path @@ -221,7 +221,7 @@ def ensure_deletable(path, consider_lock_dead_if_created_before): def try_cleanup(path, consider_lock_dead_if_created_before): - """tries to cleanup a folder if we can ensure its deletable""" + """tries to cleanup a folder if we can ensure it's deletable""" if ensure_deletable(path, consider_lock_dead_if_created_before): maybe_delete_a_numbered_dir(path) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 6fd74acb1..58f95034d 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -661,7 +661,7 @@ class Instance(PyCollector): _ALLOW_MARKERS = False # hack, destroy later # instances share the object with their parents in a way # that duplicates markers instances if not taken out - # can be removed at node strucutre reorganization time + # can be removed at node structure reorganization time def _getobj(self): return self.parent.obj() @@ -1343,7 +1343,7 @@ class Function(FunctionMixin, nodes.Item, fixtures.FuncargnamesCompatAttr): """ _genid = None - # disable since functions handle it themselfes + # disable since functions handle it themselves _ALLOW_MARKERS = False def __init__( From a035c89ea7b7cacd0aa9fddcd4de7a166fa0e266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Tue, 30 Oct 2018 09:38:55 +0200 Subject: [PATCH 06/27] Spelling fix --- src/_pytest/pathlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 5063c4651..39f23b85b 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -107,7 +107,7 @@ def _force_symlink(root, target, link_to): """helper to create the current symlink it's full of race conditions that are reasonably ok to ignore - for the contex of best effort linking to the latest testrun + for the context of best effort linking to the latest testrun the presumption being thatin case of much parallelism the inaccuracy is going to be acceptable From b17e6cea21df2420c244f1a3ba61ecee69e15a14 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 30 Oct 2018 11:02:44 -0700 Subject: [PATCH 07/27] Upgrade pre-commit hooks --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15c82228a..670fb6c4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,12 +24,12 @@ repos: - id: flake8 language_version: python3 - repo: https://github.com/asottile/reorder_python_imports - rev: v1.3.2 + rev: v1.3.3 hooks: - id: reorder-python-imports args: ['--application-directories=.:src'] - repo: https://github.com/asottile/pyupgrade - rev: v1.8.0 + rev: v1.10.0 hooks: - id: pyupgrade args: [--keep-percent-format] From f20eeebde94f12424145f8548c960081d2e3c3ab Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 30 Oct 2018 13:31:55 -0300 Subject: [PATCH 08/27] Fix access denied error when deleting a stale temporary directory Fix #4262 --- changelog/4262.bugfix.rst | 1 + src/_pytest/pathlib.py | 24 +++++++++++++++------- testing/{test_paths.py => test_pathlib.py} | 18 ++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 changelog/4262.bugfix.rst rename testing/{test_paths.py => test_pathlib.py} (78%) diff --git a/changelog/4262.bugfix.rst b/changelog/4262.bugfix.rst new file mode 100644 index 000000000..1487138b7 --- /dev/null +++ b/changelog/4262.bugfix.rst @@ -0,0 +1 @@ +Fix access denied error when deleting stale directories created by ``tmpdir`` / ``tmp_path``. diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 7cf3f40b6..0ace312e9 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -186,19 +186,29 @@ def register_cleanup_lock_removal(lock_path, register=atexit.register): def maybe_delete_a_numbered_dir(path): - """removes a numbered directory if its lock can be obtained""" + """removes a numbered directory if its lock can be obtained and it does not seem to be in use""" + lock_path = None try: - create_cleanup_lock(path) + lock_path = create_cleanup_lock(path) + parent = path.parent + + garbage = parent.joinpath("garbage-{}".format(uuid.uuid4())) + path.rename(garbage) + rmtree(garbage, force=True) except (OSError, EnvironmentError): # known races: # * other process did a cleanup at the same time # * deletable folder was found + # * process cwd (Windows) return - parent = path.parent - - garbage = parent.joinpath("garbage-{}".format(uuid.uuid4())) - path.rename(garbage) - rmtree(garbage, force=True) + finally: + # if we created the lock, ensure we remove it even if we failed + # to properly remove the numbered dir + if lock_path is not None: + try: + lock_path.unlink() + except (OSError, IOError): + pass def ensure_deletable(path, consider_lock_dead_if_created_before): diff --git a/testing/test_paths.py b/testing/test_pathlib.py similarity index 78% rename from testing/test_paths.py rename to testing/test_pathlib.py index 65ee9b634..8ac404070 100644 --- a/testing/test_paths.py +++ b/testing/test_pathlib.py @@ -4,6 +4,9 @@ import py import pytest from _pytest.pathlib import fnmatch_ex +from _pytest.pathlib import get_lock_path +from _pytest.pathlib import maybe_delete_a_numbered_dir +from _pytest.pathlib import Path class TestPort: @@ -66,3 +69,18 @@ class TestPort: ) def test_not_matching(self, match, pattern, path): assert not match(pattern, path) + + +def test_access_denied_during_cleanup(tmp_path, monkeypatch): + """Ensure that deleting a numbered dir does not fail because of OSErrors (#4262).""" + path = tmp_path / "temp-1" + path.mkdir() + + def renamed_failed(*args): + raise OSError("access denied") + + monkeypatch.setattr(Path, "rename", renamed_failed) + + lock_path = get_lock_path(path) + maybe_delete_a_numbered_dir(path) + assert not lock_path.is_file() From 3b65d190a429e86dfc87204c1994402425ec0209 Mon Sep 17 00:00:00 2001 From: Brianna Laugher Date: Wed, 31 Oct 2018 23:45:09 +1100 Subject: [PATCH 09/27] Add docs page discussing flaky tests --- doc/en/contents.rst | 1 + doc/en/flaky.rst | 128 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 doc/en/flaky.rst diff --git a/doc/en/contents.rst b/doc/en/contents.rst index 58e2324ff..49bb67de3 100644 --- a/doc/en/contents.rst +++ b/doc/en/contents.rst @@ -37,6 +37,7 @@ Full pytest documentation customize example/index bash-completion + flaky backwards-compatibility deprecations diff --git a/doc/en/flaky.rst b/doc/en/flaky.rst new file mode 100644 index 000000000..c0f06a585 --- /dev/null +++ b/doc/en/flaky.rst @@ -0,0 +1,128 @@ +.. _flaky: + +Flaky tests +----------- + +A "flaky" test is one that exhibits intermittent or sporadic failure, that seems to have non-deterministic behaviour. Sometimes it passes, sometimes it fails, and it's not clear why. This page discusses pytest features that can help and other general strategies for identifying, fixing or mitigating them. + +Why flaky tests are a problem +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Flaky tests are particularly troublesome when a continuous integration (CI) server is being used, so that all tests must pass before a new code change can be merged. If the test result is not a reliable signal -- that a test failure means the code change broke the test -- developers can become mistrustful of the test results, which can lead to overlooking genuine failures. It is also a source of wasted time as developers must re-run test suites and investigate spurious failures. + + +Potential root causes +^^^^^^^^^^^^^^^^^^^^^ + +System state +~~~~~~~~~~~~ + +Broadly speaking, a flaky test indicates that the test relies on some system state that is not being appropriately controlled - the test environment is not sufficiently isolated. Higher level tests are more likely to be flaky as they rely on more state. + +Flaky tests sometimes appear when a test suite is run in parallel (such as use of pytest-xdist). This can indicate a test is reliant on test ordering. + +- Perhaps a different test is failing to clean up after itself and leaving behind data which causes the flaky test to fail. +- The flaky test is reliant on data from a previous test that doesn't clean up after itself, and in parallel runs that previous test is not always present +- Tests that modify global state typically cannot be run in parallel. + + +Overly strict assertion +~~~~~~~~~~~~~~~~~~~~~~~ + +Overly strict assertions can cause problems with floating point comparison as well as timing issues. `pytest.approx `_ is useful here. + + +Pytest features +^^^^^^^^^^^^^^^ + +Xfail strict +~~~~~~~~~~~~ + +:ref:`pytest.mark.xfail ref` with ``strict=False`` can be used to mark a test so that its failure does not cause the whole build to break. This could be considered like a manual quarantine, and is rather dangerous to use permanently. + + +PYTEST_CURRENT_TEST +~~~~~~~~~~~~~~~~~~~ + +:ref:`pytest current test env ref` may be useful for figuring out "which test got stuck". + + +Plugins +~~~~~~~ + +Rerunning any failed tests can mitigate the negative effects of flaky tests by giving them additional chances to pass, so that the overall build does not fail. Several pytest plugins support this: + +* `flaky `_ +* `pytest-flakefinder `_ - `blog post `_ +* `pytest-rerunfailures `_ +* `pytest-replay `_: This plugin helps to reproduce locally crashes or flaky tests observed during CI runs. + +Plugins to deliberately randomize tests can help expose tests with state problems: + +* `pytest-random-order `_ +* `pytest-randomly `_ + + +Other general strategies +^^^^^^^^^^^^^^^^^^^^^^^^ + +Split up test suites +~~~~~~~~~~~~~~~~~~~~ + +It can be common to split a single test suite into two, such as unit vs integration, and only use the unit test suite as a CI gate. This also helps keep build times manageable as high level tests tend to be slower. However, it means it does become possible for code that breaks the build to be merged, so extra vigilance is needed for monitoring the integration test results. + + +Video/screenshot on failure +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For UI tests these are important for understanding what the state of the UI was when the test failed. pytest-splinter can be used with plugins like pytest-bdd and can `save a screenshot on test failure `_, which can help to isolate the cause. + + +Delete or rewrite the test +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the functionality is covered by other tests, perhaps the test can be removed. If not, perhaps it can be rewritten at a lower level which will remove the flakiness or make its source more apparent. + + +Quarantine +~~~~~~~~~~ + +Mark Lapierre discusses the `Pros and Cons of Quarantined Tests `_ in a post from 2018. + + + +CI tools that rerun on failure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Azure Pipelines (the Azure cloud CI/CD tool, formerly Visual Studio Team Services or VSTS) has a feature to `identify flaky tests `_ and rerun failed tests. + + + +Research +^^^^^^^^ + +This is a limited list, please submit an issue or pull request to expand it! + +* Gao, Zebao, Yalan Liang, Myra B. Cohen, Atif M. Memon, and Zhen Wang. "Making system user interactive tests repeatable: When and what should we control?." In *Software Engineering (ICSE), 2015 IEEE/ACM 37th IEEE International Conference on*, vol. 1, pp. 55-65. IEEE, 2015. `PDF `_ +* Palomba, Fabio, and Andy Zaidman. "Does refactoring of test smells induce fixing flaky tests?." In *Software Maintenance and Evolution (ICSME), 2017 IEEE International Conference on*, pp. 1-12. IEEE, 2017. `PDF in Google Drive `_ +* Bell, Jonathan, Owolabi Legunsen, Michael Hilton, Lamyaa Eloussi, Tifany Yung, and Darko Marinov. "DeFlaker: Automatically detecting flaky tests." In *Proceedings of the 2018 International Conference on Software Engineering*. 2018. `PDF `_ + + +Resources +^^^^^^^^^ + +* `Eradicating Non-Determinism in Tests `_ by Martin Fowler, 2011 +* `No more flaky tests on the Go team `_ by Pavan Sudarshan, 2012 +* `The Build That Cried Broken: Building Trust in your Continuous Integration Tests `_ talk (video) by `Angie Jones `_ at SeleniumConf Austin 2017 +* `Test and Code Podcast: Flaky Tests and How to Deal with Them `_ by Brian Okken and Anthony Shaw, 2018 +* Microsoft: + + * `How we approach testing VSTS to enable continuous delivery `_ by Brian Harry MS, 2017 + * `Eliminating Flaky Tests `_ blog and talk (video) by Munil Shah, 2017 + +* Google: + + * `Flaky Tests at Google and How We Mitigate Them `_ by John Micco, 2016 + * `Where do Google's flaky tests come from? `_ by Jeff Listfield, 2017 + + From d5b5be6fbe65a7f6dc5457feaa5d8b6e5affa81e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 31 Oct 2018 10:44:43 -0300 Subject: [PATCH 10/27] Fix linting --- doc/en/contents.rst | 2 +- doc/en/example/multipython.py | 4 ++-- doc/en/flaky.rst | 17 ++++++++--------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/doc/en/contents.rst b/doc/en/contents.rst index 49bb67de3..9883eaa64 100644 --- a/doc/en/contents.rst +++ b/doc/en/contents.rst @@ -33,11 +33,11 @@ Full pytest documentation reference goodpractices + flaky pythonpath customize example/index bash-completion - flaky backwards-compatibility deprecations diff --git a/doc/en/example/multipython.py b/doc/en/example/multipython.py index 21fdf6fc2..a5360ed5a 100644 --- a/doc/en/example/multipython.py +++ b/doc/en/example/multipython.py @@ -33,7 +33,7 @@ class Python(object): dumpfile = self.picklefile.dirpath("dump.py") dumpfile.write( textwrap.dedent( - """\ + r"""\ import pickle f = open({!r}, 'wb') s = pickle.dump({!r}, f, protocol=2) @@ -49,7 +49,7 @@ class Python(object): loadfile = self.picklefile.dirpath("load.py") loadfile.write( textwrap.dedent( - """\ + r"""\ import pickle f = open({!r}, 'rb') obj = pickle.load(f) diff --git a/doc/en/flaky.rst b/doc/en/flaky.rst index c0f06a585..734ca1cde 100644 --- a/doc/en/flaky.rst +++ b/doc/en/flaky.rst @@ -1,4 +1,3 @@ -.. _flaky: Flaky tests ----------- @@ -25,7 +24,7 @@ Flaky tests sometimes appear when a test suite is run in parallel (such as use o - The flaky test is reliant on data from a previous test that doesn't clean up after itself, and in parallel runs that previous test is not always present - Tests that modify global state typically cannot be run in parallel. - + Overly strict assertion ~~~~~~~~~~~~~~~~~~~~~~~ @@ -44,7 +43,7 @@ Xfail strict PYTEST_CURRENT_TEST ~~~~~~~~~~~~~~~~~~~ -:ref:`pytest current test env ref` may be useful for figuring out "which test got stuck". +:ref:`pytest current test env` may be useful for figuring out "which test got stuck". Plugins @@ -62,7 +61,7 @@ Plugins to deliberately randomize tests can help expose tests with state problem * `pytest-random-order `_ * `pytest-randomly `_ - + Other general strategies ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -81,7 +80,7 @@ For UI tests these are important for understanding what the state of the UI was Delete or rewrite the test ~~~~~~~~~~~~~~~~~~~~~~~~~~ -If the functionality is covered by other tests, perhaps the test can be removed. If not, perhaps it can be rewritten at a lower level which will remove the flakiness or make its source more apparent. +If the functionality is covered by other tests, perhaps the test can be removed. If not, perhaps it can be rewritten at a lower level which will remove the flakiness or make its source more apparent. Quarantine @@ -103,9 +102,9 @@ Research This is a limited list, please submit an issue or pull request to expand it! -* Gao, Zebao, Yalan Liang, Myra B. Cohen, Atif M. Memon, and Zhen Wang. "Making system user interactive tests repeatable: When and what should we control?." In *Software Engineering (ICSE), 2015 IEEE/ACM 37th IEEE International Conference on*, vol. 1, pp. 55-65. IEEE, 2015. `PDF `_ -* Palomba, Fabio, and Andy Zaidman. "Does refactoring of test smells induce fixing flaky tests?." In *Software Maintenance and Evolution (ICSME), 2017 IEEE International Conference on*, pp. 1-12. IEEE, 2017. `PDF in Google Drive `_ -* Bell, Jonathan, Owolabi Legunsen, Michael Hilton, Lamyaa Eloussi, Tifany Yung, and Darko Marinov. "DeFlaker: Automatically detecting flaky tests." In *Proceedings of the 2018 International Conference on Software Engineering*. 2018. `PDF `_ +* Gao, Zebao, Yalan Liang, Myra B. Cohen, Atif M. Memon, and Zhen Wang. "Making system user interactive tests repeatable: When and what should we control?." In *Software Engineering (ICSE), 2015 IEEE/ACM 37th IEEE International Conference on*, vol. 1, pp. 55-65. IEEE, 2015. `PDF `__ +* Palomba, Fabio, and Andy Zaidman. "Does refactoring of test smells induce fixing flaky tests?." In *Software Maintenance and Evolution (ICSME), 2017 IEEE International Conference on*, pp. 1-12. IEEE, 2017. `PDF in Google Drive `__ +* Bell, Jonathan, Owolabi Legunsen, Michael Hilton, Lamyaa Eloussi, Tifany Yung, and Darko Marinov. "DeFlaker: Automatically detecting flaky tests." In *Proceedings of the 2018 International Conference on Software Engineering*. 2018. `PDF `__ Resources @@ -124,5 +123,5 @@ Resources * `Flaky Tests at Google and How We Mitigate Them `_ by John Micco, 2016 * `Where do Google's flaky tests come from? `_ by Jeff Listfield, 2017 - + From da04ff52e4bfeeb604f52b44d9031b3dc74c7004 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 5 Oct 2018 11:51:36 +0200 Subject: [PATCH 11/27] ignore _CompatProperty when parsing fixtures this avoid triggering the warnings when parsing the session node as session plugin --- changelog/2701.bugfix.rst | 1 + src/_pytest/fixtures.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog/2701.bugfix.rst diff --git a/changelog/2701.bugfix.rst b/changelog/2701.bugfix.rst new file mode 100644 index 000000000..af16a08ad --- /dev/null +++ b/changelog/2701.bugfix.rst @@ -0,0 +1 @@ +avoid "RemovedInPytest4Warning: usage of Session... is deprecated, please use pytest... instead" warnings.:w diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 49c3402dc..26a28ca80 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -1193,6 +1193,7 @@ class FixtureManager(object): nodeid = p.dirpath().relto(self.config.rootdir) if p.sep != nodes.SEP: nodeid = nodeid.replace(p.sep, nodes.SEP) + self.parsefactories(plugin, nodeid) def _getautousenames(self, nodeid): @@ -1297,11 +1298,18 @@ class FixtureManager(object): nodeid = node_or_obj.nodeid if holderobj in self._holderobjseen: return + + from _pytest.nodes import _CompatProperty + self._holderobjseen.add(holderobj) autousenames = [] for name in dir(holderobj): # The attribute can be an arbitrary descriptor, so the attribute # access below can raise. safe_getatt() ignores such exceptions. + maybe_property = safe_getattr(type(holderobj), name, None) + if isinstance(maybe_property, _CompatProperty): + # deprecated + continue obj = safe_getattr(holderobj, name, None) marker = getfixturemarker(obj) # fixture functions have a pytest_funcarg__ prefix (pre-2.3 style) From cc252569823f9ecf8399d1f9a281c43350464237 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 31 Oct 2018 11:05:58 -0300 Subject: [PATCH 12/27] Fix linting2 --- doc/en/flaky.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/en/flaky.rst b/doc/en/flaky.rst index 734ca1cde..8e340316e 100644 --- a/doc/en/flaky.rst +++ b/doc/en/flaky.rst @@ -123,5 +123,3 @@ Resources * `Flaky Tests at Google and How We Mitigate Them `_ by John Micco, 2016 * `Where do Google's flaky tests come from? `_ by Jeff Listfield, 2017 - - From b5d62cdb553484267a005707c94faf0287433f5e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 31 Oct 2018 11:07:24 -0300 Subject: [PATCH 13/27] Update 2701.bugfix.rst --- changelog/2701.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/2701.bugfix.rst b/changelog/2701.bugfix.rst index af16a08ad..a942234fd 100644 --- a/changelog/2701.bugfix.rst +++ b/changelog/2701.bugfix.rst @@ -1 +1 @@ -avoid "RemovedInPytest4Warning: usage of Session... is deprecated, please use pytest... instead" warnings.:w +Fix false ``RemovedInPytest4Warning: usage of Session... is deprecated, please use pytest`` warnings. From 5404246e64aa61f0f61e4825fe1cd3de963d4262 Mon Sep 17 00:00:00 2001 From: William Jamir Silva Date: Wed, 31 Oct 2018 14:22:42 -0300 Subject: [PATCH 14/27] Improve the warning message for the implicitly str conversion Signed-off-by: William Jamir Silva --- src/_pytest/monkeypatch.py | 8 +++++--- testing/test_monkeypatch.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/_pytest/monkeypatch.py b/src/_pytest/monkeypatch.py index 2efdb73ae..d536b7746 100644 --- a/src/_pytest/monkeypatch.py +++ b/src/_pytest/monkeypatch.py @@ -230,10 +230,12 @@ class MonkeyPatch(object): if not isinstance(value, str): warnings.warn( pytest.PytestWarning( - "Environment variable value {!r} should be str, converted to str implicitly".format( - value + "Value of environment variable {name} type should be str, but got " + "{value!r} (type: {type}); converted to str implicitly".format( + name=name, value=value, type=type(value).__name__ ) - ) + ), + stacklevel=2, ) value = str(value) if prepend and name in os.environ: diff --git a/testing/test_monkeypatch.py b/testing/test_monkeypatch.py index d250d24e7..b81a656ea 100644 --- a/testing/test_monkeypatch.py +++ b/testing/test_monkeypatch.py @@ -3,6 +3,7 @@ from __future__ import division from __future__ import print_function import os +import re import sys import textwrap @@ -226,9 +227,10 @@ class TestEnvironWarnings(object): def test_setenv_non_str_warning(self, monkeypatch): value = 2 msg = ( - "Environment variable value {!r} should be str, converted to str implicitly" + "Value of environment variable PYTEST_INTERNAL_MY_VAR type should be str, " + "but got 2 (type: int); converted to str implicitly" ) - with pytest.warns(pytest.PytestWarning, match=msg.format(value)): + with pytest.warns(pytest.PytestWarning, match=re.escape(msg)): monkeypatch.setenv(str(self.VAR_NAME), value) @@ -337,7 +339,7 @@ def test_importerror(testdir): ) testdir.tmpdir.join("test_importerror.py").write( textwrap.dedent( - """\ + r"""\ def test_importerror(monkeypatch): monkeypatch.setattr('package.a.x', 2) """ From 9b94313b4446a683ad0efaeea1a30210bab419b4 Mon Sep 17 00:00:00 2001 From: William Jamir Silva Date: Wed, 31 Oct 2018 17:12:08 -0300 Subject: [PATCH 15/27] Update changelog --- changelog/4279.trivial.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4279.trivial.rst diff --git a/changelog/4279.trivial.rst b/changelog/4279.trivial.rst new file mode 100644 index 000000000..8a8275049 --- /dev/null +++ b/changelog/4279.trivial.rst @@ -0,0 +1 @@ +Improve message and stack level of warnings issued by ``monkeypatch.setenv`` when the value of the environment variable is not a ``str``. \ No newline at end of file From af00367fed5ab25d72065a768b75667730f6e74b Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Wed, 31 Oct 2018 13:26:49 -0700 Subject: [PATCH 16/27] Upgrade pyupgrade for crlf fixes (again) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 670fb6c4b..ecfc004ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: reorder-python-imports args: ['--application-directories=.:src'] - repo: https://github.com/asottile/pyupgrade - rev: v1.10.0 + rev: v1.10.1 hooks: - id: pyupgrade args: [--keep-percent-format] From d4ca634ef6961ae42eb476f4500aed594e1007b1 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 31 Oct 2018 18:21:55 -0300 Subject: [PATCH 17/27] Fix linting --- changelog/4279.trivial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/4279.trivial.rst b/changelog/4279.trivial.rst index 8a8275049..9f4c4c473 100644 --- a/changelog/4279.trivial.rst +++ b/changelog/4279.trivial.rst @@ -1 +1 @@ -Improve message and stack level of warnings issued by ``monkeypatch.setenv`` when the value of the environment variable is not a ``str``. \ No newline at end of file +Improve message and stack level of warnings issued by ``monkeypatch.setenv`` when the value of the environment variable is not a ``str``. From c31abb117694f54e5b1b86d0d1d2f62ea33d05bf Mon Sep 17 00:00:00 2001 From: Andreu Vallbona Plazas Date: Wed, 31 Oct 2018 23:06:44 +0100 Subject: [PATCH 18/27] Update talks.rst Added the slides of a PyconES 2017 talk about pytest and its plugins ecosystem. --- doc/en/talks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/en/talks.rst b/doc/en/talks.rst index c4310b522..eef3f7e10 100644 --- a/doc/en/talks.rst +++ b/doc/en/talks.rst @@ -69,6 +69,8 @@ Talks and blog postings - `monkey patching done right`_ (blog post, consult `monkeypatch plugin`_ for up-to-date API) +- pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyconES 2017 (`slides `_, `video in spanish `_) + Test parametrization: - `generating parametrized tests with fixtures`_. From 0994829afeccbdc590deca3e93cccc2f10916712 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 31 Oct 2018 19:35:47 -0300 Subject: [PATCH 19/27] Move pytest talk to the start of the section --- doc/en/talks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/en/talks.rst b/doc/en/talks.rst index eef3f7e10..639c07287 100644 --- a/doc/en/talks.rst +++ b/doc/en/talks.rst @@ -23,6 +23,8 @@ Books Talks and blog postings --------------------------------------------- +- pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyconES 2017 (`slides `_, `video in spanish `_) + - `Pythonic testing, Igor Starikov (Russian, PyNsk, November 2016) `_. @@ -69,8 +71,6 @@ Talks and blog postings - `monkey patching done right`_ (blog post, consult `monkeypatch plugin`_ for up-to-date API) -- pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyconES 2017 (`slides `_, `video in spanish `_) - Test parametrization: - `generating parametrized tests with fixtures`_. From 9871d5ec2d41bd4df4cfb793dda543b3604628f3 Mon Sep 17 00:00:00 2001 From: Andreu Vallbona Plazas Date: Thu, 1 Nov 2018 01:24:18 +0100 Subject: [PATCH 20/27] Updated the talks.rst corrected the target name --- doc/en/talks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/en/talks.rst b/doc/en/talks.rst index 639c07287..aa1fb00e7 100644 --- a/doc/en/talks.rst +++ b/doc/en/talks.rst @@ -23,7 +23,7 @@ Books Talks and blog postings --------------------------------------------- -- pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyconES 2017 (`slides `_, `video in spanish `_) +- pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyconES 2017 (`slides in english `_, `video in spanish `_) - `Pythonic testing, Igor Starikov (Russian, PyNsk, November 2016) `_. From 948fd7b8b027b944cc817161637735235e89636a Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 1 Nov 2018 08:40:35 -0700 Subject: [PATCH 21/27] fixup pyupgrade crlf incorrect fixes --- testing/test_monkeypatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_monkeypatch.py b/testing/test_monkeypatch.py index b81a656ea..ebc233fbf 100644 --- a/testing/test_monkeypatch.py +++ b/testing/test_monkeypatch.py @@ -339,7 +339,7 @@ def test_importerror(testdir): ) testdir.tmpdir.join("test_importerror.py").write( textwrap.dedent( - r"""\ + """\ def test_importerror(monkeypatch): monkeypatch.setattr('package.a.x', 2) """ From d65f300988be1b1cfc0a98254da4aa8c75fe45e2 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 25 Oct 2018 20:03:14 +0200 Subject: [PATCH 22/27] Move handling of duplicate files This removes the hack added in https://github.com/pytest-dev/pytest/pull/3802. Adjusts test: - it appears to not have been changed to 7 intentionally. - removes XXX comment, likely not relevant anymore since 6dac7743. --- src/_pytest/main.py | 19 ++++++++++--------- src/_pytest/python.py | 9 --------- testing/test_session.py | 2 +- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index f27270f26..7e5d096a5 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -281,15 +281,6 @@ def pytest_ignore_collect(path, config): if _in_venv(path) and not allow_in_venv: return True - # Skip duplicate paths. - keepduplicates = config.getoption("keepduplicates") - duplicate_paths = config.pluginmanager._duplicatepaths - if not keepduplicates: - if path in duplicate_paths: - return True - else: - duplicate_paths.add(path) - return False @@ -559,6 +550,16 @@ class Session(nodes.FSCollector): if not self.isinitpath(path): if ihook.pytest_ignore_collect(path=path, config=self.config): return () + + # Skip duplicate paths. + keepduplicates = self.config.getoption("keepduplicates") + if not keepduplicates: + duplicate_paths = self.config.pluginmanager._duplicatepaths + if path in duplicate_paths: + return () + else: + duplicate_paths.add(path) + return ihook.pytest_collect_file(path=path, parent=self) def _recurse(self, path): diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 58f95034d..414eabec6 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -552,15 +552,6 @@ class Package(Module): return path in self.session._initialpaths def collect(self): - # XXX: HACK! - # Before starting to collect any files from this package we need - # to cleanup the duplicate paths added by the session's collect(). - # Proper fix is to not track these as duplicates in the first place. - for path in list(self.session.config.pluginmanager._duplicatepaths): - # if path.parts()[:len(self.fspath.dirpath().parts())] == self.fspath.dirpath().parts(): - if path.dirname.startswith(self.name): - self.session.config.pluginmanager._duplicatepaths.remove(path) - this_path = self.fspath.dirpath() init_module = this_path.join("__init__.py") if init_module.check(file=1) and path_matches_patterns( diff --git a/testing/test_session.py b/testing/test_session.py index 6225a2c0d..c1785b916 100644 --- a/testing/test_session.py +++ b/testing/test_session.py @@ -219,7 +219,7 @@ class TestNewSession(SessionTests): started = reprec.getcalls("pytest_collectstart") finished = reprec.getreports("pytest_collectreport") assert len(started) == len(finished) - assert len(started) == 7 # XXX extra TopCollector + assert len(started) == 8 colfail = [x for x in finished if x.failed] assert len(colfail) == 1 From 70976b04be195b6ae6fc9a74811429148479a83c Mon Sep 17 00:00:00 2001 From: Mick Koch Date: Wed, 31 Oct 2018 14:18:12 -0400 Subject: [PATCH 23/27] Add test for __init__.py collection with package directory as argument --- testing/test_collection.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/testing/test_collection.py b/testing/test_collection.py index 7f6791dae..58f4afb38 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -957,6 +957,15 @@ def test_collect_init_tests(testdir): "*", ] ) + result = testdir.runpytest("./tests", "--collect-only") + result.stdout.fnmatch_lines( + [ + "*", + "*", + "*", + "*", + ] + ) def test_collect_invalid_signature_message(testdir): From 320e41b142f14e5b864131e30f4c18bdf29199c6 Mon Sep 17 00:00:00 2001 From: Mick Koch Date: Thu, 1 Nov 2018 08:20:01 -0400 Subject: [PATCH 24/27] Add failing test for __init__.py also including other package files --- testing/test_collection.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testing/test_collection.py b/testing/test_collection.py index 58f4afb38..06f8d40ee 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -966,6 +966,12 @@ def test_collect_init_tests(testdir): "*", ] ) + result = testdir.runpytest("./tests/test_foo.py", "--collect-only") + result.stdout.fnmatch_lines(["*", "*"]) + assert "test_init" not in result.stdout.str() + result = testdir.runpytest("./tests/__init__.py", "--collect-only") + result.stdout.fnmatch_lines(["*", "*"]) + assert "test_foo" not in result.stdout.str() def test_collect_invalid_signature_message(testdir): From 5ac4eff09b8514a5b46bdff464605a60051abc83 Mon Sep 17 00:00:00 2001 From: Mick Koch Date: Thu, 1 Nov 2018 08:20:57 -0400 Subject: [PATCH 25/27] Fix __init__.py as argument also including other package files --- src/_pytest/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 7e5d096a5..2bb405081 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -542,7 +542,15 @@ class Session(nodes.FSCollector): col = root._collectfile(argpath) if col: self._node_cache[argpath] = col - for y in self.matchnodes(col, names): + m = self.matchnodes(col, names) + # If __init__.py was the only file requested, then the matched node will be + # the corresponding Package, and the first yielded item will be the __init__ + # Module itself, so just use that. If this special case isn't taken, then all + # the files in the package will be yielded. + if argpath.basename == "__init__.py": + yield next(m[0].collect()) + return + for y in m: yield y def _collectfile(self, path): From 51973543754b0f1149fcf36298bb47e82ccc9ead Mon Sep 17 00:00:00 2001 From: Mick Koch Date: Thu, 1 Nov 2018 08:51:51 -0400 Subject: [PATCH 26/27] Add changelog entry --- changelog/4046.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4046.bugfix.rst diff --git a/changelog/4046.bugfix.rst b/changelog/4046.bugfix.rst new file mode 100644 index 000000000..2b0da70cd --- /dev/null +++ b/changelog/4046.bugfix.rst @@ -0,0 +1 @@ +Fix problems with running tests in package ``__init__.py`` files. From e30f7094f38fa12196c08862be00cfae418b4bf6 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 1 Nov 2018 00:54:48 +0100 Subject: [PATCH 27/27] python: collect: ignore exceptions with isinstance Fixes https://github.com/pytest-dev/pytest/issues/4266. --- doc/4266.bugfix.rst | 1 + src/_pytest/compat.py | 8 ++++++++ src/_pytest/python.py | 3 ++- testing/python/raises.py | 24 ++++++++++++++++++++++++ testing/test_collection.py | 27 +++++++++++++++++++++++++++ testing/test_compat.py | 12 ++++++++++++ 6 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 doc/4266.bugfix.rst diff --git a/doc/4266.bugfix.rst b/doc/4266.bugfix.rst new file mode 100644 index 000000000..f19a7cc1f --- /dev/null +++ b/doc/4266.bugfix.rst @@ -0,0 +1 @@ +Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class. diff --git a/src/_pytest/compat.py b/src/_pytest/compat.py index 4af0a2339..2b2cf659f 100644 --- a/src/_pytest/compat.py +++ b/src/_pytest/compat.py @@ -334,6 +334,14 @@ def safe_getattr(object, name, default): return default +def safe_isclass(obj): + """Ignore any exception via isinstance on Python 3.""" + try: + return isclass(obj) + except Exception: + return False + + def _is_unittest_unexpected_success_a_failure(): """Return if the test suite should fail if an @expectedFailure unittest test PASSES. diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 58f95034d..ec186bde1 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -33,6 +33,7 @@ from _pytest.compat import NoneType from _pytest.compat import NOTSET from _pytest.compat import REGEX_TYPE from _pytest.compat import safe_getattr +from _pytest.compat import safe_isclass from _pytest.compat import safe_str from _pytest.compat import STRING_TYPES from _pytest.config import hookimpl @@ -195,7 +196,7 @@ def pytest_pycollect_makeitem(collector, name, obj): if res is not None: return # nothing was collected elsewhere, let's do it here - if isclass(obj): + if safe_isclass(obj): if collector.istestclass(obj, name): Class = collector._getcustomclass("Class") outcome.force_result(Class(name, parent=collector)) diff --git a/testing/python/raises.py b/testing/python/raises.py index 9b9eadf1a..a72aeef63 100644 --- a/testing/python/raises.py +++ b/testing/python/raises.py @@ -1,5 +1,7 @@ import sys +import six + import pytest from _pytest.outcomes import Failed @@ -170,3 +172,25 @@ class TestRaises(object): Failed, match="DID NOT RAISE " ): pytest.raises(ClassLooksIterableException, lambda: None) + + def test_raises_with_raising_dunder_class(self): + """Test current behavior with regard to exceptions via __class__ (#4284).""" + + class CrappyClass(Exception): + @property + def __class__(self): + assert False, "via __class__" + + if six.PY2: + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.raises(CrappyClass()): + pass + assert "DID NOT RAISE" in excinfo.value.args[0] + + with pytest.raises(CrappyClass) as excinfo: + raise CrappyClass() + else: + with pytest.raises(AssertionError) as excinfo: + with pytest.raises(CrappyClass()): + pass + assert "via __class__" in excinfo.value.args[0] diff --git a/testing/test_collection.py b/testing/test_collection.py index 7f6791dae..751f484aa 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -977,3 +977,30 @@ def test_collect_invalid_signature_message(testdir): result.stdout.fnmatch_lines( ["Could not determine arguments of *.fix *: invalid method signature"] ) + + +def test_collect_handles_raising_on_dunder_class(testdir): + """Handle proxy classes like Django's LazySettings that might raise on + ``isinstance`` (#4266). + """ + testdir.makepyfile( + """ + class ImproperlyConfigured(Exception): + pass + + class RaisesOnGetAttr(object): + def raises(self): + raise ImproperlyConfigured + + __class__ = property(raises) + + raises = RaisesOnGetAttr() + + + def test_1(): + pass + """ + ) + result = testdir.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed in*"]) diff --git a/testing/test_compat.py b/testing/test_compat.py index 494acd738..79224deef 100644 --- a/testing/test_compat.py +++ b/testing/test_compat.py @@ -12,6 +12,7 @@ from _pytest.compat import _PytestWrapper from _pytest.compat import get_real_func from _pytest.compat import is_generator from _pytest.compat import safe_getattr +from _pytest.compat import safe_isclass from _pytest.outcomes import OutcomeException @@ -140,3 +141,14 @@ def test_safe_getattr(): helper = ErrorsHelper() assert safe_getattr(helper, "raise_exception", "default") == "default" assert safe_getattr(helper, "raise_fail", "default") == "default" + + +def test_safe_isclass(): + assert safe_isclass(type) is True + + class CrappyClass(Exception): + @property + def __class__(self): + assert False, "Should be ignored" + + assert safe_isclass(CrappyClass()) is False