diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 27dadb328..0ac1b8102 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -5,7 +5,12 @@ the name cache was not chosen to ensure pluggy automatically ignores the external pytest-cache """ from __future__ import absolute_import, division, print_function + +from collections import OrderedDict + import py +import six + import pytest import json import os @@ -168,6 +173,39 @@ class LFPlugin(object): config.cache.set("cache/lastfailed", self.lastfailed) +class NFPlugin(object): + """ Plugin which implements the --nf (run new-first) option """ + + def __init__(self, config): + self.config = config + self.active = config.option.newfirst + self.cached_nodeids = config.cache.get("cache/nodeids", []) + + def pytest_collection_modifyitems(self, session, config, items): + if self.active: + new_items = OrderedDict() + other_items = OrderedDict() + for item in items: + if item.nodeid not in self.cached_nodeids: + new_items[item.nodeid] = item + else: + other_items[item.nodeid] = item + + items[:] = self._get_increasing_order(six.itervalues(new_items)) + \ + self._get_increasing_order(six.itervalues(other_items)) + self.cached_nodeids = [x.nodeid for x in items if isinstance(x, pytest.Item)] + + def _get_increasing_order(self, items): + return sorted(items, key=lambda item: item.fspath.mtime(), reverse=True) + + def pytest_sessionfinish(self, session): + config = self.config + if config.getoption("cacheshow") or hasattr(config, "slaveinput"): + return + + config.cache.set("cache/nodeids", self.cached_nodeids) + + def pytest_addoption(parser): group = parser.getgroup("general") group.addoption( @@ -179,6 +217,10 @@ def pytest_addoption(parser): help="run all tests but run the last failures first. " "This may re-order tests and thus lead to " "repeated fixture setup/teardown") + group.addoption( + '--nf', '--new-first', action='store_true', dest="newfirst", + help="run tests from new files first, then the rest of the tests " + "sorted by file mtime") group.addoption( '--cache-show', action='store_true', dest="cacheshow", help="show cache contents, don't perform collection or tests") @@ -200,6 +242,7 @@ def pytest_cmdline_main(config): def pytest_configure(config): config.cache = Cache(config) config.pluginmanager.register(LFPlugin(config), "lfplugin") + config.pluginmanager.register(NFPlugin(config), "nfplugin") @pytest.fixture diff --git a/changelog/3034.feature b/changelog/3034.feature new file mode 100644 index 000000000..12330cdd6 --- /dev/null +++ b/changelog/3034.feature @@ -0,0 +1 @@ +New ``--nf``, ``--new-first`` options: run new tests first followed by the rest of the tests, in both cases tests are also sorted by the file modified time, with more recent files coming first. diff --git a/doc/en/cache.rst b/doc/en/cache.rst index e3423e95b..db72249f9 100644 --- a/doc/en/cache.rst +++ b/doc/en/cache.rst @@ -152,6 +152,10 @@ of ``FF`` and dots):: .. _`config.cache`: +New ``--nf``, ``--new-first`` options: run new tests first followed by the rest +of the tests, in both cases tests are also sorted by the file modified time, +with more recent files coming first. + The new config.cache object -------------------------------- diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index d33719920..4fb08862a 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -56,7 +56,7 @@ class TestNewAPI(object): assert result.ret == 1 result.stdout.fnmatch_lines([ "*could not create cache path*", - "*1 warnings*", + "*2 warnings*", ]) def test_config_cache(self, testdir): @@ -495,15 +495,15 @@ class TestLastFailed(object): # Issue #1342 testdir.makepyfile(test_empty='') testdir.runpytest('-q', '--lf') - assert not os.path.exists('.pytest_cache') + assert not os.path.exists('.pytest_cache/v/cache/lastfailed') testdir.makepyfile(test_successful='def test_success():\n assert True') testdir.runpytest('-q', '--lf') - assert not os.path.exists('.pytest_cache') + assert not os.path.exists('.pytest_cache/v/cache/lastfailed') testdir.makepyfile(test_errored='def test_error():\n assert False') testdir.runpytest('-q', '--lf') - assert os.path.exists('.pytest_cache') + assert os.path.exists('.pytest_cache/v/cache/lastfailed') def test_xfail_not_considered_failure(self, testdir): testdir.makepyfile(''' @@ -603,3 +603,112 @@ class TestLastFailed(object): result = testdir.runpytest('--last-failed') result.stdout.fnmatch_lines('*4 passed*') assert self.get_cached_last_failed(testdir) == [] + + +class TestNewFirst(object): + def test_newfirst_usecase(self, testdir): + testdir.makepyfile(**{ + 'test_1/test_1.py': ''' + def test_1(): assert 1 + def test_2(): assert 1 + def test_3(): assert 1 + ''', + 'test_2/test_2.py': ''' + def test_1(): assert 1 + def test_2(): assert 1 + def test_3(): assert 1 + ''' + }) + + testdir.tmpdir.join('test_1/test_1.py').setmtime(1) + + result = testdir.runpytest("-v") + result.stdout.fnmatch_lines([ + "*test_1/test_1.py::test_1 PASSED*", + "*test_1/test_1.py::test_2 PASSED*", + "*test_1/test_1.py::test_3 PASSED*", + "*test_2/test_2.py::test_1 PASSED*", + "*test_2/test_2.py::test_2 PASSED*", + "*test_2/test_2.py::test_3 PASSED*", + ]) + + result = testdir.runpytest("-v", "--nf") + + result.stdout.fnmatch_lines([ + "*test_2/test_2.py::test_1 PASSED*", + "*test_2/test_2.py::test_2 PASSED*", + "*test_2/test_2.py::test_3 PASSED*", + "*test_1/test_1.py::test_1 PASSED*", + "*test_1/test_1.py::test_2 PASSED*", + "*test_1/test_1.py::test_3 PASSED*", + ]) + + testdir.tmpdir.join("test_1/test_1.py").write( + "def test_1(): assert 1\n" + "def test_2(): assert 1\n" + "def test_3(): assert 1\n" + "def test_4(): assert 1\n" + ) + testdir.tmpdir.join('test_1/test_1.py').setmtime(1) + + result = testdir.runpytest("-v", "--nf") + + result.stdout.fnmatch_lines([ + "*test_1/test_1.py::test_4 PASSED*", + "*test_2/test_2.py::test_1 PASSED*", + "*test_2/test_2.py::test_2 PASSED*", + "*test_2/test_2.py::test_3 PASSED*", + "*test_1/test_1.py::test_1 PASSED*", + "*test_1/test_1.py::test_2 PASSED*", + "*test_1/test_1.py::test_3 PASSED*", + ]) + + def test_newfirst_parametrize(self, testdir): + testdir.makepyfile(**{ + 'test_1/test_1.py': ''' + import pytest + @pytest.mark.parametrize('num', [1, 2]) + def test_1(num): assert num + ''', + 'test_2/test_2.py': ''' + import pytest + @pytest.mark.parametrize('num', [1, 2]) + def test_1(num): assert num + ''' + }) + + testdir.tmpdir.join('test_1/test_1.py').setmtime(1) + + result = testdir.runpytest("-v") + result.stdout.fnmatch_lines([ + "*test_1/test_1.py::test_1[1*", + "*test_1/test_1.py::test_1[2*", + "*test_2/test_2.py::test_1[1*", + "*test_2/test_2.py::test_1[2*" + ]) + + result = testdir.runpytest("-v", "--nf") + + result.stdout.fnmatch_lines([ + "*test_2/test_2.py::test_1[1*", + "*test_2/test_2.py::test_1[2*", + "*test_1/test_1.py::test_1[1*", + "*test_1/test_1.py::test_1[2*", + ]) + + testdir.tmpdir.join("test_1/test_1.py").write( + "import pytest\n" + "@pytest.mark.parametrize('num', [1, 2, 3])\n" + "def test_1(num): assert num\n" + ) + testdir.tmpdir.join('test_1/test_1.py').setmtime(1) + + result = testdir.runpytest("-v", "--nf") + + result.stdout.fnmatch_lines([ + "*test_1/test_1.py::test_1[3*", + "*test_2/test_2.py::test_1[1*", + "*test_2/test_2.py::test_1[2*", + "*test_1/test_1.py::test_1[1*", + "*test_1/test_1.py::test_1[2*", + ])