Merge pull request #3218 from feuillemorte/3034-new-tests-first

#3034 Added new option "--new-first"

_pytest/cacheprovider.py

@@ -5,7 +5,12 @@ the name cache was not chosen to ensure pluggy automatically
 ignores the external pytest-cache
 """
 from __future__ import absolute_import, division, print_function
+
+from collections import OrderedDict
+
 import py
+import six
+
 import pytest
 import json
 import os
@@ -168,6 +173,39 @@ class LFPlugin(object):
             config.cache.set("cache/lastfailed", self.lastfailed)
 
 
+class NFPlugin(object):
+    """ Plugin which implements the --nf (run new-first) option """
+
+    def __init__(self, config):
+        self.config = config
+        self.active = config.option.newfirst
+        self.cached_nodeids = config.cache.get("cache/nodeids", [])
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        if self.active:
+            new_items = OrderedDict()
+            other_items = OrderedDict()
+            for item in items:
+                if item.nodeid not in self.cached_nodeids:
+                    new_items[item.nodeid] = item
+                else:
+                    other_items[item.nodeid] = item
+
+            items[:] = self._get_increasing_order(six.itervalues(new_items)) + \
+                self._get_increasing_order(six.itervalues(other_items))
+        self.cached_nodeids = [x.nodeid for x in items if isinstance(x, pytest.Item)]
+
+    def _get_increasing_order(self, items):
+        return sorted(items, key=lambda item: item.fspath.mtime(), reverse=True)
+
+    def pytest_sessionfinish(self, session):
+        config = self.config
+        if config.getoption("cacheshow") or hasattr(config, "slaveinput"):
+            return
+
+        config.cache.set("cache/nodeids", self.cached_nodeids)
+
+
 def pytest_addoption(parser):
     group = parser.getgroup("general")
     group.addoption(
@@ -179,6 +217,10 @@ def pytest_addoption(parser):
         help="run all tests but run the last failures first.  "
              "This may re-order tests and thus lead to "
              "repeated fixture setup/teardown")
+    group.addoption(
+        '--nf', '--new-first', action='store_true', dest="newfirst",
+        help="run tests from new files first, then the rest of the tests "
+             "sorted by file mtime")
     group.addoption(
         '--cache-show', action='store_true', dest="cacheshow",
         help="show cache contents, don't perform collection or tests")
@@ -200,6 +242,7 @@ def pytest_cmdline_main(config):
 def pytest_configure(config):
     config.cache = Cache(config)
     config.pluginmanager.register(LFPlugin(config), "lfplugin")
+    config.pluginmanager.register(NFPlugin(config), "nfplugin")
 
 
 @pytest.fixture

changelog/3034.feature

@@ -0,0 +1 @@
+New ``--nf``, ``--new-first`` options: run new tests first followed by the rest of the tests, in both cases tests are also sorted by the file modified time, with more recent files coming first.

doc/en/cache.rst

@@ -152,6 +152,10 @@ of ``FF`` and dots)::
 
 .. _`config.cache`:
 
+New ``--nf``, ``--new-first`` options: run new tests first followed by the rest
+of the tests, in both cases tests are also sorted by the file modified time,
+with more recent files coming first.
+
 The new config.cache object
 --------------------------------
 

testing/test_cacheprovider.py

@@ -56,7 +56,7 @@ class TestNewAPI(object):
         assert result.ret == 1
         result.stdout.fnmatch_lines([
             "*could not create cache path*",
-            "*1 warnings*",
+            "*2 warnings*",
         ])
 
     def test_config_cache(self, testdir):
@@ -495,15 +495,15 @@ class TestLastFailed(object):
         # Issue #1342
         testdir.makepyfile(test_empty='')
         testdir.runpytest('-q', '--lf')
-        assert not os.path.exists('.pytest_cache')
+        assert not os.path.exists('.pytest_cache/v/cache/lastfailed')
 
         testdir.makepyfile(test_successful='def test_success():\n    assert True')
         testdir.runpytest('-q', '--lf')
-        assert not os.path.exists('.pytest_cache')
+        assert not os.path.exists('.pytest_cache/v/cache/lastfailed')
 
         testdir.makepyfile(test_errored='def test_error():\n    assert False')
         testdir.runpytest('-q', '--lf')
-        assert os.path.exists('.pytest_cache')
+        assert os.path.exists('.pytest_cache/v/cache/lastfailed')
 
     def test_xfail_not_considered_failure(self, testdir):
         testdir.makepyfile('''
@@ -603,3 +603,112 @@ class TestLastFailed(object):
         result = testdir.runpytest('--last-failed')
         result.stdout.fnmatch_lines('*4 passed*')
         assert self.get_cached_last_failed(testdir) == []
+
+
+class TestNewFirst(object):
+    def test_newfirst_usecase(self, testdir):
+        testdir.makepyfile(**{
+            'test_1/test_1.py': '''
+                def test_1(): assert 1
+                def test_2(): assert 1
+                def test_3(): assert 1
+            ''',
+            'test_2/test_2.py': '''
+                def test_1(): assert 1
+                def test_2(): assert 1
+                def test_3(): assert 1
+            '''
+        })
+
+        testdir.tmpdir.join('test_1/test_1.py').setmtime(1)
+
+        result = testdir.runpytest("-v")
+        result.stdout.fnmatch_lines([
+            "*test_1/test_1.py::test_1 PASSED*",
+            "*test_1/test_1.py::test_2 PASSED*",
+            "*test_1/test_1.py::test_3 PASSED*",
+            "*test_2/test_2.py::test_1 PASSED*",
+            "*test_2/test_2.py::test_2 PASSED*",
+            "*test_2/test_2.py::test_3 PASSED*",
+        ])
+
+        result = testdir.runpytest("-v", "--nf")
+
+        result.stdout.fnmatch_lines([
+            "*test_2/test_2.py::test_1 PASSED*",
+            "*test_2/test_2.py::test_2 PASSED*",
+            "*test_2/test_2.py::test_3 PASSED*",
+            "*test_1/test_1.py::test_1 PASSED*",
+            "*test_1/test_1.py::test_2 PASSED*",
+            "*test_1/test_1.py::test_3 PASSED*",
+        ])
+
+        testdir.tmpdir.join("test_1/test_1.py").write(
+            "def test_1(): assert 1\n"
+            "def test_2(): assert 1\n"
+            "def test_3(): assert 1\n"
+            "def test_4(): assert 1\n"
+        )
+        testdir.tmpdir.join('test_1/test_1.py').setmtime(1)
+
+        result = testdir.runpytest("-v", "--nf")
+
+        result.stdout.fnmatch_lines([
+            "*test_1/test_1.py::test_4 PASSED*",
+            "*test_2/test_2.py::test_1 PASSED*",
+            "*test_2/test_2.py::test_2 PASSED*",
+            "*test_2/test_2.py::test_3 PASSED*",
+            "*test_1/test_1.py::test_1 PASSED*",
+            "*test_1/test_1.py::test_2 PASSED*",
+            "*test_1/test_1.py::test_3 PASSED*",
+        ])
+
+    def test_newfirst_parametrize(self, testdir):
+        testdir.makepyfile(**{
+            'test_1/test_1.py': '''
+                import pytest
+                @pytest.mark.parametrize('num', [1, 2])
+                def test_1(num): assert num
+            ''',
+            'test_2/test_2.py': '''
+                import pytest
+                @pytest.mark.parametrize('num', [1, 2])
+                def test_1(num): assert num
+            '''
+        })
+
+        testdir.tmpdir.join('test_1/test_1.py').setmtime(1)
+
+        result = testdir.runpytest("-v")
+        result.stdout.fnmatch_lines([
+            "*test_1/test_1.py::test_1[1*",
+            "*test_1/test_1.py::test_1[2*",
+            "*test_2/test_2.py::test_1[1*",
+            "*test_2/test_2.py::test_1[2*"
+        ])
+
+        result = testdir.runpytest("-v", "--nf")
+
+        result.stdout.fnmatch_lines([
+            "*test_2/test_2.py::test_1[1*",
+            "*test_2/test_2.py::test_1[2*",
+            "*test_1/test_1.py::test_1[1*",
+            "*test_1/test_1.py::test_1[2*",
+        ])
+
+        testdir.tmpdir.join("test_1/test_1.py").write(
+            "import pytest\n"
+            "@pytest.mark.parametrize('num', [1, 2, 3])\n"
+            "def test_1(num): assert num\n"
+        )
+        testdir.tmpdir.join('test_1/test_1.py').setmtime(1)
+
+        result = testdir.runpytest("-v", "--nf")
+
+        result.stdout.fnmatch_lines([
+            "*test_1/test_1.py::test_1[3*",
+            "*test_2/test_2.py::test_1[1*",
+            "*test_2/test_2.py::test_1[2*",
+            "*test_1/test_1.py::test_1[1*",
+            "*test_1/test_1.py::test_1[2*",
+        ])