Fix ordering of tests to minimize fixture creating

_pytest/fixtures.py

@@ -166,7 +166,7 @@ def reorder_items(items):
     items_by_argkey = {}
     for scopenum in range(0, scopenum_function):
         argkeys_cache[scopenum] = d = {}
-        items_by_argkey[scopenum] = item_d = defaultdict(list)
+        items_by_argkey[scopenum] = item_d = defaultdict(deque)
         for item in items:
             keys = OrderedDict.fromkeys(get_parametrized_fixture_keys(item, scopenum))
             if keys:
@@ -174,12 +174,19 @@ def reorder_items(items):
                 for key in keys:
                     item_d[key].append(item)
     items = OrderedDict.fromkeys(items)
-    return list(reorder_items_atscope(items, set(), argkeys_cache, items_by_argkey, 0))
+    return list(reorder_items_atscope(items, argkeys_cache, items_by_argkey, 0))
 
 
-def reorder_items_atscope(items, ignore, argkeys_cache, items_by_argkey, scopenum):
+def fix_cache_order(item, argkeys_cache, items_by_argkey):
+    for scopenum in range(0, scopenum_function):
+        for key in argkeys_cache[scopenum].get(item, []):
+            items_by_argkey[scopenum][key].appendleft(item)
+
+
+def reorder_items_atscope(items, argkeys_cache, items_by_argkey, scopenum):
     if scopenum >= scopenum_function or len(items) < 3:
         return items
+    ignore = set()
     items_deque = deque(items)
     items_done = OrderedDict()
     scoped_items_by_argkey = items_by_argkey[scopenum]
@@ -197,13 +204,14 @@ def reorder_items_atscope(items, ignore, argkeys_cache, items_by_argkey, scopenu
             else:
                 slicing_argkey, _ = argkeys.popitem()
                 # we don't have to remove relevant items from later in the deque because they'll just be ignored
-                for i in reversed(scoped_items_by_argkey[slicing_argkey]):
-                    if i in items:
-                        items_deque.appendleft(i)
+                matching_items = [i for i in scoped_items_by_argkey[slicing_argkey] if i in items]
+                for i in reversed(matching_items):
+                    fix_cache_order(i, argkeys_cache, items_by_argkey)
+                    items_deque.appendleft(i)
                 break
         if no_argkey_group:
             no_argkey_group = reorder_items_atscope(
-                                no_argkey_group, set(), argkeys_cache, items_by_argkey, scopenum + 1)
+                                no_argkey_group, argkeys_cache, items_by_argkey, scopenum + 1)
             for item in no_argkey_group:
                 items_done[item] = None
         ignore.add(slicing_argkey)

changelog/3161.bugfix

@@ -0,0 +1 @@
+Fix test ordering bug introduced by PR #3108.

testing/python/fixture.py

@@ -2168,6 +2168,47 @@ class TestFixtureMarker(object):
             test_mod1.py::test_func1[m2] PASSED
         """)
 
+    def test_dynamic_parametrized_ordering(self, testdir):
+        testdir.makeini("""
+            [pytest]
+            console_output_style=classic
+        """)
+        testdir.makeconftest("""
+            import pytest
+
+            def pytest_configure(config):
+                class DynamicFixturePlugin(object):
+                    @pytest.fixture(scope='session', params=['flavor1', 'flavor2'])
+                    def flavor(self, request):
+                        return request.param
+                config.pluginmanager.register(DynamicFixturePlugin(), 'flavor-fixture')
+
+            @pytest.fixture(scope='session', params=['vxlan', 'vlan'])
+            def encap(request):
+                return request.param
+
+            @pytest.fixture(scope='session', autouse='True')
+            def reprovision(request, flavor, encap):
+                pass
+        """)
+        testdir.makepyfile("""
+            def test(reprovision):
+                pass
+            def test2(reprovision):
+                pass
+        """)
+        result = testdir.runpytest("-v")
+        result.stdout.fnmatch_lines("""
+            test_dynamic_parametrized_ordering.py::test[flavor1-vxlan] PASSED
+            test_dynamic_parametrized_ordering.py::test2[flavor1-vxlan] PASSED
+            test_dynamic_parametrized_ordering.py::test[flavor2-vxlan] PASSED
+            test_dynamic_parametrized_ordering.py::test2[flavor2-vxlan] PASSED
+            test_dynamic_parametrized_ordering.py::test[flavor2-vlan] PASSED
+            test_dynamic_parametrized_ordering.py::test2[flavor2-vlan] PASSED
+            test_dynamic_parametrized_ordering.py::test[flavor1-vlan] PASSED
+            test_dynamic_parametrized_ordering.py::test2[flavor1-vlan] PASSED
+        """)
+
     def test_class_ordering(self, testdir):
         testdir.makeini("""
             [pytest]