improve the parametrization scenario example to sort by id, rather than by file-order, see also: http://stackoverflow.com/questions/12521924/pytest-running-scenarios-in-the-correct-order-in-the-class

CHANGELOG

@@ -6,6 +6,8 @@ Changes between 2.2.4 and 2.3.0.dev
   node.markers allows reading and manipulating of MarkInfo objects
   previously attached with @pytest.mark.* or request.applymarker or
   setattr(node.markers, name, pytest.mark.*) calls.
+- introduce re-ordering of tests by resource and parametrization setup
+  which takes precedence to the usual file-ordering
 - fix issue185 monkeypatching time.time does not cause pytest to fail
 - fix issue172 duplicate call of pytest.setup-decoratored setup_module
   functions

_pytest/__init__.py

@@ -1,2 +1,2 @@
 #
-__version__ = '2.3.0.dev14'
+__version__ = '2.3.0.dev15'

_pytest/python.py

@@ -574,6 +574,10 @@ class CallSpec2(object):
         for arg,val in zip(argnames, valset):
             self._checkargnotcontained(arg)
             getattr(self, valtype)[arg] = val
+            # we want self.params to be always set because of
+            # parametrize_sorted() which groups tests by params/scope
+            if valtype == "funcargs":
+                self.params[arg] = id
             self._arg2scopenum[arg] = scopenum
         self._idlist.append(id)
 

doc/en/example/parametrize.txt

@@ -96,7 +96,7 @@ This means that we only run 2 tests if we do not pass ``--all``::
     $ py.test -q test_compute.py
     collecting ... collected 2 items
     ..
-    2 passed in 0.02 seconds
+    2 passed in 0.01 seconds
 
 We run only two computations, so we see two dots.
 let's run the full monty::
@@ -139,7 +139,7 @@ only have to work a bit to construct the correct arguments for pytest's
             items = scenario[1].items()
             argnames = [x[0] for x in items]
             argvalues.append(([x[1] for x in items]))
-        metafunc.parametrize(argnames, argvalues, ids=idlist)
+        metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class")
 
     scenario1 = ('basic', {'attribute': 'value'})
     scenario2 = ('advanced', {'attribute': 'value2'})
@@ -147,36 +147,45 @@ only have to work a bit to construct the correct arguments for pytest's
     class TestSampleWithScenarios:
         scenarios = [scenario1, scenario2]
 
-        def test_demo(self, attribute):
+        def test_demo1(self, attribute):
+            assert isinstance(attribute, str)
+
+        def test_demo2(self, attribute):
             assert isinstance(attribute, str)
 
 this is a fully self-contained example which you can run with::
 
     $ py.test test_scenarios.py
     =========================== test session starts ============================
-    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev14
-    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    plugins: xdist, bugzilla, cache, oejskit, cli, timeout, pep8, cov
-    collecting ... collected 2 items
+    collecting ... collected 4 items
     
-    test_scenarios.py ..
+    test_scenarios.py ....
     
-    ========================= 2 passed in 0.02 seconds =========================
+    ========================= 4 passed in 0.02 seconds =========================
 
 If you just collect tests you'll also nicely see 'advanced' and 'basic' as variants for the test function::
 
 
     $ py.test --collectonly test_scenarios.py
     =========================== test session starts ============================
-    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev14
-    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    plugins: xdist, bugzilla, cache, oejskit, cli, timeout, pep8, cov
-    collecting ... collected 2 items
+    collecting ... collected 4 items
     <Module 'test_scenarios.py'>
       <Class 'TestSampleWithScenarios'>
         <Instance '()'>
-          <Function 'test_demo[basic]'>
+          <Function 'test_demo1[basic]'>
-          <Function 'test_demo[advanced]'>
+          <Function 'test_demo2[basic]'>
+          <Function 'test_demo1[advanced]'>
+          <Function 'test_demo2[advanced]'>
     
-    =============================  in 0.02 seconds =============================
+    =============================  in 0.01 seconds =============================
+
+Note that we told ``metafunc.parametrize()`` that your scenario values
+should be considered class-scoped.  With pytest-2.3 this leads to a
+resource-based ordering.
 
 Deferring the setup of parametrized resources
 ---------------------------------------------------
@@ -224,14 +233,14 @@ Let's first see how it looks like at collection time::
 
     $ py.test test_backends.py --collectonly
     =========================== test session starts ============================
-    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev14
-    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    plugins: xdist, bugzilla, cache, oejskit, cli, timeout, pep8, cov
     collecting ... collected 2 items
     <Module 'test_backends.py'>
       <Function 'test_db_initialized[d1]'>
       <Function 'test_db_initialized[d2]'>
     
-    =============================  in 0.02 seconds =============================
+    =============================  in 0.01 seconds =============================
 
 And then when we run the test::
 
@@ -241,7 +250,7 @@ And then when we run the test::
     ================================= FAILURES =================================
     _________________________ test_db_initialized[d2] __________________________
     
-    db = <conftest.DB2 instance at 0x26bcea8>
+    db = <conftest.DB2 instance at 0x17dd440>
     
         def test_db_initialized(db):
             # a dummy test
@@ -250,7 +259,7 @@ And then when we run the test::
     E           Failed: deliberately failing for demo purposes
     
     test_backends.py:6: Failed
-    1 failed, 1 passed in 0.02 seconds
+    1 failed, 1 passed in 0.01 seconds
 
 The first invocation with ``db == "DB1"`` passed while the second with ``db == "DB2"`` failed.  Our ``pytest_funcarg__db`` factory has instantiated each of the DB values during the setup phase while the ``pytest_generate_tests`` generated two according calls to the ``test_db_initialized`` during the collection phase.
 
@@ -298,7 +307,7 @@ argument sets to use for each test function.  Let's run it::
     ================================= FAILURES =================================
     ________________________ TestClass.test_equals[1-2] ________________________
     
-    self = <test_parametrize.TestClass instance at 0x2fa9050>, a = 1, b = 2
+    self = <test_parametrize.TestClass instance at 0x19a6d88>, a = 1, b = 2
     
         def test_equals(self, a, b):
     >       assert a == b
@@ -327,5 +336,5 @@ Running it results in some skips if we don't have all the python interpreters in
    collecting ... collected 75 items
    ............sss............sss............sss............ssssssssssssssssss
    ========================= short test summary info ==========================
-   SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:36: 'python2.8' not found
+   SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:21: 'python2.8' not found
-   48 passed, 27 skipped in 1.70 seconds
+   48 passed, 27 skipped in 3.11 seconds

setup.py

@@ -24,7 +24,7 @@ def main():
         name='pytest',
         description='py.test: simple powerful testing with Python',
         long_description = long_description,
-        version='2.3.0.dev14',
+        version='2.3.0.dev15',
         url='http://pytest.org',
         license='MIT license',
         platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],

testing/test_python.py

@@ -1095,6 +1095,44 @@ class TestMetafunc:
             "*6 fail*",
         ])
 
+    def test_parametrize_class_scenarios(self, testdir):
+        testdir.makepyfile("""
+        # same as doc/en/example/parametrize scenario example
+        def pytest_generate_tests(metafunc):
+            idlist = []
+            argvalues = []
+            for scenario in metafunc.cls.scenarios:
+                idlist.append(scenario[0])
+                items = scenario[1].items()
+                argnames = [x[0] for x in items]
+                argvalues.append(([x[1] for x in items]))
+            metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class")
+
+        class Test(object):
+               scenarios = [['1', {'arg': {1: 2}, "arg2": "value2"}],
+                            ['2', {'arg':'value2', "arg2": "value2"}]]
+
+               def test_1(self, arg, arg2):
+                  pass
+
+               def test_2(self, arg2, arg):
+                  pass
+
+               def test_3(self, arg, arg2):
+                  pass
+        """)
+        result = testdir.runpytest("-v")
+        assert result.ret == 0
+        result.stdout.fnmatch_lines("""
+            *test_1*1*
+            *test_2*1*
+            *test_3*1*
+            *test_1*2*
+            *test_2*2*
+            *test_3*2*
+            *6 passed*
+        """)
+
 class TestMetafuncFunctional:
     def test_attributes(self, testdir):
         p = testdir.makepyfile("""