import pytest, py from _pytest import python as funcargs class TestMetafunc: def Metafunc(self, func): # the unit tests of this class check if things work correctly # on the funcarg level, so we don't need a full blown # initiliazation class FixtureInfo: name2fixturedefs = None def __init__(self, names): self.names_closure = names names = funcargs.getfuncargnames(func) fixtureinfo = FixtureInfo(names) return funcargs.Metafunc(func, fixtureinfo, None) def test_no_funcargs(self, testdir): def function(): pass metafunc = self.Metafunc(function) assert not metafunc.fixturenames repr(metafunc._calls) def test_function_basic(self): def func(arg1, arg2="qwe"): pass metafunc = self.Metafunc(func) assert len(metafunc.fixturenames) == 1 assert 'arg1' in metafunc.fixturenames assert metafunc.function is func assert metafunc.cls is None def test_addcall_no_args(self): def func(arg1): pass metafunc = self.Metafunc(func) metafunc.addcall() assert len(metafunc._calls) == 1 call = metafunc._calls[0] assert call.id == "0" assert not hasattr(call, 'param') def test_addcall_id(self): def func(arg1): pass metafunc = self.Metafunc(func) pytest.raises(ValueError, "metafunc.addcall(id=None)") metafunc.addcall(id=1) pytest.raises(ValueError, "metafunc.addcall(id=1)") pytest.raises(ValueError, "metafunc.addcall(id='1')") metafunc.addcall(id=2) assert len(metafunc._calls) == 2 assert metafunc._calls[0].id == "1" assert metafunc._calls[1].id == "2" def test_addcall_param(self): def func(arg1): pass metafunc = self.Metafunc(func) class obj: pass metafunc.addcall(param=obj) metafunc.addcall(param=obj) metafunc.addcall(param=1) assert len(metafunc._calls) == 3 assert metafunc._calls[0].getparam("arg1") == obj assert metafunc._calls[1].getparam("arg1") == obj assert metafunc._calls[2].getparam("arg1") == 1 def test_addcall_funcargs(self): def func(x): pass metafunc = self.Metafunc(func) class obj: pass metafunc.addcall(funcargs={"x": 2}) metafunc.addcall(funcargs={"x": 3}) pytest.raises(pytest.fail.Exception, "metafunc.addcall({'xyz': 0})") assert len(metafunc._calls) == 2 assert metafunc._calls[0].funcargs == {'x': 2} assert metafunc._calls[1].funcargs == {'x': 3} assert not hasattr(metafunc._calls[1], 'param') def test_parametrize_error(self): def func(x, y): pass metafunc = self.Metafunc(func) metafunc.parametrize("x", [1,2]) pytest.raises(ValueError, lambda: metafunc.parametrize("x", [5,6])) pytest.raises(ValueError, lambda: metafunc.parametrize("x", [5,6])) metafunc.parametrize("y", [1,2]) pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5,6])) pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5,6])) def test_parametrize_and_id(self): def func(x, y): pass metafunc = self.Metafunc(func) metafunc.parametrize("x", [1,2], ids=['basic', 'advanced']) metafunc.parametrize("y", ["abc", "def"]) ids = [x.id for x in metafunc._calls] assert ids == ["basic-abc", "basic-def", "advanced-abc", "advanced-def"] def test_parametrize_with_wrong_number_of_ids(self, testdir): def func(x, y): pass metafunc = self.Metafunc(func) pytest.raises(ValueError, lambda: metafunc.parametrize("x", [1,2], ids=['basic'])) pytest.raises(ValueError, lambda: metafunc.parametrize(("x","y"), [("abc", "def"), ("ghi", "jkl")], ids=["one"])) def test_parametrize_with_userobjects(self): def func(x, y): pass metafunc = self.Metafunc(func) class A: pass metafunc.parametrize("x", [A(), A()]) metafunc.parametrize("y", list("ab")) assert metafunc._calls[0].id == "x0-a" assert metafunc._calls[1].id == "x0-b" assert metafunc._calls[2].id == "x1-a" assert metafunc._calls[3].id == "x1-b" @pytest.mark.issue250 def test_idmaker_autoname(self): from _pytest.python import idmaker result = idmaker(("a", "b"), [("string", 1.0), ("st-ring", 2.0)]) assert result == ["string-1.0", "st-ring-2.0"] result = idmaker(("a", "b"), [(object(), 1.0), (object(), object())]) assert result == ["a0-1.0", "a1-b1"] # unicode mixing, issue250 result = idmaker((py.builtin._totext("a"), "b"), [({}, '\xc3\xb4')]) assert result == ['a0-\xc3\xb4'] def test_idmaker_native_strings(self): from _pytest.python import idmaker result = idmaker(("a", "b"), [(1.0, -1.1), (2, -202), ("three", "three hundred"), (True, False), (None, None), (list("six"), [66, 66]), (set([7]), set("seven")), (tuple("eight"), (8, -8, 8)) ]) assert result == ["1.0--1.1", "2--202", "three-three hundred", "True-False", "None-None", "a5-b5", "a6-b6", "a7-b7"] def test_addcall_and_parametrize(self): def func(x, y): pass metafunc = self.Metafunc(func) metafunc.addcall({'x': 1}) metafunc.parametrize('y', [2,3]) assert len(metafunc._calls) == 2 assert metafunc._calls[0].funcargs == {'x': 1, 'y': 2} assert metafunc._calls[1].funcargs == {'x': 1, 'y': 3} assert metafunc._calls[0].id == "0-2" assert metafunc._calls[1].id == "0-3" def test_parametrize_indirect(self): def func(x, y): pass metafunc = self.Metafunc(func) metafunc.parametrize('x', [1], indirect=True) metafunc.parametrize('y', [2,3], indirect=True) metafunc.parametrize('unnamed', [1], indirect=True) assert len(metafunc._calls) == 2 assert metafunc._calls[0].funcargs == {} assert metafunc._calls[1].funcargs == {} assert metafunc._calls[0].params == dict(x=1,y=2, unnamed=1) assert metafunc._calls[1].params == dict(x=1,y=3, unnamed=1) def test_addcalls_and_parametrize_indirect(self): def func(x, y): pass metafunc = self.Metafunc(func) metafunc.addcall(param="123") metafunc.parametrize('x', [1], indirect=True) metafunc.parametrize('y', [2,3], indirect=True) assert len(metafunc._calls) == 2 assert metafunc._calls[0].funcargs == {} assert metafunc._calls[1].funcargs == {} assert metafunc._calls[0].params == dict(x=1,y=2) assert metafunc._calls[1].params == dict(x=1,y=3) def test_parametrize_functional(self, testdir): testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.parametrize('x', [1,2], indirect=True) metafunc.parametrize('y', [2]) def pytest_funcarg__x(request): return request.param * 10 def pytest_funcarg__y(request): return request.param def test_simple(x,y): assert x in (10,20) assert y == 2 """) result = testdir.runpytest("-v") result.stdout.fnmatch_lines([ "*test_simple*1-2*", "*test_simple*2-2*", "*2 passed*", ]) def test_parametrize_onearg(self): metafunc = self.Metafunc(lambda x: None) metafunc.parametrize("x", [1,2]) assert len(metafunc._calls) == 2 assert metafunc._calls[0].funcargs == dict(x=1) assert metafunc._calls[0].id == "1" assert metafunc._calls[1].funcargs == dict(x=2) assert metafunc._calls[1].id == "2" def test_parametrize_onearg_indirect(self): metafunc = self.Metafunc(lambda x: None) metafunc.parametrize("x", [1,2], indirect=True) assert metafunc._calls[0].params == dict(x=1) assert metafunc._calls[0].id == "1" assert metafunc._calls[1].params == dict(x=2) assert metafunc._calls[1].id == "2" def test_parametrize_twoargs(self): metafunc = self.Metafunc(lambda x,y: None) metafunc.parametrize(("x", "y"), [(1,2), (3,4)]) assert len(metafunc._calls) == 2 assert metafunc._calls[0].funcargs == dict(x=1, y=2) assert metafunc._calls[0].id == "1-2" assert metafunc._calls[1].funcargs == dict(x=3, y=4) assert metafunc._calls[1].id == "3-4" def test_parametrize_multiple_times(self, testdir): testdir.makepyfile(""" import pytest pytestmark = pytest.mark.parametrize("x", [1,2]) def test_func(x): assert 0, x class TestClass: pytestmark = pytest.mark.parametrize("y", [3,4]) def test_meth(self, x, y): assert 0, x """) result = testdir.runpytest() assert result.ret == 1 result.stdout.fnmatch_lines([ "*6 fail*", ]) def test_parametrize_CSV(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.parametrize("x, y,", [(1,2), (2,3)]) def test_func(x, y): assert x+1 == y """) reprec = testdir.inline_run() reprec.assertoutcome(passed=2) def test_parametrize_class_scenarios(self, testdir): testdir.makepyfile(""" # same as doc/en/example/parametrize scenario example def pytest_generate_tests(metafunc): idlist = [] argvalues = [] for scenario in metafunc.cls.scenarios: idlist.append(scenario[0]) items = scenario[1].items() argnames = [x[0] for x in items] argvalues.append(([x[1] for x in items])) metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class") class Test(object): scenarios = [['1', {'arg': {1: 2}, "arg2": "value2"}], ['2', {'arg':'value2', "arg2": "value2"}]] def test_1(self, arg, arg2): pass def test_2(self, arg2, arg): pass def test_3(self, arg, arg2): pass """) result = testdir.runpytest("-v") assert result.ret == 0 result.stdout.fnmatch_lines(""" *test_1*1* *test_2*1* *test_3*1* *test_1*2* *test_2*2* *test_3*2* *6 passed* """) class TestMetafuncFunctional: def test_attributes(self, testdir): p = testdir.makepyfile(""" # assumes that generate/provide runs in the same process import py, pytest def pytest_generate_tests(metafunc): metafunc.addcall(param=metafunc) def pytest_funcarg__metafunc(request): assert request._pyfuncitem._genid == "0" return request.param def test_function(metafunc, pytestconfig): assert metafunc.config == pytestconfig assert metafunc.module.__name__ == __name__ assert metafunc.function == test_function assert metafunc.cls is None class TestClass: def test_method(self, metafunc, pytestconfig): assert metafunc.config == pytestconfig assert metafunc.module.__name__ == __name__ if py.std.sys.version_info > (3, 0): unbound = TestClass.test_method else: unbound = TestClass.test_method.im_func # XXX actually have an unbound test function here? assert metafunc.function == unbound assert metafunc.cls == TestClass """) result = testdir.runpytest(p, "-v") result.stdout.fnmatch_lines([ "*2 passed in*", ]) def test_addcall_with_two_funcargs_generators(self, testdir): testdir.makeconftest(""" def pytest_generate_tests(metafunc): assert "arg1" in metafunc.fixturenames metafunc.addcall(funcargs=dict(arg1=1, arg2=2)) """) p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.addcall(funcargs=dict(arg1=1, arg2=1)) class TestClass: def test_myfunc(self, arg1, arg2): assert arg1 == arg2 """) result = testdir.runpytest("-v", p) result.stdout.fnmatch_lines([ "*test_myfunc*0*PASS*", "*test_myfunc*1*FAIL*", "*1 failed, 1 passed*" ]) def test_two_functions(self, testdir): p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.addcall(param=10) metafunc.addcall(param=20) def pytest_funcarg__arg1(request): return request.param def test_func1(arg1): assert arg1 == 10 def test_func2(arg1): assert arg1 in (10, 20) """) result = testdir.runpytest("-v", p) result.stdout.fnmatch_lines([ "*test_func1*0*PASS*", "*test_func1*1*FAIL*", "*test_func2*PASS*", "*1 failed, 3 passed*" ]) def test_noself_in_method(self, testdir): p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): assert 'xyz' not in metafunc.fixturenames class TestHello: def test_hello(xyz): pass """) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ "*1 pass*", ]) def test_generate_plugin_and_module(self, testdir): testdir.makeconftest(""" def pytest_generate_tests(metafunc): assert "arg1" in metafunc.fixturenames metafunc.addcall(id="world", param=(2,100)) """) p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.addcall(param=(1,1), id="hello") def pytest_funcarg__arg1(request): return request.param[0] def pytest_funcarg__arg2(request): return request.param[1] class TestClass: def test_myfunc(self, arg1, arg2): assert arg1 == arg2 """) result = testdir.runpytest("-v", p) result.stdout.fnmatch_lines([ "*test_myfunc*hello*PASS*", "*test_myfunc*world*FAIL*", "*1 failed, 1 passed*" ]) def test_generate_tests_in_class(self, testdir): p = testdir.makepyfile(""" class TestClass: def pytest_generate_tests(self, metafunc): metafunc.addcall(funcargs={'hello': 'world'}, id="hello") def test_myfunc(self, hello): assert hello == "world" """) result = testdir.runpytest("-v", p) result.stdout.fnmatch_lines([ "*test_myfunc*hello*PASS*", "*1 passed*" ]) def test_two_functions_not_same_instance(self, testdir): p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.addcall({'arg1': 10}) metafunc.addcall({'arg1': 20}) class TestClass: def test_func(self, arg1): assert not hasattr(self, 'x') self.x = 1 """) result = testdir.runpytest("-v", p) result.stdout.fnmatch_lines([ "*test_func*0*PASS*", "*test_func*1*PASS*", "*2 pass*", ]) def test_issue28_setup_method_in_generate_tests(self, testdir): p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.addcall({'arg1': 1}) class TestClass: def test_method(self, arg1): assert arg1 == self.val def setup_method(self, func): self.val = 1 """) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ "*1 pass*", ]) def test_parametrize_functional2(self, testdir): testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.parametrize("arg1", [1,2]) metafunc.parametrize("arg2", [4,5]) def test_hello(arg1, arg2): assert 0, (arg1, arg2) """) result = testdir.runpytest() result.stdout.fnmatch_lines([ "*(1, 4)*", "*(1, 5)*", "*(2, 4)*", "*(2, 5)*", "*4 failed*", ]) def test_parametrize_and_inner_getfuncargvalue(self, testdir): p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): metafunc.parametrize("arg1", [1], indirect=True) metafunc.parametrize("arg2", [10], indirect=True) def pytest_funcarg__arg1(request): x = request.getfuncargvalue("arg2") return x + request.param def pytest_funcarg__arg2(request): return request.param def test_func1(arg1, arg2): assert arg1 == 11 """) result = testdir.runpytest("-v", p) result.stdout.fnmatch_lines([ "*test_func1*1*PASS*", "*1 passed*" ]) def test_parametrize_on_setup_arg(self, testdir): p = testdir.makepyfile(""" def pytest_generate_tests(metafunc): assert "arg1" in metafunc.fixturenames metafunc.parametrize("arg1", [1], indirect=True) def pytest_funcarg__arg1(request): return request.param def pytest_funcarg__arg2(request, arg1): return 10 * arg1 def test_func(arg2): assert arg2 == 10 """) result = testdir.runpytest("-v", p) result.stdout.fnmatch_lines([ "*test_func*1*PASS*", "*1 passed*" ]) def test_parametrize_with_ids(self, testdir): testdir.makepyfile(""" import pytest def pytest_generate_tests(metafunc): metafunc.parametrize(("a", "b"), [(1,1), (1,2)], ids=["basic", "advanced"]) def test_function(a, b): assert a == b """) result = testdir.runpytest("-v") assert result.ret == 1 result.stdout.fnmatch_lines_random([ "*test_function*basic*PASSED", "*test_function*advanced*FAILED", ]) def test_parametrize_without_ids(self, testdir): testdir.makepyfile(""" import pytest def pytest_generate_tests(metafunc): metafunc.parametrize(("a", "b"), [(1,object()), (1.3,object())]) def test_function(a, b): assert 1 """) result = testdir.runpytest("-v") result.stdout.fnmatch_lines(""" *test_function*1-b0* *test_function*1.3-b1* """) @pytest.mark.parametrize(("scope", "length"), [("module", 2), ("function", 4)]) def test_parametrize_scope_overrides(self, testdir, scope, length): testdir.makepyfile(""" import pytest l = [] def pytest_generate_tests(metafunc): if "arg" in metafunc.funcargnames: metafunc.parametrize("arg", [1,2], indirect=True, scope=%r) def pytest_funcarg__arg(request): l.append(request.param) return request.param def test_hello(arg): assert arg in (1,2) def test_world(arg): assert arg in (1,2) def test_checklength(): assert len(l) == %d """ % (scope, length)) reprec = testdir.inline_run() reprec.assertoutcome(passed=5) def test_parametrize_issue323(self, testdir): testdir.makepyfile(""" import pytest @pytest.fixture(scope='module', params=range(966)) def foo(request): return request.param def test_it(foo): pass """) reprec = testdir.inline_run("--collect-only") assert not reprec.getcalls("pytest_internalerror") def test_usefixtures_seen_in_generate_tests(self, testdir): testdir.makepyfile(""" import pytest def pytest_generate_tests(metafunc): assert "abc" in metafunc.fixturenames metafunc.parametrize("abc", [1]) @pytest.mark.usefixtures("abc") def test_function(): pass """) reprec = testdir.inline_run() reprec.assertoutcome(passed=1) def test_generate_tests_only_done_in_subdir(self, testdir): sub1 = testdir.mkpydir("sub1") sub2 = testdir.mkpydir("sub2") sub1.join("conftest.py").write(py.code.Source(""" def pytest_generate_tests(metafunc): assert metafunc.function.__name__ == "test_1" """)) sub2.join("conftest.py").write(py.code.Source(""" def pytest_generate_tests(metafunc): assert metafunc.function.__name__ == "test_2" """)) sub1.join("test_in_sub1.py").write("def test_1(): pass") sub2.join("test_in_sub2.py").write("def test_2(): pass") result = testdir.runpytest("-v", "-s", sub1, sub2, sub1) result.stdout.fnmatch_lines([ "*3 passed*" ]) class TestMarkersWithParametrization: pytestmark = pytest.mark.issue308 def test_simple_mark(self, testdir): s = """ import pytest @pytest.mark.foo @pytest.mark.parametrize(("n", "expected"), [ (1, 2), pytest.mark.bar((1, 3)), (2, 3), ]) def test_increment(n, expected): assert n + 1 == expected """ items = testdir.getitems(s) assert len(items) == 3 for item in items: assert 'foo' in item.keywords assert 'bar' not in items[0].keywords assert 'bar' in items[1].keywords assert 'bar' not in items[2].keywords def test_select_based_on_mark(self, testdir): s = """ import pytest @pytest.mark.parametrize(("n", "expected"), [ (1, 2), pytest.mark.foo((2, 3)), (3, 4), ]) def test_increment(n, expected): assert n + 1 == expected """ testdir.makepyfile(s) rec = testdir.inline_run("-m", 'foo') passed, skipped, fail = rec.listoutcomes() assert len(passed) == 1 assert len(skipped) == 0 assert len(fail) == 0 @pytest.mark.xfail(reason="is this important to support??") def test_nested_marks(self, testdir): s = """ import pytest mastermark = pytest.mark.foo(pytest.mark.bar) @pytest.mark.parametrize(("n", "expected"), [ (1, 2), mastermark((1, 3)), (2, 3), ]) def test_increment(n, expected): assert n + 1 == expected """ items = testdir.getitems(s) assert len(items) == 3 for mark in ['foo', 'bar']: assert mark not in items[0].keywords assert mark in items[1].keywords assert mark not in items[2].keywords def test_simple_xfail(self, testdir): s = """ import pytest @pytest.mark.parametrize(("n", "expected"), [ (1, 2), pytest.mark.xfail((1, 3)), (2, 3), ]) def test_increment(n, expected): assert n + 1 == expected """ testdir.makepyfile(s) reprec = testdir.inline_run() # xfail is skip?? reprec.assertoutcome(passed=2, skipped=1) def test_simple_xfail_single_argname(self, testdir): s = """ import pytest @pytest.mark.parametrize("n", [ 2, pytest.mark.xfail(3), 4, ]) def test_isEven(n): assert n % 2 == 0 """ testdir.makepyfile(s) reprec = testdir.inline_run() reprec.assertoutcome(passed=2, skipped=1) def test_xfail_with_arg(self, testdir): s = """ import pytest @pytest.mark.parametrize(("n", "expected"), [ (1, 2), pytest.mark.xfail("True")((1, 3)), (2, 3), ]) def test_increment(n, expected): assert n + 1 == expected """ testdir.makepyfile(s) reprec = testdir.inline_run() reprec.assertoutcome(passed=2, skipped=1) def test_xfail_with_kwarg(self, testdir): s = """ import pytest @pytest.mark.parametrize(("n", "expected"), [ (1, 2), pytest.mark.xfail(reason="some bug")((1, 3)), (2, 3), ]) def test_increment(n, expected): assert n + 1 == expected """ testdir.makepyfile(s) reprec = testdir.inline_run() reprec.assertoutcome(passed=2, skipped=1) def test_xfail_with_arg_and_kwarg(self, testdir): s = """ import pytest @pytest.mark.parametrize(("n", "expected"), [ (1, 2), pytest.mark.xfail("True", reason="some bug")((1, 3)), (2, 3), ]) def test_increment(n, expected): assert n + 1 == expected """ testdir.makepyfile(s) reprec = testdir.inline_run() reprec.assertoutcome(passed=2, skipped=1) def test_xfail_passing_is_xpass(self, testdir): s = """ import pytest @pytest.mark.parametrize(("n", "expected"), [ (1, 2), pytest.mark.xfail("sys.version > 0", reason="some bug")((2, 3)), (3, 4), ]) def test_increment(n, expected): assert n + 1 == expected """ testdir.makepyfile(s) reprec = testdir.inline_run() # xpass is fail, obviously :) reprec.assertoutcome(passed=2, failed=1) def test_parametrize_called_in_generate_tests(self, testdir): s = """ import pytest def pytest_generate_tests(metafunc): passingTestData = [(1, 2), (2, 3)] failingTestData = [(1, 3), (2, 2)] testData = passingTestData + [pytest.mark.xfail(d) for d in failingTestData] metafunc.parametrize(("n", "expected"), testData) def test_increment(n, expected): assert n + 1 == expected """ testdir.makepyfile(s) reprec = testdir.inline_run() reprec.assertoutcome(passed=2, skipped=2) @pytest.mark.xfail(reason="issue 290") def test_parametrize_ID_generation_string_int_works(self, testdir): testdir.makepyfile(""" import pytest @pytest.fixture def myfixture(): return 'example' @pytest.mark.parametrize( 'limit', (0, '0')) def test_limit(limit, myfixture): return """) reprec = testdir.inline_run() reprec.assertoutcome(passed=2)