diff --git a/changelog/3623.feature.rst b/changelog/3623.feature.rst new file mode 100644 index 000000000..2e6f4c428 --- /dev/null +++ b/changelog/3623.feature.rst @@ -0,0 +1 @@ +introduce ``pytester.copy_example`` as helper to do acceptance tests against examples from the project diff --git a/doc/en/example/reportingdemo.rst b/doc/en/example/reportingdemo.rst index 4691b128b..a7cc81694 100644 --- a/doc/en/example/reportingdemo.rst +++ b/doc/en/example/reportingdemo.rst @@ -32,7 +32,6 @@ get on the terminal - we are working on that):: self = def test_simple(self): - def f(): return 42 @@ -44,7 +43,7 @@ get on the terminal - we are working on that):: E + where 42 = .f at 0xdeadbeef>() E + and 43 = .g at 0xdeadbeef>() - failure_demo.py:37: AssertionError + failure_demo.py:35: AssertionError ____________________ TestFailing.test_simple_multiline _____________________ self = @@ -52,7 +51,7 @@ get on the terminal - we are working on that):: def test_simple_multiline(self): > otherfunc_multi(42, 6 * 9) - failure_demo.py:40: + failure_demo.py:38: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ a = 42, b = 54 @@ -67,7 +66,6 @@ get on the terminal - we are working on that):: self = def test_not(self): - def f(): return 42 @@ -75,7 +73,7 @@ get on the terminal - we are working on that):: E assert not 42 E + where 42 = .f at 0xdeadbeef>() - failure_demo.py:47: AssertionError + failure_demo.py:44: AssertionError _________________ TestSpecialisedExplanations.test_eq_text _________________ self = @@ -86,7 +84,7 @@ get on the terminal - we are working on that):: E - spam E + eggs - failure_demo.py:53: AssertionError + failure_demo.py:49: AssertionError _____________ TestSpecialisedExplanations.test_eq_similar_text _____________ self = @@ -99,7 +97,7 @@ get on the terminal - we are working on that):: E + foo 2 bar E ? ^ - failure_demo.py:56: AssertionError + failure_demo.py:52: AssertionError ____________ TestSpecialisedExplanations.test_eq_multiline_text ____________ self = @@ -112,7 +110,7 @@ get on the terminal - we are working on that):: E + eggs E bar - failure_demo.py:59: AssertionError + failure_demo.py:55: AssertionError ______________ TestSpecialisedExplanations.test_eq_long_text _______________ self = @@ -129,7 +127,7 @@ get on the terminal - we are working on that):: E + 1111111111b222222222 E ? ^ - failure_demo.py:64: AssertionError + failure_demo.py:60: AssertionError _________ TestSpecialisedExplanations.test_eq_long_text_multiline __________ self = @@ -149,7 +147,7 @@ get on the terminal - we are working on that):: E E ...Full output truncated (7 lines hidden), use '-vv' to show - failure_demo.py:69: AssertionError + failure_demo.py:65: AssertionError _________________ TestSpecialisedExplanations.test_eq_list _________________ self = @@ -160,7 +158,7 @@ get on the terminal - we are working on that):: E At index 2 diff: 2 != 3 E Use -v to get the full diff - failure_demo.py:72: AssertionError + failure_demo.py:68: AssertionError ______________ TestSpecialisedExplanations.test_eq_list_long _______________ self = @@ -173,7 +171,7 @@ get on the terminal - we are working on that):: E At index 100 diff: 1 != 2 E Use -v to get the full diff - failure_demo.py:77: AssertionError + failure_demo.py:73: AssertionError _________________ TestSpecialisedExplanations.test_eq_dict _________________ self = @@ -191,7 +189,7 @@ get on the terminal - we are working on that):: E E ...Full output truncated (2 lines hidden), use '-vv' to show - failure_demo.py:80: AssertionError + failure_demo.py:76: AssertionError _________________ TestSpecialisedExplanations.test_eq_set __________________ self = @@ -209,7 +207,7 @@ get on the terminal - we are working on that):: E E ...Full output truncated (2 lines hidden), use '-vv' to show - failure_demo.py:83: AssertionError + failure_demo.py:79: AssertionError _____________ TestSpecialisedExplanations.test_eq_longer_list ______________ self = @@ -220,7 +218,7 @@ get on the terminal - we are working on that):: E Right contains more items, first extra item: 3 E Use -v to get the full diff - failure_demo.py:86: AssertionError + failure_demo.py:82: AssertionError _________________ TestSpecialisedExplanations.test_in_list _________________ self = @@ -229,7 +227,7 @@ get on the terminal - we are working on that):: > assert 1 in [0, 2, 3, 4, 5] E assert 1 in [0, 2, 3, 4, 5] - failure_demo.py:89: AssertionError + failure_demo.py:85: AssertionError __________ TestSpecialisedExplanations.test_not_in_text_multiline __________ self = @@ -248,7 +246,7 @@ get on the terminal - we are working on that):: E E ...Full output truncated (2 lines hidden), use '-vv' to show - failure_demo.py:93: AssertionError + failure_demo.py:89: AssertionError ___________ TestSpecialisedExplanations.test_not_in_text_single ____________ self = @@ -261,7 +259,7 @@ get on the terminal - we are working on that):: E single foo line E ? +++ - failure_demo.py:97: AssertionError + failure_demo.py:93: AssertionError _________ TestSpecialisedExplanations.test_not_in_text_single_long _________ self = @@ -274,7 +272,7 @@ get on the terminal - we are working on that):: E head head foo tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail E ? +++ - failure_demo.py:101: AssertionError + failure_demo.py:97: AssertionError ______ TestSpecialisedExplanations.test_not_in_text_single_long_term _______ self = @@ -287,11 +285,10 @@ get on the terminal - we are working on that):: E head head fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffftail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail E ? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - failure_demo.py:105: AssertionError + failure_demo.py:101: AssertionError ______________________________ test_attribute ______________________________ def test_attribute(): - class Foo(object): b = 1 @@ -300,11 +297,10 @@ get on the terminal - we are working on that):: E assert 1 == 2 E + where 1 = .Foo object at 0xdeadbeef>.b - failure_demo.py:114: AssertionError + failure_demo.py:109: AssertionError _________________________ test_attribute_instance __________________________ def test_attribute_instance(): - class Foo(object): b = 1 @@ -313,13 +309,11 @@ get on the terminal - we are working on that):: E + where 1 = .Foo object at 0xdeadbeef>.b E + where .Foo object at 0xdeadbeef> = .Foo'>() - failure_demo.py:122: AssertionError + failure_demo.py:116: AssertionError __________________________ test_attribute_failure __________________________ def test_attribute_failure(): - class Foo(object): - def _get_b(self): raise Exception("Failed to get attrib") @@ -328,7 +322,7 @@ get on the terminal - we are working on that):: i = Foo() > assert i.b == 2 - failure_demo.py:135: + failure_demo.py:127: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = .Foo object at 0xdeadbeef> @@ -337,11 +331,10 @@ get on the terminal - we are working on that):: > raise Exception("Failed to get attrib") E Exception: Failed to get attrib - failure_demo.py:130: Exception + failure_demo.py:122: Exception _________________________ test_attribute_multiple __________________________ def test_attribute_multiple(): - class Foo(object): b = 1 @@ -355,7 +348,7 @@ get on the terminal - we are working on that):: E + and 2 = .Bar object at 0xdeadbeef>.b E + where .Bar object at 0xdeadbeef> = .Bar'>() - failure_demo.py:146: AssertionError + failure_demo.py:137: AssertionError __________________________ TestRaises.test_raises __________________________ self = @@ -364,13 +357,13 @@ get on the terminal - we are working on that):: s = "qwe" # NOQA > raises(TypeError, "int(s)") - failure_demo.py:157: + failure_demo.py:147: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ > int(s) E ValueError: invalid literal for int() with base 10: 'qwe' - <0-codegen $PYTHON_PREFIX/lib/python3.5/site-packages/_pytest/python_api.py:634>:1: ValueError + <0-codegen $PYTHON_PREFIX/lib/python3.5/site-packages/_pytest/python_api.py:635>:1: ValueError ______________________ TestRaises.test_raises_doesnt _______________________ self = @@ -379,7 +372,7 @@ get on the terminal - we are working on that):: > raises(IOError, "int('3')") E Failed: DID NOT RAISE - failure_demo.py:160: Failed + failure_demo.py:150: Failed __________________________ TestRaises.test_raise ___________________________ self = @@ -388,7 +381,7 @@ get on the terminal - we are working on that):: > raise ValueError("demo error") E ValueError: demo error - failure_demo.py:163: ValueError + failure_demo.py:153: ValueError ________________________ TestRaises.test_tupleerror ________________________ self = @@ -397,7 +390,7 @@ get on the terminal - we are working on that):: > a, b = [1] # NOQA E ValueError: not enough values to unpack (expected 2, got 1) - failure_demo.py:166: ValueError + failure_demo.py:156: ValueError ______ TestRaises.test_reinterpret_fails_with_print_for_the_fun_of_it ______ self = @@ -408,7 +401,7 @@ get on the terminal - we are working on that):: > a, b = items.pop() E TypeError: 'int' object is not iterable - failure_demo.py:171: TypeError + failure_demo.py:161: TypeError --------------------------- Captured stdout call --------------------------- items is [1, 2, 3] ________________________ TestRaises.test_some_error ________________________ @@ -419,7 +412,7 @@ get on the terminal - we are working on that):: > if namenotexi: # NOQA E NameError: name 'namenotexi' is not defined - failure_demo.py:174: NameError + failure_demo.py:164: NameError ____________________ test_dynamic_compile_shows_nicely _____________________ def test_dynamic_compile_shows_nicely(): @@ -434,20 +427,19 @@ get on the terminal - we are working on that):: sys.modules[name] = module > module.foo() - failure_demo.py:192: + failure_demo.py:182: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def foo(): > assert 1 == 0 E AssertionError - <2-codegen 'abc-123' $REGENDOC_TMPDIR/assertion/failure_demo.py:189>:2: AssertionError + <2-codegen 'abc-123' $REGENDOC_TMPDIR/assertion/failure_demo.py:179>:2: AssertionError ____________________ TestMoreErrors.test_complex_error _____________________ self = def test_complex_error(self): - def f(): return 44 @@ -456,7 +448,7 @@ get on the terminal - we are working on that):: > somefunc(f(), g()) - failure_demo.py:205: + failure_demo.py:193: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ failure_demo.py:11: in somefunc otherfunc(x, y) @@ -478,7 +470,7 @@ get on the terminal - we are working on that):: > a, b = items E ValueError: not enough values to unpack (expected 2, got 0) - failure_demo.py:209: ValueError + failure_demo.py:197: ValueError ____________________ TestMoreErrors.test_z2_type_error _____________________ self = @@ -488,7 +480,7 @@ get on the terminal - we are working on that):: > a, b = items E TypeError: 'int' object is not iterable - failure_demo.py:213: TypeError + failure_demo.py:201: TypeError ______________________ TestMoreErrors.test_startswith ______________________ self = @@ -501,13 +493,12 @@ get on the terminal - we are working on that):: E + where False = ('456') E + where = '123'.startswith - failure_demo.py:218: AssertionError + failure_demo.py:206: AssertionError __________________ TestMoreErrors.test_startswith_nested ___________________ self = def test_startswith_nested(self): - def f(): return "123" @@ -521,7 +512,7 @@ get on the terminal - we are working on that):: E + where '123' = .f at 0xdeadbeef>() E + and '456' = .g at 0xdeadbeef>() - failure_demo.py:228: AssertionError + failure_demo.py:215: AssertionError _____________________ TestMoreErrors.test_global_func ______________________ self = @@ -532,7 +523,7 @@ get on the terminal - we are working on that):: E + where False = isinstance(43, float) E + where 43 = globf(42) - failure_demo.py:231: AssertionError + failure_demo.py:218: AssertionError _______________________ TestMoreErrors.test_instance _______________________ self = @@ -543,7 +534,7 @@ get on the terminal - we are working on that):: E assert 42 != 42 E + where 42 = .x - failure_demo.py:235: AssertionError + failure_demo.py:222: AssertionError _______________________ TestMoreErrors.test_compare ________________________ self = @@ -553,7 +544,7 @@ get on the terminal - we are working on that):: E assert 11 < 5 E + where 11 = globf(10) - failure_demo.py:238: AssertionError + failure_demo.py:225: AssertionError _____________________ TestMoreErrors.test_try_finally ______________________ self = @@ -564,13 +555,12 @@ get on the terminal - we are working on that):: > assert x == 0 E assert 1 == 0 - failure_demo.py:243: AssertionError + failure_demo.py:230: AssertionError ___________________ TestCustomAssertMsg.test_single_line ___________________ self = def test_single_line(self): - class A(object): a = 1 @@ -580,13 +570,12 @@ get on the terminal - we are working on that):: E assert 1 == 2 E + where 1 = .A'>.a - failure_demo.py:256: AssertionError + failure_demo.py:241: AssertionError ____________________ TestCustomAssertMsg.test_multiline ____________________ self = def test_multiline(self): - class A(object): a = 1 @@ -600,13 +589,12 @@ get on the terminal - we are working on that):: E assert 1 == 2 E + where 1 = .A'>.a - failure_demo.py:264: AssertionError + failure_demo.py:248: AssertionError ___________________ TestCustomAssertMsg.test_custom_repr ___________________ self = def test_custom_repr(self): - class JSON(object): a = 1 @@ -623,7 +611,7 @@ get on the terminal - we are working on that):: E assert 1 == 2 E + where 1 = This is JSON\n{\n 'foo': 'bar'\n}.a - failure_demo.py:278: AssertionError + failure_demo.py:261: AssertionError ============================= warnings summary ============================= Metafunc.addcall is deprecated and scheduled to be removed in pytest 4.0. diff --git a/doc/en/writing_plugins.rst b/doc/en/writing_plugins.rst index 95b810bcb..aa361799b 100644 --- a/doc/en/writing_plugins.rst +++ b/doc/en/writing_plugins.rst @@ -386,11 +386,52 @@ return a result object, with which we can assert the tests' outcomes. result.assert_outcomes(passed=4) +additionally it is possible to copy examples for a example folder before running pytest on it + +.. code:: ini + + # content of pytest.ini + [pytest] + pytester_example_dir = . + + +.. code:: python + + # content of test_example.py + + + def test_plugin(testdir): + testdir.copy_example("test_example.py") + testdir.runpytest("-k", "test_example") + + def test_example(): + pass + +.. code:: + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-3.x.y, py-1.x.y, pluggy-0.x.y + rootdir: $REGENDOC_TMPDIR, inifile: pytest.ini + collected 2 items + + test_example.py .. [100%] + + ============================= warnings summary ============================= + test_example.py::test_plugin + $REGENDOC_TMPDIR/test_example.py:4: PytestExerimentalApiWarning: testdir.copy_example is an experimental api that may change over time + testdir.copy_example("test_example.py") + + -- Docs: http://doc.pytest.org/en/latest/warnings.html + =================== 2 passed, 1 warnings in 0.12 seconds =================== + For more information about the result object that ``runpytest()`` returns, and the methods that it provides please check out the :py:class:`RunResult <_pytest.pytester.RunResult>` documentation. + + .. _`writinghooks`: Writing hook functions diff --git a/src/_pytest/experiments.py b/src/_pytest/experiments.py new file mode 100644 index 000000000..aa6b66446 --- /dev/null +++ b/src/_pytest/experiments.py @@ -0,0 +1,13 @@ +class PytestExerimentalApiWarning(FutureWarning): + "warning category used to denote experiments in pytest" + + @classmethod + def simple(cls, apiname): + return cls( + "{apiname} is an experimental api that may change over time".format( + apiname=apiname + ) + ) + + +PYTESTER_COPY_EXAMPLE = PytestExerimentalApiWarning.simple("testdir.copy_example") diff --git a/src/_pytest/pytester.py b/src/_pytest/pytester.py index ce1c8ea1c..58e4d5d54 100644 --- a/src/_pytest/pytester.py +++ b/src/_pytest/pytester.py @@ -53,6 +53,10 @@ def pytest_addoption(parser): ), ) + parser.addini( + "pytester_example_dir", help="directory to take the pytester example files from" + ) + def pytest_configure(config): if config.getvalue("lsof"): @@ -628,6 +632,22 @@ class Testdir(object): p.ensure("__init__.py") return p + def copy_example(self, name): + from . import experiments + import warnings + + warnings.warn(experiments.PYTESTER_COPY_EXAMPLE, stacklevel=2) + example_dir = self.request.config.getini("pytester_example_dir") + if example_dir is None: + raise ValueError("pytester_example_dir is unset, can't copy examples") + example_path = self.request.config.rootdir.join(example_dir, name) + if example_path.isdir() and not example_path.join("__init__.py").isfile(): + example_path.copy(self.tmpdir) + elif example_path.isfile(): + example_path.copy(self.tmpdir.join(example_path.basename)) + else: + raise LookupError("example is not found as a file or directory") + Session = Session def getnode(self, config, arg): diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 88c1f8740..6f12cc335 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -14,13 +14,7 @@ from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR class TestGeneralUsage(object): def test_config_error(self, testdir): - testdir.makeconftest( - """ - def pytest_configure(config): - import pytest - raise pytest.UsageError("hello") - """ - ) + testdir.copy_example("conftest_usageerror/conftest.py") result = testdir.runpytest(testdir.tmpdir) assert result.ret != 0 result.stderr.fnmatch_lines(["*ERROR: hello"]) @@ -170,18 +164,7 @@ class TestGeneralUsage(object): result.stdout.fnmatch_lines(["*1 skip*"]) def test_issue88_initial_file_multinodes(self, testdir): - testdir.makeconftest( - """ - import pytest - class MyFile(pytest.File): - def collect(self): - return [MyItem("hello", parent=self)] - def pytest_collect_file(path, parent): - return MyFile(path, parent) - class MyItem(pytest.Item): - pass - """ - ) + testdir.copy_example("issue88_initial_file_multinodes") p = testdir.makepyfile("def test_hello(): pass") result = testdir.runpytest(p, "--collect-only") result.stdout.fnmatch_lines(["*MyFile*test_issue88*", "*Module*test_issue88*"]) diff --git a/testing/example_scripts/conftest_usageerror/conftest.py b/testing/example_scripts/conftest_usageerror/conftest.py new file mode 100644 index 000000000..7d0d39d3a --- /dev/null +++ b/testing/example_scripts/conftest_usageerror/conftest.py @@ -0,0 +1,4 @@ +def pytest_configure(config): + import pytest + + raise pytest.UsageError("hello") diff --git a/testing/example_scripts/issue88_initial_file_multinodes/conftest.py b/testing/example_scripts/issue88_initial_file_multinodes/conftest.py new file mode 100644 index 000000000..aa5d87831 --- /dev/null +++ b/testing/example_scripts/issue88_initial_file_multinodes/conftest.py @@ -0,0 +1,14 @@ +import pytest + + +class MyFile(pytest.File): + def collect(self): + return [MyItem("hello", parent=self)] + + +def pytest_collect_file(path, parent): + return MyFile(path, parent) + + +class MyItem(pytest.Item): + pass diff --git a/testing/example_scripts/issue88_initial_file_multinodes/test_hello.py b/testing/example_scripts/issue88_initial_file_multinodes/test_hello.py new file mode 100644 index 000000000..56444d147 --- /dev/null +++ b/testing/example_scripts/issue88_initial_file_multinodes/test_hello.py @@ -0,0 +1,2 @@ +def test_hello(): + pass diff --git a/testing/examples/test_issue519.py b/testing/examples/test_issue519.py new file mode 100644 index 000000000..e83f18fdc --- /dev/null +++ b/testing/examples/test_issue519.py @@ -0,0 +1,3 @@ +def test_510(testdir): + testdir.copy_example("issue_519.py") + testdir.runpytest("issue_519.py") diff --git a/tox.ini b/tox.ini index e0ef2dccc..2a4f6566b 100644 --- a/tox.ini +++ b/tox.ini @@ -204,7 +204,9 @@ filterwarnings = ignore:.*type argument to addoption.*:DeprecationWarning # produced by python >=3.5 on execnet (pytest-xdist) ignore:.*inspect.getargspec.*deprecated, use inspect.signature.*:DeprecationWarning - + #pytests own futurewarnings + ignore::_pytest.experiments.PytestExerimentalApiWarning +pytester_example_dir = testing/example_scripts [flake8] max-line-length = 120 ignore = E203,W503