Merge pull request #3623 from RonnyPfannschmidt/pytester-runexamples

Pytester runexamples
This commit is contained in:
Bruno Oliveira 2018-06-27 09:13:29 -03:00 committed by GitHub
commit ea379ba10f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 147 additions and 76 deletions

View File

@ -0,0 +1 @@
introduce ``pytester.copy_example`` as helper to do acceptance tests against examples from the project

View File

@ -32,7 +32,6 @@ get on the terminal - we are working on that)::
self = <failure_demo.TestFailing object at 0xdeadbeef>
def test_simple(self):
def f():
return 42
@ -44,7 +43,7 @@ get on the terminal - we are working on that)::
E + where 42 = <function TestFailing.test_simple.<locals>.f at 0xdeadbeef>()
E + and 43 = <function TestFailing.test_simple.<locals>.g at 0xdeadbeef>()
failure_demo.py:37: AssertionError
failure_demo.py:35: AssertionError
____________________ TestFailing.test_simple_multiline _____________________
self = <failure_demo.TestFailing object at 0xdeadbeef>
@ -52,7 +51,7 @@ get on the terminal - we are working on that)::
def test_simple_multiline(self):
> otherfunc_multi(42, 6 * 9)
failure_demo.py:40:
failure_demo.py:38:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
a = 42, b = 54
@ -67,7 +66,6 @@ get on the terminal - we are working on that)::
self = <failure_demo.TestFailing object at 0xdeadbeef>
def test_not(self):
def f():
return 42
@ -75,7 +73,7 @@ get on the terminal - we are working on that)::
E assert not 42
E + where 42 = <function TestFailing.test_not.<locals>.f at 0xdeadbeef>()
failure_demo.py:47: AssertionError
failure_demo.py:44: AssertionError
_________________ TestSpecialisedExplanations.test_eq_text _________________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -86,7 +84,7 @@ get on the terminal - we are working on that)::
E - spam
E + eggs
failure_demo.py:53: AssertionError
failure_demo.py:49: AssertionError
_____________ TestSpecialisedExplanations.test_eq_similar_text _____________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -99,7 +97,7 @@ get on the terminal - we are working on that)::
E + foo 2 bar
E ? ^
failure_demo.py:56: AssertionError
failure_demo.py:52: AssertionError
____________ TestSpecialisedExplanations.test_eq_multiline_text ____________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -112,7 +110,7 @@ get on the terminal - we are working on that)::
E + eggs
E bar
failure_demo.py:59: AssertionError
failure_demo.py:55: AssertionError
______________ TestSpecialisedExplanations.test_eq_long_text _______________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -129,7 +127,7 @@ get on the terminal - we are working on that)::
E + 1111111111b222222222
E ? ^
failure_demo.py:64: AssertionError
failure_demo.py:60: AssertionError
_________ TestSpecialisedExplanations.test_eq_long_text_multiline __________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -149,7 +147,7 @@ get on the terminal - we are working on that)::
E
E ...Full output truncated (7 lines hidden), use '-vv' to show
failure_demo.py:69: AssertionError
failure_demo.py:65: AssertionError
_________________ TestSpecialisedExplanations.test_eq_list _________________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -160,7 +158,7 @@ get on the terminal - we are working on that)::
E At index 2 diff: 2 != 3
E Use -v to get the full diff
failure_demo.py:72: AssertionError
failure_demo.py:68: AssertionError
______________ TestSpecialisedExplanations.test_eq_list_long _______________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -173,7 +171,7 @@ get on the terminal - we are working on that)::
E At index 100 diff: 1 != 2
E Use -v to get the full diff
failure_demo.py:77: AssertionError
failure_demo.py:73: AssertionError
_________________ TestSpecialisedExplanations.test_eq_dict _________________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -191,7 +189,7 @@ get on the terminal - we are working on that)::
E
E ...Full output truncated (2 lines hidden), use '-vv' to show
failure_demo.py:80: AssertionError
failure_demo.py:76: AssertionError
_________________ TestSpecialisedExplanations.test_eq_set __________________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -209,7 +207,7 @@ get on the terminal - we are working on that)::
E
E ...Full output truncated (2 lines hidden), use '-vv' to show
failure_demo.py:83: AssertionError
failure_demo.py:79: AssertionError
_____________ TestSpecialisedExplanations.test_eq_longer_list ______________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -220,7 +218,7 @@ get on the terminal - we are working on that)::
E Right contains more items, first extra item: 3
E Use -v to get the full diff
failure_demo.py:86: AssertionError
failure_demo.py:82: AssertionError
_________________ TestSpecialisedExplanations.test_in_list _________________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -229,7 +227,7 @@ get on the terminal - we are working on that)::
> assert 1 in [0, 2, 3, 4, 5]
E assert 1 in [0, 2, 3, 4, 5]
failure_demo.py:89: AssertionError
failure_demo.py:85: AssertionError
__________ TestSpecialisedExplanations.test_not_in_text_multiline __________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -248,7 +246,7 @@ get on the terminal - we are working on that)::
E
E ...Full output truncated (2 lines hidden), use '-vv' to show
failure_demo.py:93: AssertionError
failure_demo.py:89: AssertionError
___________ TestSpecialisedExplanations.test_not_in_text_single ____________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -261,7 +259,7 @@ get on the terminal - we are working on that)::
E single foo line
E ? +++
failure_demo.py:97: AssertionError
failure_demo.py:93: AssertionError
_________ TestSpecialisedExplanations.test_not_in_text_single_long _________
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -274,7 +272,7 @@ get on the terminal - we are working on that)::
E head head foo tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail
E ? +++
failure_demo.py:101: AssertionError
failure_demo.py:97: AssertionError
______ TestSpecialisedExplanations.test_not_in_text_single_long_term _______
self = <failure_demo.TestSpecialisedExplanations object at 0xdeadbeef>
@ -287,11 +285,10 @@ get on the terminal - we are working on that)::
E head head fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffftail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail
E ? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
failure_demo.py:105: AssertionError
failure_demo.py:101: AssertionError
______________________________ test_attribute ______________________________
def test_attribute():
class Foo(object):
b = 1
@ -300,11 +297,10 @@ get on the terminal - we are working on that)::
E assert 1 == 2
E + where 1 = <failure_demo.test_attribute.<locals>.Foo object at 0xdeadbeef>.b
failure_demo.py:114: AssertionError
failure_demo.py:109: AssertionError
_________________________ test_attribute_instance __________________________
def test_attribute_instance():
class Foo(object):
b = 1
@ -313,13 +309,11 @@ get on the terminal - we are working on that)::
E + where 1 = <failure_demo.test_attribute_instance.<locals>.Foo object at 0xdeadbeef>.b
E + where <failure_demo.test_attribute_instance.<locals>.Foo object at 0xdeadbeef> = <class 'failure_demo.test_attribute_instance.<locals>.Foo'>()
failure_demo.py:122: AssertionError
failure_demo.py:116: AssertionError
__________________________ test_attribute_failure __________________________
def test_attribute_failure():
class Foo(object):
def _get_b(self):
raise Exception("Failed to get attrib")
@ -328,7 +322,7 @@ get on the terminal - we are working on that)::
i = Foo()
> assert i.b == 2
failure_demo.py:135:
failure_demo.py:127:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <failure_demo.test_attribute_failure.<locals>.Foo object at 0xdeadbeef>
@ -337,11 +331,10 @@ get on the terminal - we are working on that)::
> raise Exception("Failed to get attrib")
E Exception: Failed to get attrib
failure_demo.py:130: Exception
failure_demo.py:122: Exception
_________________________ test_attribute_multiple __________________________
def test_attribute_multiple():
class Foo(object):
b = 1
@ -355,7 +348,7 @@ get on the terminal - we are working on that)::
E + and 2 = <failure_demo.test_attribute_multiple.<locals>.Bar object at 0xdeadbeef>.b
E + where <failure_demo.test_attribute_multiple.<locals>.Bar object at 0xdeadbeef> = <class 'failure_demo.test_attribute_multiple.<locals>.Bar'>()
failure_demo.py:146: AssertionError
failure_demo.py:137: AssertionError
__________________________ TestRaises.test_raises __________________________
self = <failure_demo.TestRaises object at 0xdeadbeef>
@ -364,13 +357,13 @@ get on the terminal - we are working on that)::
s = "qwe" # NOQA
> raises(TypeError, "int(s)")
failure_demo.py:157:
failure_demo.py:147:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> int(s)
E ValueError: invalid literal for int() with base 10: 'qwe'
<0-codegen $PYTHON_PREFIX/lib/python3.5/site-packages/_pytest/python_api.py:634>:1: ValueError
<0-codegen $PYTHON_PREFIX/lib/python3.5/site-packages/_pytest/python_api.py:635>:1: ValueError
______________________ TestRaises.test_raises_doesnt _______________________
self = <failure_demo.TestRaises object at 0xdeadbeef>
@ -379,7 +372,7 @@ get on the terminal - we are working on that)::
> raises(IOError, "int('3')")
E Failed: DID NOT RAISE <class 'OSError'>
failure_demo.py:160: Failed
failure_demo.py:150: Failed
__________________________ TestRaises.test_raise ___________________________
self = <failure_demo.TestRaises object at 0xdeadbeef>
@ -388,7 +381,7 @@ get on the terminal - we are working on that)::
> raise ValueError("demo error")
E ValueError: demo error
failure_demo.py:163: ValueError
failure_demo.py:153: ValueError
________________________ TestRaises.test_tupleerror ________________________
self = <failure_demo.TestRaises object at 0xdeadbeef>
@ -397,7 +390,7 @@ get on the terminal - we are working on that)::
> a, b = [1] # NOQA
E ValueError: not enough values to unpack (expected 2, got 1)
failure_demo.py:166: ValueError
failure_demo.py:156: ValueError
______ TestRaises.test_reinterpret_fails_with_print_for_the_fun_of_it ______
self = <failure_demo.TestRaises object at 0xdeadbeef>
@ -408,7 +401,7 @@ get on the terminal - we are working on that)::
> a, b = items.pop()
E TypeError: 'int' object is not iterable
failure_demo.py:171: TypeError
failure_demo.py:161: TypeError
--------------------------- Captured stdout call ---------------------------
items is [1, 2, 3]
________________________ TestRaises.test_some_error ________________________
@ -419,7 +412,7 @@ get on the terminal - we are working on that)::
> if namenotexi: # NOQA
E NameError: name 'namenotexi' is not defined
failure_demo.py:174: NameError
failure_demo.py:164: NameError
____________________ test_dynamic_compile_shows_nicely _____________________
def test_dynamic_compile_shows_nicely():
@ -434,20 +427,19 @@ get on the terminal - we are working on that)::
sys.modules[name] = module
> module.foo()
failure_demo.py:192:
failure_demo.py:182:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def foo():
> assert 1 == 0
E AssertionError
<2-codegen 'abc-123' $REGENDOC_TMPDIR/assertion/failure_demo.py:189>:2: AssertionError
<2-codegen 'abc-123' $REGENDOC_TMPDIR/assertion/failure_demo.py:179>:2: AssertionError
____________________ TestMoreErrors.test_complex_error _____________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
def test_complex_error(self):
def f():
return 44
@ -456,7 +448,7 @@ get on the terminal - we are working on that)::
> somefunc(f(), g())
failure_demo.py:205:
failure_demo.py:193:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
failure_demo.py:11: in somefunc
otherfunc(x, y)
@ -478,7 +470,7 @@ get on the terminal - we are working on that)::
> a, b = items
E ValueError: not enough values to unpack (expected 2, got 0)
failure_demo.py:209: ValueError
failure_demo.py:197: ValueError
____________________ TestMoreErrors.test_z2_type_error _____________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
@ -488,7 +480,7 @@ get on the terminal - we are working on that)::
> a, b = items
E TypeError: 'int' object is not iterable
failure_demo.py:213: TypeError
failure_demo.py:201: TypeError
______________________ TestMoreErrors.test_startswith ______________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
@ -501,13 +493,12 @@ get on the terminal - we are working on that)::
E + where False = <built-in method startswith of str object at 0xdeadbeef>('456')
E + where <built-in method startswith of str object at 0xdeadbeef> = '123'.startswith
failure_demo.py:218: AssertionError
failure_demo.py:206: AssertionError
__________________ TestMoreErrors.test_startswith_nested ___________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
def test_startswith_nested(self):
def f():
return "123"
@ -521,7 +512,7 @@ get on the terminal - we are working on that)::
E + where '123' = <function TestMoreErrors.test_startswith_nested.<locals>.f at 0xdeadbeef>()
E + and '456' = <function TestMoreErrors.test_startswith_nested.<locals>.g at 0xdeadbeef>()
failure_demo.py:228: AssertionError
failure_demo.py:215: AssertionError
_____________________ TestMoreErrors.test_global_func ______________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
@ -532,7 +523,7 @@ get on the terminal - we are working on that)::
E + where False = isinstance(43, float)
E + where 43 = globf(42)
failure_demo.py:231: AssertionError
failure_demo.py:218: AssertionError
_______________________ TestMoreErrors.test_instance _______________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
@ -543,7 +534,7 @@ get on the terminal - we are working on that)::
E assert 42 != 42
E + where 42 = <failure_demo.TestMoreErrors object at 0xdeadbeef>.x
failure_demo.py:235: AssertionError
failure_demo.py:222: AssertionError
_______________________ TestMoreErrors.test_compare ________________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
@ -553,7 +544,7 @@ get on the terminal - we are working on that)::
E assert 11 < 5
E + where 11 = globf(10)
failure_demo.py:238: AssertionError
failure_demo.py:225: AssertionError
_____________________ TestMoreErrors.test_try_finally ______________________
self = <failure_demo.TestMoreErrors object at 0xdeadbeef>
@ -564,13 +555,12 @@ get on the terminal - we are working on that)::
> assert x == 0
E assert 1 == 0
failure_demo.py:243: AssertionError
failure_demo.py:230: AssertionError
___________________ TestCustomAssertMsg.test_single_line ___________________
self = <failure_demo.TestCustomAssertMsg object at 0xdeadbeef>
def test_single_line(self):
class A(object):
a = 1
@ -580,13 +570,12 @@ get on the terminal - we are working on that)::
E assert 1 == 2
E + where 1 = <class 'failure_demo.TestCustomAssertMsg.test_single_line.<locals>.A'>.a
failure_demo.py:256: AssertionError
failure_demo.py:241: AssertionError
____________________ TestCustomAssertMsg.test_multiline ____________________
self = <failure_demo.TestCustomAssertMsg object at 0xdeadbeef>
def test_multiline(self):
class A(object):
a = 1
@ -600,13 +589,12 @@ get on the terminal - we are working on that)::
E assert 1 == 2
E + where 1 = <class 'failure_demo.TestCustomAssertMsg.test_multiline.<locals>.A'>.a
failure_demo.py:264: AssertionError
failure_demo.py:248: AssertionError
___________________ TestCustomAssertMsg.test_custom_repr ___________________
self = <failure_demo.TestCustomAssertMsg object at 0xdeadbeef>
def test_custom_repr(self):
class JSON(object):
a = 1
@ -623,7 +611,7 @@ get on the terminal - we are working on that)::
E assert 1 == 2
E + where 1 = This is JSON\n{\n 'foo': 'bar'\n}.a
failure_demo.py:278: AssertionError
failure_demo.py:261: AssertionError
============================= warnings summary =============================
<undetermined location>
Metafunc.addcall is deprecated and scheduled to be removed in pytest 4.0.

View File

@ -386,11 +386,52 @@ return a result object, with which we can assert the tests' outcomes.
result.assert_outcomes(passed=4)
additionally it is possible to copy examples for a example folder before running pytest on it
.. code:: ini
# content of pytest.ini
[pytest]
pytester_example_dir = .
.. code:: python
# content of test_example.py
def test_plugin(testdir):
testdir.copy_example("test_example.py")
testdir.runpytest("-k", "test_example")
def test_example():
pass
.. code::
$ pytest
=========================== test session starts ============================
platform linux -- Python 3.x.y, pytest-3.x.y, py-1.x.y, pluggy-0.x.y
rootdir: $REGENDOC_TMPDIR, inifile: pytest.ini
collected 2 items
test_example.py .. [100%]
============================= warnings summary =============================
test_example.py::test_plugin
$REGENDOC_TMPDIR/test_example.py:4: PytestExerimentalApiWarning: testdir.copy_example is an experimental api that may change over time
testdir.copy_example("test_example.py")
-- Docs: http://doc.pytest.org/en/latest/warnings.html
=================== 2 passed, 1 warnings in 0.12 seconds ===================
For more information about the result object that ``runpytest()`` returns, and
the methods that it provides please check out the :py:class:`RunResult
<_pytest.pytester.RunResult>` documentation.
.. _`writinghooks`:
Writing hook functions

View File

@ -0,0 +1,13 @@
class PytestExerimentalApiWarning(FutureWarning):
"warning category used to denote experiments in pytest"
@classmethod
def simple(cls, apiname):
return cls(
"{apiname} is an experimental api that may change over time".format(
apiname=apiname
)
)
PYTESTER_COPY_EXAMPLE = PytestExerimentalApiWarning.simple("testdir.copy_example")

View File

@ -53,6 +53,10 @@ def pytest_addoption(parser):
),
)
parser.addini(
"pytester_example_dir", help="directory to take the pytester example files from"
)
def pytest_configure(config):
if config.getvalue("lsof"):
@ -628,6 +632,22 @@ class Testdir(object):
p.ensure("__init__.py")
return p
def copy_example(self, name):
from . import experiments
import warnings
warnings.warn(experiments.PYTESTER_COPY_EXAMPLE, stacklevel=2)
example_dir = self.request.config.getini("pytester_example_dir")
if example_dir is None:
raise ValueError("pytester_example_dir is unset, can't copy examples")
example_path = self.request.config.rootdir.join(example_dir, name)
if example_path.isdir() and not example_path.join("__init__.py").isfile():
example_path.copy(self.tmpdir)
elif example_path.isfile():
example_path.copy(self.tmpdir.join(example_path.basename))
else:
raise LookupError("example is not found as a file or directory")
Session = Session
def getnode(self, config, arg):

View File

@ -14,13 +14,7 @@ from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
class TestGeneralUsage(object):
def test_config_error(self, testdir):
testdir.makeconftest(
"""
def pytest_configure(config):
import pytest
raise pytest.UsageError("hello")
"""
)
testdir.copy_example("conftest_usageerror/conftest.py")
result = testdir.runpytest(testdir.tmpdir)
assert result.ret != 0
result.stderr.fnmatch_lines(["*ERROR: hello"])
@ -170,18 +164,7 @@ class TestGeneralUsage(object):
result.stdout.fnmatch_lines(["*1 skip*"])
def test_issue88_initial_file_multinodes(self, testdir):
testdir.makeconftest(
"""
import pytest
class MyFile(pytest.File):
def collect(self):
return [MyItem("hello", parent=self)]
def pytest_collect_file(path, parent):
return MyFile(path, parent)
class MyItem(pytest.Item):
pass
"""
)
testdir.copy_example("issue88_initial_file_multinodes")
p = testdir.makepyfile("def test_hello(): pass")
result = testdir.runpytest(p, "--collect-only")
result.stdout.fnmatch_lines(["*MyFile*test_issue88*", "*Module*test_issue88*"])

View File

@ -0,0 +1,4 @@
def pytest_configure(config):
import pytest
raise pytest.UsageError("hello")

View File

@ -0,0 +1,14 @@
import pytest
class MyFile(pytest.File):
def collect(self):
return [MyItem("hello", parent=self)]
def pytest_collect_file(path, parent):
return MyFile(path, parent)
class MyItem(pytest.Item):
pass

View File

@ -0,0 +1,2 @@
def test_hello():
pass

View File

@ -0,0 +1,3 @@
def test_510(testdir):
testdir.copy_example("issue_519.py")
testdir.runpytest("issue_519.py")

View File

@ -204,7 +204,9 @@ filterwarnings =
ignore:.*type argument to addoption.*:DeprecationWarning
# produced by python >=3.5 on execnet (pytest-xdist)
ignore:.*inspect.getargspec.*deprecated, use inspect.signature.*:DeprecationWarning
#pytests own futurewarnings
ignore::_pytest.experiments.PytestExerimentalApiWarning
pytester_example_dir = testing/example_scripts
[flake8]
max-line-length = 120
ignore = E203,W503