diff --git a/CHANGELOG b/CHANGELOG index a865f0cd6..cbd47878f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,8 @@ -Changes between 1.3.4 and 1.4.0a1 +Changes between 1.3.4 and 1.4.0.dev0 ================================================== +- introduce (customizable) assertion failure representations (Floris Bruynooghe) - major refactoring of internal collection handling - majorly reduce py.test core code, shift function/python testing to own plugin - fix issue88 (finding custom test nodes from command line arg) diff --git a/doc/example/assertion/failure_demo.py b/doc/example/assertion/failure_demo.py index 5fba99db7..f0266b82c 100644 --- a/doc/example/assertion/failure_demo.py +++ b/doc/example/assertion/failure_demo.py @@ -118,5 +118,43 @@ def test_dynamic_compile_shows_nicely(): module.foo() +class TestSpecialisedExplanations(object): + def test_eq_text(self): + assert 'spam' == 'eggs' + + def test_eq_similar_text(self): + assert 'foo 1 bar' == 'foo 2 bar' + + def test_eq_multiline_text(self): + assert 'foo\nspam\nbar' == 'foo\neggs\nbar' + + def test_eq_long_text(self): + a = '1'*100 + 'a' + '2'*100 + b = '1'*100 + 'b' + '2'*100 + assert a == b + + def test_eq_long_text_multiline(self): + a = '1\n'*100 + 'a' + '2\n'*100 + b = '1\n'*100 + 'b' + '2\n'*100 + assert a == b + + def test_eq_list(self): + assert [0, 1, 2] == [0, 1, 3] + + def test_eq_list_long(self): + a = [0]*100 + [1] + [3]*100 + b = [0]*100 + [2] + [3]*100 + assert a == b + + def test_eq_dict(self): + assert {'a': 0, 'b': 1} == {'a': 0, 'b': 2} + + def test_eq_set(self): + assert set([0, 10, 11, 12]) == set([0, 20, 21]) + + def test_in_list(self): + assert 1 in [0, 2, 3, 4, 5] + + def globf(x): return x+1 diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py index 0c7b0090e..2c40a93dc 100644 --- a/py/_code/_assertionnew.py +++ b/py/_code/_assertionnew.py @@ -108,10 +108,16 @@ unary_map = { class DebugInterpreter(ast.NodeVisitor): - """Interpret AST nodes to gleam useful debugging information.""" + """Interpret AST nodes to gleam useful debugging information. + + The _pytesthook attribute is used to detect if the py.test + pytest_assertion plugin is loaded and if so call it's hooks. + """ def __init__(self, frame): self.frame = frame + self._pytesthook = getattr(py.builtin.builtins.AssertionError, + "_pytesthook") def generic_visit(self, node): # Fallback when we don't have a special implementation. @@ -162,10 +168,7 @@ class DebugInterpreter(ast.NodeVisitor): def visit_Compare(self, comp): left = comp.left left_explanation, left_result = self.visit(left) - got_result = False for op, next_op in zip(comp.ops, comp.comparators): - if got_result and not result: - break next_explanation, next_result = self.visit(next_op) op_symbol = operator_map[op.__class__] explanation = "%s %s %s" % (left_explanation, op_symbol, @@ -177,9 +180,17 @@ class DebugInterpreter(ast.NodeVisitor): __exprinfo_right=next_result) except Exception: raise Failure(explanation) - else: - got_result = True + if not result: + break left_explanation, left_result = next_explanation, next_result + if self._pytesthook: + hook_result = self._pytesthook.pytest_assert_binrepr( + op=op_symbol, left=left_result, right=next_result) + if hook_result: + for new_expl in hook_result: + if new_expl: + explanation = '\n~'.join(new_expl) + break return explanation, result def visit_BoolOp(self, boolop): diff --git a/py/_code/assertion.py b/py/_code/assertion.py index efedec63d..adbbce7c5 100644 --- a/py/_code/assertion.py +++ b/py/_code/assertion.py @@ -5,12 +5,20 @@ BuiltinAssertionError = py.builtin.builtins.AssertionError def _format_explanation(explanation): - # uck! See CallFunc for where \n{ and \n} escape sequences are used + """This formats an explanation + + Normally all embedded newlines are escaped, however there are + three exceptions: \n{, \n} and \n~. The first two are intended + cover nested explanations, see function and attribute explanations + for examples (.visit_Call(), visit_Attribute()). The last one is + for when one explanation needs to span multiple lines, e.g. when + displaying diffs. + """ raw_lines = (explanation or '').split('\n') - # escape newlines not followed by { and } + # escape newlines not followed by {, } and ~ lines = [raw_lines[0]] for l in raw_lines[1:]: - if l.startswith('{') or l.startswith('}'): + if l.startswith('{') or l.startswith('}') or l.startswith('~'): lines.append(l) else: lines[-1] += '\\n' + l @@ -28,11 +36,14 @@ def _format_explanation(explanation): stackcnt[-1] += 1 stackcnt.append(0) result.append(' +' + ' '*(len(stack)-1) + s + line[1:]) - else: + elif line.startswith('}'): assert line.startswith('}') stack.pop() stackcnt.pop() result[stack[-1]] += line[1:] + else: + assert line.startswith('~') + result.append(' '*len(stack) + line[1:]) assert len(stack) == 1 return '\n'.join(result) diff --git a/py/_plugin/hookspec.py b/py/_plugin/hookspec.py index 3b75d8925..315da074a 100644 --- a/py/_plugin/hookspec.py +++ b/py/_plugin/hookspec.py @@ -147,6 +147,20 @@ def pytest_sessionfinish(session, exitstatus): """ whole test run finishes. """ +# ------------------------------------------------------------------------- +# hooks for customising the assert methods +# ------------------------------------------------------------------------- + +def pytest_assert_binrepr(op, left, right): + """Customise explanation for binary operators + + Return None or an empty list for no custom explanation, otherwise + return a list of strings. The strings will be joined by newlines + but any newlines *in* a string will be escaped. Note that all but + the first line will be indented sligthly, the intention is for the + first line to be a summary. + """ + # ------------------------------------------------------------------------- # hooks for influencing reporting (invoked from pytest_terminal) # ------------------------------------------------------------------------- diff --git a/py/_plugin/pytest_assertion.py b/py/_plugin/pytest_assertion.py index f18350e7c..b58b91985 100644 --- a/py/_plugin/pytest_assertion.py +++ b/py/_plugin/pytest_assertion.py @@ -8,10 +8,15 @@ def pytest_addoption(parser): help="disable python assert expression reinterpretation."), def pytest_configure(config): + # The _pytesthook attribute on the AssertionError is used by + # py._code._assertionnew to detect this plugin was loaded and in + # turn call the hooks defined here as part of the + # DebugInterpreter. if not config.getvalue("noassert") and not config.getvalue("nomagic"): warn_about_missing_assertion() config._oldassertion = py.builtin.builtins.AssertionError py.builtin.builtins.AssertionError = py.code._AssertionError + py.builtin.builtins.AssertionError._pytesthook = config.hook def pytest_unconfigure(config): if hasattr(config, '_oldassertion'): @@ -26,3 +31,109 @@ def warn_about_missing_assertion(): else: py.std.warnings.warn("Assertions are turned off!" " (are you using python -O?)") + + +# Provide basestring in python3 +try: + basestring = basestring +except NameError: + basestring = str + + +def pytest_assert_binrepr(op, left, right): + """Make specialised explanations for some operators/operands""" + left_repr = py.io.saferepr(left, maxsize=30) + right_repr = py.io.saferepr(right, maxsize=30) + summary = '%s %s %s' % (left_repr, op, right_repr) + + issequence = lambda x: isinstance(x, (list, tuple)) + istext = lambda x: isinstance(x, basestring) + isdict = lambda x: isinstance(x, dict) + isset = lambda x: isinstance(x, set) + + explanation = None + if op == '==': + if istext(left) and istext(right): + explanation = _diff_text(left, right) + elif issequence(left) and issequence(right): + explanation = _compare_eq_sequence(left, right) + elif isset(left) and isset(right): + explanation = _compare_eq_set(left, right) + elif isdict(left) and isdict(right): + explanation = _diff_text(py.std.pprint.pformat(left), + py.std.pprint.pformat(right)) + elif op == 'in': + pass # XXX + + if not explanation: + return None + + # Don't include pageloads of data, should be configurable + if len(''.join(explanation)) > 80*8: + explanation = ['Detailed information too verbose, truncated'] + + return [summary] + explanation + + +def _diff_text(left, right): + """Return the explanation for the diff between text + + This will skip leading and trailing characters which are + identical to keep the diff minimal. + """ + explanation = [] + for i in range(min(len(left), len(right))): + if left[i] != right[i]: + break + if i > 42: + i -= 10 # Provide some context + explanation = ['Skipping %s identical ' + 'leading characters in diff' % i] + left = left[i:] + right = right[i:] + if len(left) == len(right): + for i in range(len(left)): + if left[-i] != right[-i]: + break + if i > 42: + i -= 10 # Provide some context + explanation += ['Skipping %s identical ' + 'trailing characters in diff' % i] + left = left[:-i] + right = right[:-i] + explanation += [line.strip('\n') + for line in py.std.difflib.ndiff(left.splitlines(), + right.splitlines())] + return explanation + + +def _compare_eq_sequence(left, right): + explanation = [] + for i in range(min(len(left), len(right))): + if left[i] != right[i]: + explanation += ['First differing item %s: %s != %s' % + (i, left[i], right[i])] + break + if len(left) > len(right): + explanation += ['Left contains more items, ' + 'first extra item: %s' % left[len(right)]] + elif len(left) < len(right): + explanation += ['Right contains more items, ' + 'first extra item: %s' % right[len(left)]] + return explanation + _diff_text(py.std.pprint.pformat(left), + py.std.pprint.pformat(right)) + + +def _compare_eq_set(left, right): + explanation = [] + diff_left = left - right + diff_right = right - left + if diff_left: + explanation.append('Extra items in the left set:') + for item in diff_left: + explanation.append(py.io.saferepr(item)) + if diff_right: + explanation.append('Extra items in the right set:') + for item in diff_right: + explanation.append(py.io.saferepr(item)) + return explanation diff --git a/testing/plugin/test_pytest_assertion.py b/testing/plugin/test_pytest_assertion.py index c76a93b4d..0dd64f473 100644 --- a/testing/plugin/test_pytest_assertion.py +++ b/testing/plugin/test_pytest_assertion.py @@ -1,3 +1,32 @@ +import sys + +import py +import py._plugin.pytest_assertion as plugin + + +def getframe(): + """Return the frame of the caller as a py.code.Frame object""" + return py.code.Frame(sys._getframe(1)) + +def interpret(expr, frame): + anew = py.test.importorskip('py._code._assertionnew') + return anew.interpret(expr, frame) + +def pytest_funcarg__hook(request): + class MockHook(object): + def __init__(self): + self.called = False + self.args = tuple() + self.kwargs = dict() + + def __call__(self, op, left, right): + self.called = True + self.op = op + self.left = left + self.right = right + return MockHook() + + def test_functional(testdir): testdir.makepyfile(""" def test_hello(): @@ -49,3 +78,57 @@ def test_traceback_failure(testdir): "*test_traceback_failure.py:4: AssertionError" ]) + +def test_pytest_assert_binrepr_called(monkeypatch, hook): + monkeypatch.setattr(py._plugin.pytest_assertion, + 'pytest_assert_binrepr', hook) + interpret('assert 0 == 1', getframe()) + assert hook.called + + +def test_pytest_assert_binrepr_args(monkeypatch, hook): + monkeypatch.setattr(py._plugin.pytest_assertion, + 'pytest_assert_binrepr', hook) + interpret('assert [0, 1] == [0, 2]', getframe()) + assert hook.op == '==' + assert hook.left == [0, 1] + assert hook.right == [0, 2] + + +class TestAssertCompare: + def test_different_types(self): + assert plugin.pytest_assert_binrepr('==', [0, 1], 'foo') is None + + def test_summary(self): + summary = plugin.pytest_assert_binrepr('==', [0, 1], [0, 2])[0] + assert len(summary) < 65 + + def test_text_diff(self): + diff = plugin.pytest_assert_binrepr('==', 'spam', 'eggs')[1:] + assert '- spam' in diff + assert '+ eggs' in diff + + def test_multiline_text_diff(self): + left = 'foo\nspam\nbar' + right = 'foo\neggs\nbar' + diff = plugin.pytest_assert_binrepr('==', left, right) + assert '- spam' in diff + assert '+ eggs' in diff + + def test_list(self): + expl = plugin.pytest_assert_binrepr('==', [0, 1], [0, 2]) + assert len(expl) > 1 + + def test_list_different_lenghts(self): + expl = plugin.pytest_assert_binrepr('==', [0, 1], [0, 1, 2]) + assert len(expl) > 1 + expl = plugin.pytest_assert_binrepr('==', [0, 1, 2], [0, 1]) + assert len(expl) > 1 + + def test_dict(self): + expl = plugin.pytest_assert_binrepr('==', {'a': 0}, {'a': 1}) + assert len(expl) > 1 + + def test_set(self): + expl = plugin.pytest_assert_binrepr('==', set([0, 1]), set([0, 2])) + assert len(expl) > 1