From 379390a8aab7f20039346fe44e566232246598f5 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 11 May 2010 22:54:04 +0200 Subject: [PATCH] remove code.new() function and store lines directly into linecache.cache instead. This avoids the need for custom code objects, improving compatibility for jython and pypy-c. --HG-- branch : trunk --- CHANGELOG | 5 ++ py/_builtin.py | 5 +- py/_code/code.py | 62 +++-------------------- py/_code/source.py | 60 ++++++++-------------- testing/code/test_code.py | 75 ---------------------------- testing/code/test_excinfo.py | 1 - testing/code/test_source.py | 14 ++---- testing/plugin/test_pytest_runner.py | 6 ++- 8 files changed, 43 insertions(+), 185 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 34dc8c4a3..e6e3eace1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -11,6 +11,11 @@ Changes between 1.3.0 and 1.3.1 - improve tracebacks presentation: - raises shows shorter more relevant tracebacks +- improve support for raises and other dynamically compiled code by + manipulating python's linecache.cache instead of the previous + rather hacky way of creating custom code objects. This makes + it seemlessly work on Jython and PyPy at least. + Changes between 1.2.1 and 1.3.0 ================================================== diff --git a/py/_builtin.py b/py/_builtin.py index 4df3daae4..b9395a8de 100644 --- a/py/_builtin.py +++ b/py/_builtin.py @@ -151,7 +151,10 @@ else: return getattr(function, "__dict__", None) def _getcode(function): - return getattr(function, "func_code", None) + try: + return getattr(function, "__code__") + except AttributeError: + return getattr(function, "func_code", None) def print_(*args, **kwargs): """ minimal backport of py3k print statement. """ diff --git a/py/_code/code.py b/py/_code/code.py index e8bddb904..efe422a53 100644 --- a/py/_code/code.py +++ b/py/_code/code.py @@ -23,64 +23,14 @@ class Code(object): def __ne__(self, other): return not self == other - def new(self, rec=False, **kwargs): - """ return new code object with modified attributes. - if rec-cursive is true then dive into code - objects contained in co_consts. - """ - if sys.platform.startswith("java"): - # XXX jython does not support the below co_filename hack - return self.raw - names = [x for x in dir(self.raw) if x[:3] == 'co_'] - for name in kwargs: - if name not in names: - raise TypeError("unknown code attribute: %r" %(name, )) - if rec and hasattr(self.raw, 'co_consts'): # jython - newconstlist = [] - co = self.raw - cotype = type(co) - for c in co.co_consts: - if isinstance(c, cotype): - c = self.__class__(c).new(rec=True, **kwargs) - newconstlist.append(c) - return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) - for name in names: - if name not in kwargs: - kwargs[name] = getattr(self.raw, name) - arglist = [ - kwargs['co_argcount'], - kwargs['co_nlocals'], - kwargs.get('co_stacksize', 0), # jython - kwargs.get('co_flags', 0), # jython - kwargs.get('co_code', ''), # jython - kwargs.get('co_consts', ()), # jython - kwargs.get('co_names', []), # - kwargs['co_varnames'], - kwargs['co_filename'], - kwargs['co_name'], - kwargs['co_firstlineno'], - kwargs.get('co_lnotab', ''), #jython - kwargs.get('co_freevars', None), #jython - kwargs.get('co_cellvars', None), # jython - ] - if sys.version_info >= (3,0): - arglist.insert(1, kwargs['co_kwonlyargcount']) - return self.raw.__class__(*arglist) - else: - return py.std.new.code(*arglist) - def path(self): """ return a path object pointing to source code""" - fn = self.raw.co_filename - try: - return fn.__path__ - except AttributeError: - p = py.path.local(self.raw.co_filename) - if not p.check(): - # XXX maybe try harder like the weird logic - # in the standard lib [linecache.updatecache] does? - p = self.raw.co_filename - return p + p = py.path.local(self.raw.co_filename) + if not p.check(): + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + p = self.raw.co_filename + return p path = property(path, None, None, "path of this code object") diff --git a/py/_code/source.py b/py/_code/source.py index ffe78716f..9c25f545c 100644 --- a/py/_code/source.py +++ b/py/_code/source.py @@ -212,10 +212,10 @@ class Source(object): else: if flag & _AST_FLAG: return co - co_filename = MyStr(filename) - co_filename.__source__ = self - return py.code.Code(co).new(rec=1, co_filename=co_filename) - #return newcode_withfilename(co, co_filename) + from types import ModuleType + lines = [(x + "\n") for x in self.lines] + py.std.linecache.cache[filename] = (1, None, lines, filename) + return co # # public API shortcut functions @@ -224,11 +224,9 @@ class Source(object): def compile_(source, filename=None, mode='exec', flags= generators.compiler_flag, dont_inherit=0): """ compile the given source to a raw code object, - which points back to the source code through - "co_filename.__source__". All code objects - contained in the code object will recursively - also have this special subclass-of-string - filename. + and maintain an internal cache which allows later + retrieval of the source code for the code object + and any recursively created code objects. """ if _ast is not None and isinstance(source, _ast.AST): # XXX should Source support having AST? @@ -262,44 +260,26 @@ def getfslineno(obj): # # helper functions # -class MyStr(str): - """ custom string which allows to add attributes. """ def findsource(obj): - obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - sourcelines, lineno = py.std.inspect.findsource(obj) - except (KeyboardInterrupt, SystemExit): - raise - except: - return None, None - source = Source() - source.lines = [line.rstrip() for line in sourcelines] - return source, lineno - else: - lineno = obj.co_firstlineno - 1 - return fullsource, lineno - + sourcelines, lineno = py.std.inspect.findsource(obj) + except (KeyboardInterrupt, SystemExit): + raise + except: + return None, None + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno def getsource(obj, **kwargs): obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - strsrc = inspect.getsource(obj) - except IndentationError: - strsrc = "\"Buggy python version consider upgrading, cannot get source\"" - assert isinstance(strsrc, str) - return Source(strsrc, **kwargs) - else: - lineno = obj.co_firstlineno - 1 - end = fullsource.getblockend(lineno) - return Source(fullsource[lineno:end+1], deident=True) - + strsrc = inspect.getsource(obj) + except IndentationError: + strsrc = "\"Buggy python version consider upgrading, cannot get source\"" + assert isinstance(strsrc, str) + return Source(strsrc, **kwargs) def deindent(lines, offset=None): if offset is None: diff --git a/testing/code/test_code.py b/testing/code/test_code.py index feb88ebea..13c1fc646 100644 --- a/testing/code/test_code.py +++ b/testing/code/test_code.py @@ -1,87 +1,12 @@ -from __future__ import generators import py import sys -failsonjython = py.test.mark.xfail("sys.platform.startswith('java')") - -def test_newcode(): - source = "i = 3" - co = compile(source, '', 'exec') - code = py.code.Code(co) - newco = code.new() - assert co == newco - def test_ne(): code1 = py.code.Code(compile('foo = "bar"', '', 'exec')) assert code1 == code1 code2 = py.code.Code(compile('foo = "baz"', '', 'exec')) assert code2 != code1 -@failsonjython -def test_newcode_unknown_args(): - code = py.code.Code(compile("", '', 'exec')) - py.test.raises(TypeError, 'code.new(filename="hello")') - -@failsonjython -def test_newcode_withfilename(): - source = py.code.Source(""" - def f(): - def g(): - pass - """) - co = compile(str(source)+'\n', 'nada', 'exec') - obj = 'hello' - newco = py.code.Code(co).new(rec=True, co_filename=obj) - def walkcode(co): - for x in co.co_consts: - if isinstance(x, type(co)): - for y in walkcode(x): - yield y - yield co - - names = [] - for code in walkcode(newco): - assert newco.co_filename == obj - assert newco.co_filename is obj - names.append(code.co_name) - assert 'f' in names - assert 'g' in names - -@failsonjython -def test_newcode_with_filename(): - source = "i = 3" - co = compile(source, '', 'exec') - code = py.code.Code(co) - class MyStr(str): - pass - filename = MyStr("hello") - filename.__source__ = py.code.Source(source) - newco = code.new(rec=True, co_filename=filename) - assert newco.co_filename.__source__ == filename.__source__ - s = py.code.Source(newco) - assert str(s) == source - - -@failsonjython -def test_new_code_object_carries_filename_through(): - class mystr(str): - pass - filename = mystr("dummy") - co = compile("hello\n", filename, 'exec') - assert not isinstance(co.co_filename, mystr) - args = [ - co.co_argcount, co.co_nlocals, co.co_stacksize, - co.co_flags, co.co_code, co.co_consts, - co.co_names, co.co_varnames, - filename, - co.co_name, co.co_firstlineno, co.co_lnotab, - co.co_freevars, co.co_cellvars - ] - if sys.version_info > (3,0): - args.insert(1, co.co_kwonlyargcount) - c2 = py.std.types.CodeType(*args) - assert c2.co_filename is filename - def test_code_gives_back_name_for_not_existing_file(): name = 'abc-123' co_code = compile("pass\n", name, 'exec') diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index 9264026bb..b4538fa38 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -293,7 +293,6 @@ class TestFormattedExcinfo: assert lines[0] == "| def f(x):" assert lines[1] == " pass" - @failsonjython def test_repr_source_excinfo(self): """ check if indentation is right """ pr = FormattedExcinfo() diff --git a/testing/code/test_source.py b/testing/code/test_source.py index ef3d805ce..d94401161 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -80,11 +80,10 @@ def test_source_strip_multiline(): source2 = source.strip() assert source2.lines == [" hello"] -@failsonjython def test_syntaxerror_rerepresentation(): - ex = py.test.raises(SyntaxError, py.code.compile, 'x x') + ex = py.test.raises(SyntaxError, py.code.compile, 'xyz xyz') assert ex.value.lineno == 1 - assert ex.value.offset == 3 + assert ex.value.offset in (4,7) # XXX pypy/jython versus cpython? assert ex.value.text.strip(), 'x x' def test_isparseable(): @@ -132,7 +131,6 @@ class TestSourceParsingAndCompiling: exec (co, d) assert d['x'] == 3 - @failsonjython def test_compile_and_getsource_simple(self): co = py.code.compile("x=3") exec (co) @@ -203,7 +201,6 @@ class TestSourceParsingAndCompiling: assert isinstance(mod, ast.Module) compile(mod, "", "exec") - @failsonjython def test_compile_and_getsource(self): co = self.source.compile() py.builtin.exec_(co, globals()) @@ -260,7 +257,6 @@ def test_getstartingblock_multiline(): l = [i for i in x.source.lines if i.strip()] assert len(l) == 4 -@failsonjython def test_getline_finally(): def c(): pass excinfo = py.test.raises(TypeError, """ @@ -274,7 +270,6 @@ def test_getline_finally(): source = excinfo.traceback[-1].statement assert str(source).strip() == 'c(1)' -@failsonjython def test_getfuncsource_dynamic(): source = """ def f(): @@ -341,7 +336,6 @@ def test_getsource_fallback(): src = getsource(x) assert src == expected -@failsonjython def test_idem_compile_and_getsource(): from py._code.source import getsource expected = "def x(): pass" @@ -355,8 +349,7 @@ def test_findsource_fallback(): assert 'test_findsource_simple' in str(src) assert src[lineno] == ' def x():' -@failsonjython -def test_findsource___source__(): +def test_findsource(): from py._code.source import findsource co = py.code.compile("""if 1: def x(): @@ -373,7 +366,6 @@ def test_findsource___source__(): assert src[lineno] == " def x():" -@failsonjython def test_getfslineno(): from py.code import getfslineno diff --git a/testing/plugin/test_pytest_runner.py b/testing/plugin/test_pytest_runner.py index 72c67954d..941909d89 100644 --- a/testing/plugin/test_pytest_runner.py +++ b/testing/plugin/test_pytest_runner.py @@ -316,7 +316,11 @@ def test_runtest_in_module_ordering(testdir): class TestRaises: def test_raises(self): - py.test.raises(ValueError, "int('qwe')") + source = "int('qwe')" + excinfo = py.test.raises(ValueError, source) + code = excinfo.traceback[-1].frame.code + s = str(code.fullsource) + assert s == source def test_raises_exec(self): py.test.raises(ValueError, "a,x = []")