jiuyuan
/
test_ok1
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[svn r62492] removing apigen bits from py lib, they are now in svn/apigen 

--HG--
branch : trunk
This commit is contained in:
hpk 2009-03-03 19:06:16 +01:00
parent 30149574c7
commit 7688f88c4f
59 changed files with 56 additions and 7131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
py/apigen/__init__.py
View File

22
py/apigen/api.js
View File

@ -1,22 +0,0 @@
function showhideel(el) {
/* show or hide the element
sets the value of el.style.display to 'none' or 'block' depending
on the current value
*/
if (el.style.display == 'none') {
el.style.display = 'block';
} else {
el.style.display = 'none';
};
};
function getnextsibling(el) {
/* return next non-text sibling (or null) */
var node = el.nextSibling;
while (node && node.nodeType != 1) {
node = node.nextSibling;
};
return node;
};

11
py/apigen/apigen.js
View File

@ -1,11 +0,0 @@
function loadloc() {
/* load iframe content using # part of the url */
var loc = document.location.toString();
if (loc.indexOf('#') == -1) {
return;
};
var chunks = loc.split('#');
var anchor = chunks[chunks.length - 1];
var iframe = document.getElementsByTagName('iframe')[0];
iframe.src = anchor;
};

66
py/apigen/apigen.py
View File

@ -1,66 +0,0 @@
""" run 'py.test --apigen=<this script>' to get documentation exported
"""
import os
import py
import sys
from py.__.apigen import htmlgen
from py.__.apigen import linker
from py.__.apigen import project
from py.__.apigen.tracer.docstorage import pkg_to_dict
from layout import LayoutPage
def get_documentable_items_pkgdir(pkgdir):
""" get all documentable items from an initpkg pkgdir
this is a generic implementation, import as 'get_documentable_items'
from your module when using initpkg to get all public stuff in the
package documented
"""
sys.path.insert(0, str(pkgdir.dirpath()))
rootmod = __import__(pkgdir.basename)
d = pkg_to_dict(rootmod)
return pkgdir.basename, d
def get_documentable_items(pkgdir):
pkgname, pkgdict = get_documentable_items_pkgdir(pkgdir)
#from py.__.execnet.channel import Channel
#pkgdict['execnet.Channel'] = Channel
#Channel.__apigen_hide_from_nav__ = True
return pkgname, pkgdict
def sourcedirfilter(p):
return ('.svn' not in str(p).split(p.sep) and
not p.basename.startswith('.') and
str(p).find('c-extension%sgreenlet%sbuild' % (p.sep, p.sep)) == -1)
def build(config, pkgdir, dsa, capture):
# create a linker (link database) for cross-linking
l = linker.TempLinker()
# create a project.Project instance to contain the LayoutPage instances
proj = project.Project()
# output dir
targetdir = proj.apigenpath
targetdir.ensure(dir=True)
# find out what to build
all_names = dsa._get_names(filter=lambda x, y: True)
namespace_tree = htmlgen.create_namespace_tree(all_names)
# and build it
apb = htmlgen.ApiPageBuilder(targetdir, l, dsa, pkgdir, namespace_tree,
proj, capture, LayoutPage)
spb = htmlgen.SourcePageBuilder(targetdir, l, pkgdir, proj, capture,
LayoutPage, dirfilter=sourcedirfilter)
apb.build_namespace_pages()
class_names = dsa.get_class_names()
apb.build_class_pages(class_names)
function_names = dsa.get_function_names()
apb.build_function_pages(function_names)
spb.build_pages(pkgdir)
l.replace_dirpath(targetdir)

8
py/apigen/conftest.py
View File

@ -1,8 +0,0 @@
import py
class ConftestPlugin:
def pytest_addoption(self, parser):
parser.addoption('--webcheck',
action="store_true", dest="webcheck", default=False,
help="run XHTML validation tests"
)

216
py/apigen/html.py
View File

@ -1,216 +0,0 @@
import py
html = py.xml.html
# HTML related stuff
class H(html):
class Content(html.div):
def __init__(self, *args):
super(H.Content, self).__init__(id='apigen-content', *args)
class Description(html.div):
pass
class NamespaceDescription(Description):
pass
class NamespaceItem(html.div):
pass
class NamespaceDef(html.h1):
pass
class ClassDescription(Description):
pass
class ClassDef(html.div):
def __init__(self, classname, bases, docstring, sourcelink,
attrs, methods):
header = H.h1('class %s(' % (classname,))
for i, (name, href) in py.builtin.enumerate(bases):
if i > 0:
header.append(', ')
link = name
if href is not None:
link = H.a(name, href=href)
header.append(H.BaseDescription(link))
header.append('):')
super(H.ClassDef, self).__init__(header)
self.append(H.div(H.Docstring(docstring or
'*no docstring available*'),
sourcelink,
class_='classdoc'))
if attrs:
self.append(H.h2('class attributes and properties:'))
for name, val in attrs:
self.append(H.PropertyDescription(name, val))
if methods:
self.append(H.h2('methods:'))
for methodhtml in methods:
self.append(methodhtml)
class MethodDescription(Description):
pass
class MethodDef(html.h2):
pass
class FunctionDescription(Description):
def __init__(self, localname, argdesc, docstring, valuedesc, excdesc,
csource, callstack):
infoid = 'info_%s' % (localname.replace('.', '_dot_'),)
docstringid = 'docstring_%s' % (localname.replace('.', '_dot_'),)
fd = H.FunctionDef(localname, argdesc,
title='click to view details',
onclick=('showhideel('
'document.getElementById("%s")); '
% (infoid,)))
infodiv = H.div(
H.Docstring(docstring or '*no docstring available*',
id=docstringid),
H.FunctionInfo(valuedesc, excdesc, csource, callstack,
id=infoid, style="display: none"),
class_='funcdocinfo')
super(H.FunctionDescription, self).__init__(fd, infodiv)
class FunctionDef(html.h2):
style = html.Style(cursor='pointer')
def __init__(self, name, argdesc, **kwargs):
class_ = kwargs.pop('class_', 'funcdef')
super(H.FunctionDef, self).__init__('def %s%s:' % (name, argdesc),
class_=class_, **kwargs)
class FunctionInfo(html.div):
def __init__(self, valuedesc, excdesc, csource, callstack, **kwargs):
super(H.FunctionInfo, self).__init__(valuedesc, H.br(), excdesc,
H.br(), csource,
callstack, class_='funcinfo',
**kwargs)
class PropertyDescription(html.div):
def __init__(self, name, value):
if type(value) not in [str, unicode]:
value = str(value)
if len(value) > 100:
value = value[:100] + '...'
super(H.PropertyDescription, self).__init__(name, ': ',
H.em(value),
class_='property')
class ParameterDescription(html.div):
pass
class Docstring(html.div):
style = html.Style(white_space='pre', color='#666',
margin_left='1em', margin_bottom='1em')
class Navigation(html.div):
#style = html.Style(min_height='99%', float='left', margin_top='1.2em',
# overflow='auto', width='15em', white_space='nowrap')
pass
class NavigationItem(html.div):
def __init__(self, linker, linkid, name, indent, selected):
href = linker.get_lazyhref(linkid)
super(H.NavigationItem, self).__init__((indent * 2 * u'\xa0'),
H.a(name, href=href))
if selected:
self.attr.class_ = 'selected'
class BaseDescription(html.span):
pass
class SourceSnippet(html.div):
def __init__(self, text, href, sourceels=None):
if sourceels is None:
sourceels = []
link = text
if href:
link = H.a(text, href=href)
super(H.SourceSnippet, self).__init__(
link, H.div(*sourceels))
class PythonSource(Content):
style = html.Style(font_size='0.8em')
def __init__(self, *sourceels):
super(H.PythonSource, self).__init__(
H.div(*sourceels))
class SourceBlock(html.table):
def __init__(self):
tbody = H.tbody()
row = H.tr()
tbody.append(row)
linenocell = H.td(style='width: 1%')
row.append(linenocell)
linecell = H.td()
row.append(linecell)
self.linenotable = lntable = H.table()
self.linenotbody = lntbody = H.tbody()
lntable.append(lntbody)
linenocell.append(lntable)
self.linetable = ltable = H.table()
self.linetbody = ltbody = H.tbody()
ltable.append(ltbody)
linecell.append(ltable)
super(H.SourceBlock, self).__init__(tbody, class_='codeblock')
def add_line(self, lineno, els):
self.linenotbody.append(H.tr(H.td(lineno, class_='lineno')))
self.linetbody.append(H.tr(H.td(H.pre(class_='code', *els),
class_='codecell')))
class NonPythonSource(Content):
def __init__(self, *args):
super(H.NonPythonSource, self).__init__(H.pre(*args))
class DirList(Content):
def __init__(self, dirs, files):
dirs = [H.DirListItem(text, href) for (text, href) in dirs]
files = [H.DirListItem(text, href) for (text, href) in files]
super(H.DirList, self).__init__(
H.h2('directories'), dirs,
H.h2('files'), files,
)
class DirListItem(html.div):
def __init__(self, text, href):
super(H.DirListItem, self).__init__(H.a(text, href=href))
class ValueDescList(html.ul):
def __init__(self, *args, **kwargs):
super(H.ValueDescList, self).__init__(*args, **kwargs)
class ExceptionDescList(html.ul):
def __init__(self, *args, **kwargs):
super(H.ExceptionDescList, self).__init__(*args, **kwargs)
def append(self, t):
super(H.ExceptionDescList, self).append(html.li(t))
class ValueDescItem(html.li):
pass
class CallStackDescription(Description):
pass
class CallStackLink(html.div):
def __init__(self, filename, lineno, href):
super(H.CallStackLink, self).__init__(
H.a("stack trace %s - line %s" % (filename, lineno),
href=href))
class Hideable(html.div):
def __init__(self, name, class_, *content):
super(H.Hideable, self).__init__(
H.div(H.a('show/hide %s' % (name,),
href='#',
onclick=('showhideel(getnextsibling(this));'
'return false;')),
H.div(style='display: none',
class_=class_,
*content)))

838
py/apigen/htmlgen.py
View File

@ -1,838 +0,0 @@
import py
import os
import inspect
from py.__.apigen.layout import LayoutPage
from py.__.apigen.source import browser as source_browser
from py.__.apigen.source import html as source_html
from py.__.apigen.source import color as source_color
from py.__.apigen.tracer.description import is_private
from py.__.apigen.rest.genrest import split_of_last_part
from py.__.apigen.linker import relpath
from py.__.apigen.html import H
reversed = py.builtin.reversed
sorted = py.builtin.sorted
html = py.xml.html
raw = py.xml.raw
REDUCE_CALLSITES = True
def find_method_origin(meth):
cls = getattr(meth, 'im_class', None)
if cls is None:
return None # XXX unknown origin (built-in function or method or sth)
name = meth.im_func.func_name
origin = cls
# XXX old-style classes support required? :|
mro = inspect.getmro(cls)
for base in mro:
m = getattr(base, name, None)
if m is None:
continue
if not hasattr(m, 'im_func'):
# builtin
return None
if m.im_func is meth.im_func:
origin = base
return origin
def is_navigateable(name):
return (not is_private(name) and name != '__doc__')
def show_property(name):
if not name.startswith('_'):
return True
if name.startswith('__') and name.endswith('__'):
# XXX do we need to skip more manually here?
if (name not in dir(object) and
name not in ['__doc__', '__dict__', '__name__', '__module__',
'__weakref__', '__apigen_hide_from_nav__']):
return True
return False
def deindent(str, linesep='\n'):
""" de-indent string
can be used to de-indent Python docstrings, it de-indents the first
line to the side always, and determines the indentation of the rest
of the text by taking that of the least indented (filled) line
"""
lines = str.strip().split(linesep)
normalized = []
deindent = None
normalized.append(lines[0].strip())
# replace tabs with spaces, empty lines that contain spaces only, and
# find out what the smallest indentation is
for line in lines[1:]:
line = line.replace('\t', ' ' * 4)
stripped = line.strip()
if not stripped:
normalized.append('')
else:
rstripped = line.rstrip()
indent = len(rstripped) - len(stripped)
if deindent is None or indent < deindent:
deindent = indent
normalized.append(line)
ret = [normalized[0]]
for line in normalized[1:]:
if not line:
ret.append(line)
else:
ret.append(line[deindent:])
return '%s\n' % (linesep.join(ret),)
def get_linesep(s, default='\n'):
""" return the line seperator of a string
returns 'default' if no seperator can be found
"""
for sep in ('\r\n', '\r', '\n'):
if sep in s:
return sep
return default
def get_param_htmldesc(linker, func):
""" get the html for the parameters of a function """
import inspect
# XXX copy and modify formatargspec to produce html
return inspect.formatargspec(*inspect.getargspec(func))
# some helper functionality
def source_dirs_files(fspath, fil=None):
""" returns a tuple (dirs, files) for fspath
dirs are all the subdirs, files are the files which are interesting
in building source documentation for a Python code tree (basically all
normal files excluding .pyc and .pyo ones)
all files and dirs that have a name starting with . are considered
hidden
"""
dirs = []
files = []
for child in fspath.listdir(fil=fil):
if child.basename.startswith('.'):
continue
if child.check(dir=True):
dirs.append(child)
elif child.check(file=True):
if child.ext in ['.pyc', '.pyo']:
continue
files.append(child)
return sorted(dirs), sorted(files)
def create_namespace_tree(dotted_names):
""" creates a tree (in dict form) from a set of dotted names
"""
ret = {}
for dn in dotted_names:
path = dn.split('.')
for i in xrange(len(path)):
ns = '.'.join(path[:i])
itempath = '.'.join(path[:i + 1])
if ns not in ret:
ret[ns] = []
if itempath not in ret[ns]:
ret[ns].append(itempath)
return ret
def wrap_page(project, title, targetpath, contentel, navel, basepath,
pageclass):
page = pageclass(project, title, targetpath, nav=navel, encoding='UTF-8')
page.set_content(contentel)
page.setup_scripts_styles(basepath)
return page
def enumerate_and_color(codelines, firstlineno, enc):
snippet = H.SourceBlock()
tokenizer = source_color.Tokenizer(source_color.PythonSchema)
for i, line in enumerate(codelines):
try:
snippet.add_line(i + firstlineno + 1,
source_html.prepare_line([line], tokenizer, enc))
except py.error.ENOENT:
# error reading source code, giving up
snippet = codelines
break
return snippet
def enumerate_and_color_module(path, enc):
snippet = H.SourceBlock()
tokenizer = source_color.Tokenizer(source_color.PythonSchema)
for i, text in enumerate(source_html.prepare_module(path, tokenizer, enc)):
snippet.add_line(i + 1, text)
return snippet
_get_obj_cache = {}
def get_obj(dsa, pkg, dotted_name):
full_dotted_name = '%s.%s' % (pkg.__name__, dotted_name)
if dotted_name == '':
return pkg
try:
return _get_obj_cache[dotted_name]
except KeyError:
pass
path = dotted_name.split('.')
ret = pkg
for item in path:
marker = []
ret = getattr(ret, item, marker)
if ret is marker:
try:
ret = dsa.get_obj(dotted_name)
except KeyError:
raise NameError('can not access %s in %s' % (item,
full_dotted_name))
else:
break
_get_obj_cache[dotted_name] = ret
return ret
def get_rel_sourcepath(projpath, filename, default=None):
relpath = py.path.local(filename).relto(projpath)
if not relpath:
return default
return relpath
def get_package_revision(packageroot, _revcache={}):
try:
rev = _revcache[packageroot]
except KeyError:
wc = py.path.svnwc(packageroot)
rev = None
if wc.check(versioned=True):
rev = py.path.svnwc(packageroot).info().rev
_revcache[packageroot] = rev
if packageroot.basename == "py":
assert rev is not None
return rev
# the PageBuilder classes take care of producing the docs (using the stuff
# above)
class AbstractPageBuilder(object):
pageclass = LayoutPage
def write_page(self, title, reltargetpath, tag, nav):
targetpath = self.base.join(reltargetpath)
relbase= relpath('%s%s' % (targetpath.dirpath(), targetpath.sep),
self.base.strpath + '/')
page = wrap_page(self.project, title, targetpath, tag, nav, self.base,
self.pageclass)
# we write the page with _temporary_ hrefs here, need to be replaced
# from the TempLinker later
content = page.unicode()
targetpath.ensure()
targetpath.write(content.encode("utf8"))
class SourcePageBuilder(AbstractPageBuilder):
""" builds the html for a source docs page """
def __init__(self, base, linker, projroot, project, capture=None,
pageclass=LayoutPage, dirfilter=None):
self.base = base
self.linker = linker
self.projroot = projroot
self.project = project
self.capture = capture
self.pageclass = pageclass
self.dirfilter = dirfilter
def build_navigation(self, fspath):
nav = H.Navigation(class_='sidebar')
relpath = fspath.relto(self.projroot)
path = relpath.split(os.path.sep)
indent = 0
# build links to parents
if relpath != '':
for i in xrange(len(path)):
dirpath = os.path.sep.join(path[:i])
abspath = self.projroot.join(dirpath).strpath
if i == 0:
text = self.projroot.basename
else:
text = path[i-1]
nav.append(H.NavigationItem(self.linker, abspath, text,
indent, False))
indent += 1
# build siblings or children and self
if fspath.check(dir=True):
# we're a dir, build ourselves and our children
dirpath = fspath
nav.append(H.NavigationItem(self.linker, dirpath.strpath,
dirpath.basename, indent, True))
indent += 1
elif fspath.strpath == self.projroot.strpath:
dirpath = fspath
else:
# we're a file, build our parent's children only
dirpath = fspath.dirpath()
diritems, fileitems = source_dirs_files(dirpath, self.dirfilter)
for dir in diritems:
nav.append(H.NavigationItem(self.linker, dir.strpath, dir.basename,
indent, False))
for file in fileitems:
selected = (fspath.check(file=True) and
file.basename == fspath.basename)
nav.append(H.NavigationItem(self.linker, file.strpath,
file.basename, indent, selected))
return nav
re = py.std.re
_reg_body = re.compile(r'<body[^>]*>(.*)</body>', re.S)
def build_python_page(self, fspath):
# XXX two reads of the same file here... not very bad (disk caches
# and such) but also not very nice...
enc = source_html.get_module_encoding(fspath.strpath)
try:
colored = [enumerate_and_color_module(fspath, enc)]
except (KeyboardInterrupt, SystemExit):
raise
except Exception, e:
#self.capture.err.writeorg('\ncompilation exception: %s\n' % (e,))
# problem building HTML with anchors; let's try without...
source = fspath.read()
sep = get_linesep(source)
colored = [enumerate_and_color(source.split(sep), 0, enc)]
tag = H.PythonSource(colored)
nav = self.build_navigation(fspath)
return tag, nav
def build_dir_page(self, fspath):
dirs, files = source_dirs_files(fspath, self.dirfilter)
dirs = [(p.basename, self.linker.get_lazyhref(str(p))) for p in dirs]
files = [(p.basename, self.linker.get_lazyhref(str(p))) for p in files]
tag = H.DirList(dirs, files)
nav = self.build_navigation(fspath)
return tag, nav
def build_nonpython_page(self, fspath):
try:
tag = H.NonPythonSource(unicode(fspath.read(), 'utf-8'))
except UnicodeError:
tag = H.NonPythonSource('no source available (binary file?)')
nav = self.build_navigation(fspath)
return tag, nav
def build_pages(self, base):
def visit(p):
dirs, files = source_dirs_files(p, self.dirfilter)
for d in dirs:
yield d
for sp in visit(d):
yield sp
for f in files:
yield f
for fspath in [base] + list(visit(base)):
if fspath.ext in ['.pyc', '.pyo']:
continue
if self.capture:
self.capture.err.writeorg('.')
relfspath = fspath.relto(base)
if relfspath.find('%s.' % (os.path.sep,)) > -1:
# skip hidden dirs and files
continue
elif fspath.check(dir=True):
if relfspath != '':
relfspath += os.path.sep
reloutputpath = 'source%s%sindex.html' % (os.path.sep,
relfspath)
else:
reloutputpath = "source%s%s.html" % (os.path.sep, relfspath)
reloutputpath = reloutputpath.replace(os.path.sep, '/')
outputpath = self.base.join(reloutputpath)
self.linker.set_link(str(fspath), reloutputpath)
self.build_page(fspath, outputpath, base)
def build_page(self, fspath, outputpath, base):
""" build syntax-colored source views """
if fspath.check(ext='.py'):
try:
tag, nav = self.build_python_page(fspath)
except (KeyboardInterrupt, SystemExit):
raise
except: # XXX strange stuff going wrong at times... need to fix
raise
exc, e, tb = py.std.sys.exc_info()
print '%s - %s' % (exc, e)
print
print ''.join(py.std.traceback.format_tb(tb))
print '-' * 79
del tb
tag, nav = self.build_nonpython_page(fspath)
elif fspath.check(dir=True):
tag, nav = self.build_dir_page(fspath)
else:
tag, nav = self.build_nonpython_page(fspath)
title = 'sources for %s' % (fspath.basename,)
rev = self.get_revision(fspath)
if rev:
title += ' [rev. %s]' % (rev,)
reltargetpath = outputpath.relto(self.base).replace(os.path.sep,
'/')
self.write_page(title, reltargetpath, tag, nav)
_revcache = {}
def get_revision(self, path):
return get_package_revision(self.projroot)
strpath = path.strpath
if strpath in self._revcache:
return self._revcache[strpath]
wc = py.path.svnwc(path)
if wc.check(versioned=True):
rev = wc.info().created_rev
else:
rev = None
self._revcache[strpath] = rev
return rev
class ApiPageBuilder(AbstractPageBuilder):
""" builds the html for an api docs page """
def __init__(self, base, linker, dsa, projroot, namespace_tree, project,
capture=None, pageclass=LayoutPage):
self.base = base
self.linker = linker
self.dsa = dsa
self.projroot = projroot
self.projpath = py.path.local(projroot)
self.namespace_tree = namespace_tree
self.project = project
self.capture = capture
self.pageclass = pageclass
pkgname = self.dsa.get_module_name().split('/')[-1]
self.pkg = __import__(pkgname)
def build_callable_view(self, dotted_name):
""" build the html for a class method """
# XXX we may want to have seperate
func = get_obj(self.dsa, self.pkg, dotted_name)
docstring = func.__doc__
if docstring:
docstring = deindent(docstring)
localname = func.__name__
argdesc = get_param_htmldesc(self.linker, func)
excdesc = self.build_exception_description(dotted_name)
valuedesc = self.build_callable_signature_description(dotted_name)
sourcefile = inspect.getsourcefile(func)
callable_source = self.dsa.get_function_source(dotted_name)
# i assume they're both either available or unavailable(XXX ?)
is_in_pkg = self.is_in_pkg(sourcefile)
href = None
text = 'could not get to source file'
colored = []
if sourcefile and callable_source:
enc = source_html.get_module_encoding(sourcefile)
sep = get_linesep(callable_source)
colored = [enumerate_and_color(callable_source.split(sep),
func.func_code.co_firstlineno, enc)]
relpath = get_rel_sourcepath(self.projroot, sourcefile, sourcefile)
text = 'source: %s' % (relpath,)
if is_in_pkg:
#href = self.linker.get_lazyhref(sourcefile,
# self.get_anchor(func))
href = self.linker.get_lazyhref(sourcefile) #
csource = H.SourceSnippet(text, href, colored)
cslinks = self.build_callsites(dotted_name)
snippet = H.FunctionDescription(localname, argdesc, docstring,
valuedesc, excdesc, csource, cslinks)
return snippet
def build_class_view(self, dotted_name):
""" build the html for a class """
cls = get_obj(self.dsa, self.pkg, dotted_name)
# XXX is this a safe check?
try:
sourcefile = inspect.getsourcefile(cls)
except TypeError:
sourcefile = None
docstring = cls.__doc__
if docstring:
docstring = deindent(docstring)
if not hasattr(cls, '__name__'):
clsname = 'instance of %s' % (cls.__class__.__name__,)
else:
clsname = cls.__name__
bases = self.build_bases(dotted_name)
properties = self.build_properties(cls)
methods = self.build_methods(dotted_name)
if sourcefile is None:
sourcelink = H.div('no source available')
else:
if sourcefile[-1] in ['o', 'c']:
sourcefile = sourcefile[:-1]
sourcelink = H.div(H.a('view source',
href=self.linker.get_lazyhref(sourcefile) #, self.get_anchor(cls)
))
snippet = H.ClassDescription(
# XXX bases HTML
H.ClassDef(clsname, bases, docstring, sourcelink,
properties, methods),
)
return snippet
def build_bases(self, dotted_name):
ret = []
bases = self.dsa.get_possible_base_classes(dotted_name)
for base in bases:
try:
obj = self.dsa.get_obj(base.name)
except KeyError:
ret.append((base.name, None))
else:
href = self.linker.get_lazyhref(base.name)
ret.append((base.name, href))
return ret
def build_properties(self, cls):
properties = []
for attr in dir(cls):
val = getattr(cls, attr)
if show_property(attr) and not callable(val):
if isinstance(val, property):
val = '<property object (dynamically calculated value)>'
properties.append((attr, val))
properties.sort(lambda x,y : cmp(x[0], y[0])) # sort on name
return properties
def build_methods(self, dotted_name):
ret = []
methods = self.dsa.get_class_methods(dotted_name)
# move all __*__ methods to the back
methods = ([m for m in methods if not m.startswith('_')] +
[m for m in methods if m.startswith('_')])
# except for __init__, which should be first
if '__init__' in methods:
methods.remove('__init__')
methods.insert(0, '__init__')
for method in methods:
ret += self.build_callable_view('%s.%s' % (dotted_name,
method))
return ret
def build_namespace_view(self, namespace_dotted_name, item_dotted_names):
""" build the html for a namespace (module) """
obj = get_obj(self.dsa, self.pkg, namespace_dotted_name)
docstring = obj.__doc__
snippet = H.NamespaceDescription(
H.NamespaceDef(namespace_dotted_name),
H.Docstring(docstring or '*no docstring available*')
)
for dotted_name in sorted(item_dotted_names):
itemname = dotted_name.split('.')[-1]
if (not is_navigateable(itemname) or
self.is_hidden_from_nav(dotted_name)):
continue
snippet.append(
H.NamespaceItem(
H.a(itemname,
href=self.linker.get_lazyhref(dotted_name)
)
)
)
return snippet
def build_class_pages(self, classes_dotted_names):
passed = []
for dotted_name in sorted(classes_dotted_names):
if self.capture:
self.capture.err.writeorg('.')
parent_dotted_name, _ = split_of_last_part(dotted_name)
try:
sibling_dotted_names = self.namespace_tree[parent_dotted_name]
except KeyError:
# no siblings (built-in module or sth)
sibling_dotted_names = []
tag = H.Content(self.build_class_view(dotted_name))
nav = self.build_navigation(dotted_name, False)
reltargetpath = "api/%s.html" % (dotted_name,)
self.linker.set_link(dotted_name, reltargetpath)
title = '%s API' % (dotted_name,)
rev = self.get_revision(dotted_name)
if rev:
title += ' [rev. %s]' % (rev,)
self.write_page(title, reltargetpath, tag, nav)
return passed
def build_function_pages(self, method_dotted_names):
passed = []
for dotted_name in sorted(method_dotted_names):
if self.capture:
self.capture.err.writeorg('.')
# XXX should we create a build_function_view instead?
parent_dotted_name, _ = split_of_last_part(dotted_name)
sibling_dotted_names = self.namespace_tree[parent_dotted_name]
tag = H.Content(self.build_callable_view(dotted_name))
nav = self.build_navigation(dotted_name, False)
reltargetpath = "api/%s.html" % (dotted_name,)
self.linker.set_link(dotted_name, reltargetpath)
title = '%s API' % (dotted_name,)
rev = self.get_revision(dotted_name)
if rev:
title += ' [rev. %s]' % (rev,)
self.write_page(title, reltargetpath, tag, nav)
return passed
def build_namespace_pages(self):
passed = []
module_name = self.dsa.get_module_name().split('/')[-1]
names = self.namespace_tree.keys()
names.sort()
function_names = self.dsa.get_function_names()
class_names = self.dsa.get_class_names()
for dotted_name in sorted(names):
if self.capture:
self.capture.err.writeorg('.')
if dotted_name in function_names or dotted_name in class_names:
continue
subitem_dotted_names = self.namespace_tree[dotted_name]
tag = H.Content(self.build_namespace_view(dotted_name,
subitem_dotted_names))
nav = self.build_navigation(dotted_name, True)
if dotted_name == '':
reltargetpath = 'api/index.html'
else:
reltargetpath = 'api/%s.html' % (dotted_name,)
self.linker.set_link(dotted_name, reltargetpath)
title_name = dotted_name
if dotted_name == '':
title_name = self.dsa.get_module_name()
title = 'index of %s' % (title_name,)
rev = self.get_revision(dotted_name)
if rev:
title += ' [rev. %s]' % (rev,)
self.write_page(title, reltargetpath, tag, nav)
return passed
def build_navigation(self, dotted_name, build_children=True):
navitems = []
# top namespace, index.html
module_name = self.dsa.get_module_name().split('/')[-1]
navitems.append(H.NavigationItem(self.linker, '', module_name, 0,
True))
def build_nav_level(dotted_name, depth=1):
navitems = []
path = dotted_name.split('.')[:depth]
siblings = self.namespace_tree.get('.'.join(path[:-1]))
for dn in sorted(siblings):
selected = dn == '.'.join(path)
sibpath = dn.split('.')
sibname = sibpath[-1]
if not is_navigateable(sibname):
continue
if self.is_hidden_from_nav(dn):
continue
navitems.append(H.NavigationItem(self.linker, dn, sibname,
depth, selected))
if selected:
lastlevel = dn.count('.') == dotted_name.count('.')
if not lastlevel:
navitems += build_nav_level(dotted_name, depth+1)
elif lastlevel and build_children:
# XXX hack
navitems += build_nav_level('%s.' % (dotted_name,),
depth+1)
return navitems
navitems += build_nav_level(dotted_name)
return H.Navigation(class_='sidebar', *navitems)
def build_callable_signature_description(self, dotted_name):
args, retval = self.dsa.get_function_signature(dotted_name)
valuedesc = H.ValueDescList()
for name, _type in args:
valuedesc.append(self.build_sig_value_description(name, _type))
if retval:
retval = self.process_type_link(retval)
ret = H.div(H.div('arguments:'), valuedesc, H.div('return value:'),
retval or 'None')
return ret
def build_sig_value_description(self, name, _type):
l = self.process_type_link(_type)
items = []
next = "%s: " % name
for item in l:
if isinstance(item, str):
next += item
else:
if next:
items.append(next)
next = ""
items.append(item)
if next:
items.append(next)
return H.ValueDescItem(*items)
def process_type_link(self, _type):
# now we do simple type dispatching and provide a link in this case
lst = []
data = self.dsa.get_type_desc(_type)
if not data:
for i in _type.striter():
if isinstance(i, str):
lst.append(i)
else:
lst += self.process_type_link(i)
return lst
name, _desc_type, is_degenerated = data
if not is_degenerated:
try:
obj = self.dsa.get_obj(name)
except KeyError:
obj = None
linktarget = self.linker.get_lazyhref(name)
lst.append(H.a(str(_type), href=linktarget))
else:
raise IOError('do not think we ever get here?')
# we should provide here some way of linking to sourcegen directly
lst.append(name)
return lst
def build_exception_description(self, dotted_name):
excs = self.dsa.get_function_exceptions(dotted_name)
excdesc = H.ExceptionDescList()
for exc in excs:
excdesc.append(exc)
ret = H.div(H.div('possible exceptions:'), excdesc)
return ret
def is_in_pkg(self, sourcefile):
return py.path.local(sourcefile).relto(self.projpath)
_processed_callsites = {}
def build_callsites(self, dotted_name):
callstack = self.dsa.get_function_callpoints(dotted_name)
cslinks = []
for i, (cs, _) in enumerate(callstack):
if REDUCE_CALLSITES:
key = (cs[0].filename, cs[0].lineno)
if key in self._processed_callsites:
# process one call site per line of test code when
# REDUCE_CALLSITES is set to True
continue
self._processed_callsites[key] = 1
link = self.build_callsite(dotted_name, cs, i)
cslinks.append(link)
return cslinks
def build_callsite(self, dotted_name, call_site, index):
tbtag = H.Content(self.gen_traceback(dotted_name, reversed(call_site)))
parent_dotted_name, _ = split_of_last_part(dotted_name)
nav = self.build_navigation(parent_dotted_name, False)
id = 'callsite_%s_%s' % (dotted_name, index)
reltargetpath = "api/%s.html" % (id,)
self.linker.set_link(id, reltargetpath)
href = self.linker.get_lazyhref(id)
self.write_page('call site %s for %s' % (index, dotted_name),
reltargetpath, tbtag, nav)
sourcefile = call_site[0].filename
sourcepath = get_rel_sourcepath(self.projpath, sourcefile, sourcefile)
return H.CallStackLink(sourcepath, call_site[0].lineno + 1, href)
_reg_source = py.std.re.compile(r'([^>]*)<(.*)>')
def gen_traceback(self, dotted_name, call_site):
tbtag = H.CallStackDescription()
obj = self.dsa.get_obj(dotted_name)
for frame in call_site:
lineno = frame.lineno - frame.firstlineno
source = frame.source
sourcefile = frame.filename
tokenizer = source_color.Tokenizer(source_color.PythonSchema)
mangled = []
source = str(source)
sep = get_linesep(source)
for i, sline in enumerate(source.split(sep)):
if i == lineno:
l = '-> %s' % (sline,)
else:
l = ' %s' % (sline,)
mangled.append(l)
if sourcefile:
relpath = get_rel_sourcepath(self.projpath, sourcefile,
sourcefile)
linktext = '%s - line %s' % (relpath, frame.lineno + 1)
# skip py.code.Source objects and source files outside of the
# package
is_code_source = self._reg_source.match(sourcefile)
if (not is_code_source and self.is_in_pkg(sourcefile) and
py.path.local(sourcefile).check()):
enc = source_html.get_module_encoding(sourcefile)
href = self.linker.get_lazyhref(sourcefile)
sourcelink = H.a(linktext, href=href)
else:
enc = 'latin-1'
sourcelink = H.div(linktext)
colored = [enumerate_and_color(mangled,
frame.firstlineno, enc)]
else:
sourcelink = H.div('source unknown (%s)' % (sourcefile,))
colored = mangled[:]
tbtag.append(sourcelink)
tbtag.append(H.div(*colored))
return tbtag
def is_hidden_from_nav(self, dotted_name):
obj = get_obj(self.dsa, self.pkg, dotted_name)
return getattr(obj, '__apigen_hide_from_nav__', False)
_revcache = {}
def get_proj_revision(self):
if '' in self._revcache:
return self._revcache['']
wc = py.path.svnwc(self.projpath)
if wc.check(versioned=True):
rev = wc.info().created_rev
else:
rev = None
self._revcache[''] = rev
return rev
def get_revision(self, dotted_name):
return get_package_revision(self.projroot)
if dotted_name in self._revcache:
return self._revcache[dotted_name]
obj = get_obj(self.dsa, self.pkg, dotted_name)
rev = None
try:
sourcefile = inspect.getsourcefile(obj)
except TypeError:
pass
else:
if sourcefile is not None:
if sourcefile[-1] in ['o', 'c']:
sourcefile = sourcefile[:-1]
wc = py.path.svnwc(sourcefile)
if wc.check(versioned=True):
rev = wc.info().created_rev
rev = rev or self.get_proj_revision()
self._revcache[dotted_name] = rev
return rev
def get_anchor(self, obj):
# XXX may not always return the right results...
anchor = None
if hasattr(obj, 'im_func'):
# method
origin = find_method_origin(obj)
if origin:
anchor = '%s.%s' % (origin.__name__,
obj.im_func.func_name)
elif hasattr(obj, 'func_name'):
anchor = obj.func_name
elif hasattr(obj, '__name__'):
anchor = obj.__name__
elif hasattr(obj, '__class__'):
anchor = obj.__class__.__name__
return anchor

48
py/apigen/layout.py
View File

@ -1,48 +0,0 @@
""" layout definition for generating api/source documents
this is the place where customization can be done
"""
import py
from py.__.doc import confrest
from py.__.apigen import linker
here = py.magic.autopath().dirpath()
class LayoutPage(confrest.PyPage):
""" this provides the layout and style information """
stylesheets = [(here.join('../doc/style.css'), 'style.css'),
(here.join('style.css'), 'apigen_style.css')]
scripts = [(here.join('api.js'), 'api.js')]
def __init__(self, *args, **kwargs):
self.nav = kwargs.pop('nav')
super(LayoutPage, self).__init__(*args, **kwargs)
self.relpath = self.get_relpath()
self.project.logo.attr.id = 'logo'
def get_relpath(self):
return linker.relpath(self.targetpath.strpath,
self.project.apigenpath.strpath) + '/'
def set_content(self, contentel):
self.contentspace.append(contentel)
def fill(self):
super(LayoutPage, self).fill()
self.body.insert(0, self.nav)
def setup_scripts_styles(self, copyto=None):
for path, name in self.stylesheets:
if copyto:
copyto.join(name).write(path.read())
self.head.append(py.xml.html.link(type='text/css',
rel='stylesheet',
href=self.relpath + name))
for path, name in self.scripts:
if copyto:
copyto.join(name).write(path.read())
self.head.append(py.xml.html.script(type="text/javascript",
src=self.relpath + name))

154
py/apigen/linker.py
View File

@ -1,154 +0,0 @@
import py
import os
html = py.xml.html
# this here to serve two functions: first it makes the proto part of the temp
# urls (see TempLinker) customizable easily (for tests and such) and second
# it makes sure the temp links aren't replaced in generated source code etc.
# for this file (and its tests) itself.
TEMPLINK_PROTO = 'apigen.temp'
def getrelfspath(dotted_name):
# XXX need to make sure its imported on non-py lib
return eval(dotted_name, {"py": py})
class LazyHref(object):
def __init__(self, linker, linkid):
self._linker = linker
self._linkid = linkid
def __unicode__(self):
return unicode(self._linker.get_target(self._linkid))
class Linker(object):
fromlocation = None
def __init__(self):
self._linkid2target = {}
def get_lazyhref(self, linkid):
return LazyHref(self, linkid)
def set_link(self, linkid, target):
assert linkid not in self._linkid2target, (
'linkid %r already used' % (linkid,))
self._linkid2target[linkid] = target
def get_target(self, linkid):
linktarget = self._linkid2target[linkid]
if self.fromlocation is not None:
linktarget = relpath(self.fromlocation, linktarget)
return linktarget
def call_withbase(self, base, func, *args, **kwargs):
assert self.fromlocation is None
self.fromlocation = base
try:
return func(*args, **kwargs)
finally:
del self.fromlocation
class TempLinker(object):
""" performs a similar role to the Linker, but with a different approach
instead of returning 'lazy' hrefs, this returns a simple URL-style
string
the 'temporary urls' are replaced on the filesystem after building the
files, so that means even though a second pass is still required,
things don't have to be built in-memory (as with the Linker)
"""
fromlocation = None
def __init__(self):
self._linkid2target = {}
def get_lazyhref(self, linkid):
return '%s://%s' % (TEMPLINK_PROTO, linkid)
def set_link(self, linkid, target):
assert linkid not in self._linkid2target
self._linkid2target[linkid] = target
def get_target(self, tempurl, fromlocation=None):
assert tempurl.startswith('%s://' % (TEMPLINK_PROTO,))
linkid = '://'.join(tempurl.split('://')[1:])
linktarget = self._linkid2target[linkid]
if fromlocation is not None:
linktarget = relpath(fromlocation, linktarget)
return linktarget
_reg_tempurl = py.std.re.compile('["\'](%s:\/\/[^"\s]*)["\']' % (
TEMPLINK_PROTO,))
def replace_dirpath(self, dirpath, stoponerrors=True):
""" replace temporary links in all html files in dirpath and below """
for fpath in dirpath.visit('*.html'):
html = fpath.read()
while 1:
match = self._reg_tempurl.search(html)
if not match:
break
tempurl = match.group(1)
try:
html = html.replace('"' + tempurl + '"',
'"' + self.get_target(tempurl,
fpath.relto(dirpath)) + '"')
except KeyError:
if stoponerrors:
raise
html = html.replace('"' + tempurl + '"',
'"apigen.notfound://%s"' % (tempurl,))
fpath.write(html)
def relpath(p1, p2, sep=os.path.sep, back='..', normalize=True):
""" create a relative path from p1 to p2
sep is the seperator used for input and (depending
on the setting of 'normalize', see below) output
back is the string used to indicate the parent directory
when 'normalize' is True, any backslashes (\) in the path
will be replaced with forward slashes, resulting in a consistent
output on Windows and the rest of the world
paths to directories must end on a / (URL style)
"""
if normalize:
p1 = p1.replace(sep, '/')
p2 = p2.replace(sep, '/')
sep = '/'
# XXX would be cool to be able to do long filename expansion and drive
# letter fixes here, and such... iow: windows sucks :(
if (p1.startswith(sep) ^ p2.startswith(sep)):
raise ValueError("mixed absolute relative path: %r -> %r" %(p1, p2))
fromlist = p1.split(sep)
tolist = p2.split(sep)
# AA
# AA BB -> AA/BB
#
# AA BB
# AA CC -> CC
#
# AA BB
# AA -> ../AA
diffindex = 0
for x1, x2 in zip(fromlist, tolist):
if x1 != x2:
break
diffindex += 1
commonindex = diffindex - 1
fromlist_diff = fromlist[diffindex:]
tolist_diff = tolist[diffindex:]
if not fromlist_diff:
return sep.join(tolist[commonindex:])
backcount = len(fromlist_diff)
if tolist_diff:
return sep.join([back,]*(backcount-1) + tolist_diff)
return sep.join([back,]*(backcount) + tolist[commonindex:])

69
py/apigen/project.py
View File

@ -1,69 +0,0 @@
""" this contains the code that actually builds the pages using layout.py
building the docs happens in two passes: the first one takes care of
collecting contents and navigation items, the second builds the actual
HTML
"""
import py
from layout import LayoutPage
# XXX don't import from an internal py lib class
from py.__.doc import confrest
class Project(confrest.Project):
""" a full project
this takes care of storing information on the first pass, and building
pages + indexes on the second
"""
def __init__(self, *args, **kwargs):
confrest.Project.__init__(self, *args, **kwargs)
self.content_items = {}
def add_item(self, path, content):
""" add a single item (page)
path is a (relative) path to the object, used for building links
and navigation
content is an instance of some py.xml.html item
"""
assert path not in self.content_items, 'duplicate path %s' % (path,)
self.content_items[path] = content
def build(self, outputpath):
""" convert the tree to actual HTML
uses the LayoutPage class below for each page and takes care of
building index documents for the root and each sub directory
"""
opath = py.path.local(outputpath)
opath.ensure(dir=True)
paths = self.content_items.keys()
paths.sort()
for path in paths:
# build the page using the LayoutPage class
page = self.Page(self, path, stylesheeturl=self.stylesheet)
page.contentspace.append(self.content_items[path])
ipath = opath.join(path)
if not ipath.dirpath().check():
# XXX create index.html(?)
ipath.ensure(file=True)
ipath.write(page.unicode().encode(self.encoding))
def process(self, txtpath):
""" this allows using the project from confrest """
# XXX not interesting yet, but who knows later (because of the
# cool nav)
if __name__ == '__main__':
# XXX just to have an idea of how to use this...
proj = Project()
here = py.path.local('.')
for fpath in here.visit():
if fpath.check(file=True):
proj.add_item(fpath, convert_to_html_somehow(fpath))
proj.build()

0
py/apigen/rest/__init__.py
View File

524
py/apigen/rest/genrest.py
View File

@ -1,524 +0,0 @@
""" Generating ReST output (raw, not python)
out of data that we know about function calls
"""
import py
import sys
import re
from py.__.apigen.tracer.docstorage import DocStorageAccessor
from py.__.rest.rst import * # XXX Maybe we should list it here
from py.__.apigen.tracer import model
from py.__.rest.transform import RestTransformer
def split_of_last_part(name):
name = name.split(".")
return ".".join(name[:-1]), name[-1]
class AbstractLinkWriter(object):
""" Class implementing writing links to source code.
There should exist various classes for that, different for Trac,
different for CVSView, etc.
"""
def getlinkobj(self, obj, name):
return None
def getlink(self, filename, lineno, funcname):
raise NotImplementedError("Abstract link writer")
def getpkgpath(self, filename):
# XXX: very simple thing
path = py.path.local(filename).dirpath()
while 1:
try:
path.join('__init__.py').stat()
path = path.dirpath()
except py.error.ENOENT:
return path
class ViewVC(AbstractLinkWriter):
""" Link writer for ViewVC version control viewer
"""
def __init__(self, basepath):
# XXX: should try to guess from a working copy of svn
self.basepath = basepath
def getlink(self, filename, lineno, funcname):
path = str(self.getpkgpath(filename))
assert filename.startswith(path), (
"%s does not belong to package %s" % (filename, path))
relname = filename[len(path):]
if relname.endswith('.pyc'):
relname = relname[:-1]
sep = py.std.os.sep
if sep != '/':
relname = relname.replace(sep, '/')
return ('%s:%s' % (filename, lineno),
self.basepath + relname[1:] + '?view=markup')
class SourceView(AbstractLinkWriter):
def __init__(self, baseurl):
self.baseurl = baseurl
if self.baseurl.endswith("/"):
self.baseurl = baseurl[:-1]
def getlink(self, filename, lineno, funcname):
if filename.endswith('.pyc'):
filename = filename[:-1]
if filename is None:
return "<UNKNOWN>:%s" % funcname,""
pkgpath = self.getpkgpath(filename)
if not filename.startswith(str(pkgpath)):
# let's leave it
return "<UNKNOWN>:%s" % funcname,""
relname = filename[len(str(pkgpath)):]
if relname.endswith('.pyc'):
relname = relname[:-1]
sep = py.std.os.sep
if sep != '/':
relname = relname.replace(sep, '/')
return "%s:%s" % (relname, funcname),\
"%s%s#%s" % (self.baseurl, relname, funcname)
def getlinkobj(self, name, obj):
try:
filename = sys.modules[obj.__module__].__file__
return self.getlink(filename, 0, name)
except AttributeError:
return None
class DirectPaste(AbstractLinkWriter):
""" No-link writer (inliner)
"""
def getlink(self, filename, lineno, funcname):
return ('%s:%s' % (filename, lineno), "")
class DirectFS(AbstractLinkWriter):
""" Creates links to the files on the file system (for local docs)
"""
def getlink(self, filename, lineno, funcname):
return ('%s:%s' % (filename, lineno), 'file://%s' % (filename,))
class PipeWriter(object):
def __init__(self, output=sys.stdout):
self.output = output
def write_section(self, name, rest):
text = "Contents of file %s.txt:" % (name,)
self.output.write(text + "\n")
self.output.write("=" * len(text) + "\n")
self.output.write("\n")
self.output.write(rest.text() + "\n")
def getlink(self, type, targetname, targetfilename):
return '%s.txt' % (targetfilename,)
class DirWriter(object):
def __init__(self, directory=None):
if directory is None:
self.directory = py.test.ensuretemp("rstoutput")
else:
self.directory = py.path.local(directory)
def write_section(self, name, rest):
filename = '%s.txt' % (name,)
self.directory.ensure(filename).write(rest.text())
def getlink(self, type, targetname, targetfilename):
# we assume the result will get converted to HTML...
return '%s.html' % (targetfilename,)
class FileWriter(object):
def __init__(self, fpath):
self.fpath = fpath
self.fp = fpath.open('w+')
self._defined_targets = []
def write_section(self, name, rest):
self.fp.write(rest.text())
self.fp.flush()
def getlink(self, type, targetname, targetbasename):
# XXX problem: because of docutils' named anchor generation scheme,
# a method Foo.__init__ would clash with Foo.init (underscores are
# removed)
if targetname in self._defined_targets:
return None
self._defined_targets.append(targetname)
targetname = targetname.lower().replace('.', '-').replace('_', '-')
while '--' in targetname:
targetname = targetname.replace('--', '-')
if targetname.startswith('-'):
targetname = targetname[1:]
if targetname.endswith('-'):
targetname = targetname[:-1]
return '#%s-%s' % (type, targetname)
class HTMLDirWriter(object):
def __init__(self, indexhandler, filehandler, directory=None):
self.indexhandler = indexhandler
self.filehandler = filehandler
if directory is None:
self.directory = py.test.ensuretemp('dirwriter')
else:
self.directory = py.path.local(directory)
def write_section(self, name, rest):
if name == 'index':
handler = self.indexhandler
else:
handler = self.filehandler
h = handler(name)
t = RestTransformer(rest)
t.parse(h)
self.directory.ensure('%s.html' % (name,)).write(h.html)
def getlink(self, type, targetname, targetfilename):
return '%s.html' % (targetfilename,)
class RestGen(object):
def __init__(self, dsa, linkgen, writer=PipeWriter()):
#assert isinstance(linkgen, DirectPaste), (
# "Cannot use different linkgen by now")
self.dsa = dsa
self.linkgen = linkgen
self.writer = writer
self.tracebacks = {}
def write(self):
"""write the data to the writer"""
modlist = self.get_module_list()
classlist = self.get_class_list(module='')
funclist = self.get_function_list()
modlist.insert(0, ['', classlist, funclist])
indexrest = self.build_index([t[0] for t in modlist])
self.writer.write_section('index', Rest(*indexrest))
self.build_modrest(modlist)
def build_modrest(self, modlist):
modrest = self.build_modules(modlist)
for name, rest, classlist, funclist in modrest:
mname = name
if mname == '':
mname = self.dsa.get_module_name()
self.writer.write_section('module_%s' % (mname,),
Rest(*rest))
for cname, crest, cfunclist in classlist:
self.writer.write_section('class_%s' % (cname,),
Rest(*crest))
for fname, frest, tbdata in cfunclist:
self.writer.write_section('method_%s' % (fname,),
Rest(*frest))
for tbname, tbrest in tbdata:
self.writer.write_section('traceback_%s' % (tbname,),
Rest(*tbrest))
for fname, frest, tbdata in funclist:
self.writer.write_section('function_%s' % (fname,),
Rest(*frest))
for tbname, tbrest in tbdata:
self.writer.write_section('traceback_%s' % (tbname,),
Rest(*tbrest))
def build_classrest(self, classlist):
classrest = self.build_classes(classlist)
for cname, rest, cfunclist in classrest:
self.writer.write_section('class_%s' % (cname,),
Rest(*rest))
for fname, rest in cfunclist:
self.writer.write_section('method_%s' % (fname,),
Rest(*rest))
def build_funcrest(self, funclist):
funcrest = self.build_functions(funclist)
for fname, rest, tbdata in funcrest:
self.writer.write_section('function_%s' % (fname,),
Rest(*rest))
for tbname, tbrest in tbdata:
self.writer.write_section('traceback_%s' % (tbname,),
Rest(*tbrest))
def build_index(self, modules):
rest = [Title('index', abovechar='=', belowchar='=')]
rest.append(Title('exported modules:', belowchar='='))
for module in modules:
mtitle = module
if module == '':
module = self.dsa.get_module_name()
mtitle = '%s (top-level)' % (module,)
linktarget = self.writer.getlink('module', module,
'module_%s' % (module,))
rest.append(ListItem(Link(mtitle, linktarget)))
return rest
def build_modules(self, modules):
ret = []
for module, classes, functions in modules:
mname = module
if mname == '':
mname = self.dsa.get_module_name()
rest = [Title('module: %s' % (mname,), abovechar='=',
belowchar='='),
Title('index:', belowchar='=')]
if classes:
rest.append(Title('classes:', belowchar='^'))
for cls, bases, cfunclist in classes:
linktarget = self.writer.getlink('class', cls,
'class_%s' % (cls,))
rest.append(ListItem(Link(cls, linktarget)))
classrest = self.build_classes(classes)
if functions:
rest.append(Title('functions:', belowchar='^'))
for func in functions:
if module:
func = '%s.%s' % (module, func)
linktarget = self.writer.getlink('function',
func,
'function_%s' % (func,))
rest.append(ListItem(Link(func, linktarget)))
funcrest = self.build_functions(functions, module, False)
ret.append((module, rest, classrest, funcrest))
return ret
def build_classes(self, classes):
ret = []
for cls, bases, functions in classes:
rest = [Title('class: %s' % (cls,), belowchar='='),
LiteralBlock(self.dsa.get_doc(cls))]
# link to source
link_to_class = self.linkgen.getlinkobj(cls, self.dsa.get_obj(cls))
if link_to_class:
rest.append(Paragraph(Text("source: "), Link(*link_to_class)))
if bases:
rest.append(Title('base classes:', belowchar='^')),
for base in bases:
rest.append(ListItem(self.make_class_link(base)))
if functions:
rest.append(Title('functions:', belowchar='^'))
for (func, origin) in functions:
linktarget = self.writer.getlink('method',
'%s.%s' % (cls, func),
'method_%s.%s' % (cls,
func))
rest.append(ListItem(Link('%s.%s' % (cls, func),
linktarget)))
funcrest = self.build_functions(functions, cls, True)
ret.append((cls, rest, funcrest))
return ret
def build_functions(self, functions, parent='', methods=False):
ret = []
for function in functions:
origin = None
if methods:
function, origin = function
if parent:
function = '%s.%s' % (parent, function)
rest, tbrest = self.write_function(function, origin=origin,
ismethod=methods)
ret.append((function, rest, tbrest))
return ret
def get_module_list(self):
visited = []
ret = []
for name in self.dsa.get_class_names():
if '.' in name:
module, classname = split_of_last_part(name)
if module in visited:
continue
visited.append(module)
ret.append((module, self.get_class_list(module),
self.get_function_list(module)))
return ret
def get_class_list(self, module):
ret = []
for name in self.dsa.get_class_names():
classname = name
if '.' in name:
classmodule, classname = split_of_last_part(name)
if classmodule != module:
continue
elif module != '':
continue
bases = self.dsa.get_possible_base_classes(name)
ret.append((name, bases, self.get_method_list(name)))
return ret
def get_function_list(self, module=''):
ret = []
for name in self.dsa.get_function_names():
funcname = name
if '.' in name:
funcpath, funcname = split_of_last_part(name)
if funcpath != module:
continue
elif module != '':
continue
ret.append(funcname)
return ret
def get_method_list(self, classname):
methodnames = self.dsa.get_class_methods(classname)
return [(mn, self.dsa.get_method_origin('%s.%s' % (classname, mn)))
for mn in methodnames]
def process_type_link(self, _type):
# now we do simple type dispatching and provide a link in this case
lst = []
data = self.dsa.get_type_desc(_type)
if not data:
for i in _type.striter():
if isinstance(i, str):
lst.append(i)
else:
lst += self.process_type_link(i)
return lst
name, _desc_type, is_degenerated = data
if not is_degenerated:
linktarget = self.writer.getlink(_desc_type, name,
'%s_%s' % (_desc_type, name))
lst.append(Link(str(_type), linktarget))
else:
# we should provide here some way of linking to sourcegen directly
lst.append(name)
return lst
def write_function(self, functionname, origin=None, ismethod=False,
belowchar='-'):
# XXX I think the docstring should either be split on \n\n and cleaned
# from indentation, or treated as ReST too (although this is obviously
# dangerous for non-ReST docstrings)...
if ismethod:
title = Title('method: %s' % (functionname,), belowchar=belowchar)
else:
title = Title('function: %s' % (functionname,),
belowchar=belowchar)
lst = [title, LiteralBlock(self.dsa.get_doc(functionname)),
LiteralBlock(self.dsa.get_function_definition(functionname))]
link_to_function = self.linkgen.getlinkobj(functionname, self.dsa.get_obj(functionname))
if link_to_function:
lst.insert(1, Paragraph(Text("source: "), Link(*link_to_function)))
opar = Paragraph(Strong('origin'), ":")
if origin:
opar.add(self.make_class_link(origin))
else:
opar.add(Text('<UNKNOWN>'))
lst.append(opar)
lst.append(Paragraph(Strong("where"), ":"))
args, retval = self.dsa.get_function_signature(functionname)
for name, _type in args + [('return value', retval)]:
l = self.process_type_link(_type)
items = []
next = "%s :: " % name
for item in l:
if isinstance(item, str):
next += item
else:
if next:
items.append(Text(next))
next = ""
items.append(item)
if next:
items.append(Text(next))
lst.append(ListItem(*items))
local_changes = self.dsa.get_function_local_changes(functionname)
if local_changes:
lst.append(Paragraph(Strong('changes in __dict__ after execution'), ":"))
for k, changeset in local_changes.iteritems():
lst.append(ListItem('%s: %s' % (k, ', '.join(changeset))))
exceptions = self.dsa.get_function_exceptions(functionname)
if exceptions:
lst.append(Paragraph(Strong('exceptions that might appear during '
'execution'), ":"))
for exc in exceptions:
lst.append(ListItem(exc))
# XXX: right now we leave it alone
# XXX missing implementation of dsa.get_function_location()
#filename, lineno = self.dsa.get_function_location(functionname)
#linkname, linktarget = self.linkgen.getlink(filename, lineno)
#if linktarget:
# lst.append(Paragraph("Function source: ",
# Link(linkname, linktarget)))
#else:
source = self.dsa.get_function_source(functionname)
if source:
lst.append(Paragraph(Strong('function source'), ":"))
lst.append(LiteralBlock(source))
# call sites..
call_sites = self.dsa.get_function_callpoints(functionname)
tbrest = []
if call_sites:
call_site_title = Title("call sites:", belowchar='+')
lst.append(call_site_title)
# we have to think differently here. I would go for:
# 1. A quick'n'dirty statement where call has appeared first
# (topmost)
# 2. Link to short traceback
# 3. Link to long traceback
for call_site, _ in call_sites:
fdata, tbdata = self.call_site_link(functionname, call_site)
lst += fdata
tbrest.append(tbdata)
return lst, tbrest
def call_site_link(self, functionname, call_site):
tbid, tbrest = self.gen_traceback(functionname, call_site)
tbname = '%s.%s' % (functionname, tbid)
linktarget = self.writer.getlink('traceback',
tbname,
'traceback_%s' % (tbname,))
frest = [Paragraph("called in %s" % call_site[0].filename),
Paragraph(Link("traceback %s" % (tbname,),
linktarget))]
return frest, (tbname, tbrest)
def gen_traceback(self, funcname, call_site):
tbid = len(self.tracebacks.setdefault(funcname, []))
self.tracebacks[funcname].append(call_site)
tbrest = [Title('traceback for %s' % (funcname,))]
for line in call_site:
lineno = line.lineno - line.firstlineno
linkname, linktarget = self.linkgen.getlink(line.filename,
line.lineno + 1,
funcname)
if linktarget:
tbrest.append(Paragraph(Link(linkname, linktarget)))
else:
tbrest.append(Paragraph(linkname))
try:
source = line.source
except IOError:
source = "*cannot get source*"
mangled = []
for i, sline in enumerate(str(source).split('\n')):
if i == lineno:
line = '-> %s' % (sline,)
else:
line = ' %s' % (sline,)
mangled.append(line)
tbrest.append(LiteralBlock('\n'.join(mangled)))
return tbid, tbrest
def make_class_link(self, desc):
if not desc or desc.is_degenerated:
# create dummy link here, or no link at all
return Strong(desc.name)
else:
linktarget = self.writer.getlink('class', desc.name,
'class_%s' % (desc.name,))
return Link(desc.name, linktarget)

84
py/apigen/rest/htmlhandlers.py
View File

@ -1,84 +0,0 @@
from py.__.rest.transform import HTMLHandler, entitize
from py.xml import html, raw
class PageHandler(HTMLHandler):
def startDocument(self):
super(PageHandler, self).startDocument()
self.head.append(html.link(type='text/css', rel='stylesheet',
href='style.css'))
title = self.title[0]
breadcrumb = ''.join([unicode(el) for el in self.breadcrumb(title)])
self.body.append(html.div(raw(breadcrumb), class_='breadcrumb'))
def handleLink(self, text, target):
self.tagstack[-1].append(html.a(text, href=target,
target='content'))
def breadcrumb(self, title):
if title != 'index':
type, path = title.split('_', 1)
path = path.split('.')
module = None
cls = None
func = None
meth = None
if type == 'module':
module = '.'.join(path)
elif type == 'class':
module = '.'.join(path[:-1])
cls = path[-1]
elif type == 'method':
module = '.'.join(path[:-2])
cls = path[-2]
meth = path[-1]
else:
module = '.'.join(path[:-1])
func = path[-1]
if module:
yield html.a(module, href='module_%s.html' % (module,))
if type != 'module':
yield u'.'
if cls:
s = cls
if module:
s = '%s.%s' % (module, cls)
yield html.a(cls, href='class_%s.html' % (s,))
if type != 'class':
yield u'.'
if meth:
s = '%s.%s' % (cls, meth)
if module:
s = '%s.%s.%s' % (module, cls, meth)
yield html.a(meth, href='method_%s.html' % (s,))
if func:
s = func
if module:
s = '%s.%s' % (module, func)
yield html.a(func, href='function_%s.html' % (s,))
class IndexHandler(PageHandler):
ignore_text = False
def startDocument(self):
super(IndexHandler, self).startDocument()
self.head.append(html.script(type='text/javascript', src='apigen.js'))
self._push(html.div(id='sidebar'))
def endDocument(self):
maindiv = html.div(id="main")
maindiv.append(html.div(id="breadcrumb"))
maindiv.append(html.iframe(name='content', id='content',
src='module_py.html'))
self.body.append(maindiv)
def startTitle(self, depth):
self.ignore_text = True
def endTitle(self, depth):
self.ignore_text = False
def handleText(self, text):
if self.ignore_text:
return
super(IndexHandler, self).handleText(text)

0
py/apigen/rest/testing/__init__.py
View File

10
py/apigen/rest/testing/somemodule.py
View File

@ -1,10 +0,0 @@
class SomeClass(object):
"""Some class definition"""
def __init__(self, a):
self.a = a
def method(self, a, b, c):
"""method docstring"""
return a + b + c

21
py/apigen/rest/testing/someothermodule.py
View File

@ -1,21 +0,0 @@
from somemodule import SomeClass
class SomeSubClass(SomeClass):
"""Some subclass definition"""
def fun(a, b, c):
"""Some docstring
Let's make it span a couple of lines to be interesting...
Note:
* rest
* should
* be
* supported
* or
* ignored...
"""
return "d"

41
py/apigen/rest/testing/test_htmlhandlers.py
View File

@ -1,41 +0,0 @@
import py
from py.__.apigen.rest.htmlhandlers import PageHandler
def test_breadcrumb():
h = PageHandler()
for fname, expected in [
('module_py', '<a href="module_py.html">py</a>'),
('module_py.test',
'<a href="module_py.test.html">py.test</a>'),
('class_py.test',
('<a href="module_py.html">py</a>.'
'<a href="class_py.test.html">test</a>')),
('class_py.test.foo',
('<a href="module_py.test.html">py.test</a>.'
'<a href="class_py.test.foo.html">foo</a>')),
('class_py.test.foo.bar',
('<a href="module_py.test.foo.html">py.test.foo</a>.'
'<a href="class_py.test.foo.bar.html">bar</a>')),
('function_foo', '<a href="function_foo.html">foo</a>'),
('function_foo.bar',
('<a href="module_foo.html">foo</a>.'
'<a href="function_foo.bar.html">bar</a>')),
('function_foo.bar.baz',
('<a href="module_foo.bar.html">foo.bar</a>.'
'<a href="function_foo.bar.baz.html">baz</a>')),
('method_foo.bar',
('<a href="class_foo.html">foo</a>.'
'<a href="method_foo.bar.html">bar</a>')),
('method_foo.bar.baz',
('<a href="module_foo.html">foo</a>.'
'<a href="class_foo.bar.html">bar</a>.'
'<a href="method_foo.bar.baz.html">baz</a>')),
('method_foo.bar.baz.qux',
('<a href="module_foo.bar.html">foo.bar</a>.'
'<a href="class_foo.bar.baz.html">baz</a>.'
'<a href="method_foo.bar.baz.qux.html">qux</a>')),
]:
html = ''.join([unicode(el) for el in h.breadcrumb(fname)])
print fname
print html
assert html == expected

488
py/apigen/rest/testing/test_rest.py
View File

@ -1,488 +0,0 @@
""" tests document generation
"""
import py
from StringIO import StringIO
from py.__.apigen.rest.genrest import ViewVC, RestGen, PipeWriter, \
DirWriter, FileWriter, \
DirectPaste, DirectFS, \
HTMLDirWriter, SourceView
from py.__.apigen.tracer.tracer import Tracer
from py.__.apigen.tracer.docstorage import DocStorage, DocStorageAccessor
from py.__.apigen.tracer.permastore import PermaDocStorage
import pickle
from py.__.apigen.tracer.testing.runtest import cut_pyc
from py.__.rest.rst import Rest, Paragraph
from py.__.rest.transform import HTMLHandler
# XXX: UUuuuuuuuuuuuuuuuuuuuuuuu, dangerous import
sorted = py.builtin.sorted
def _nl(s):
"""normalize newlines (converting to \n)"""
s = s.replace('\r\n', '\n')
s = s.replace('\r', '\n')
return s
def setup_module(mod):
mod.temppath = py.test.ensuretemp('restgen')
def fun_():
pass
class SomeClass(object):
"""Some class definition"""
def __init__(self, a):
self.a = a
def method(self, a, b, c):
"""method docstring"""
return a + b + c
class SomeSubClass(SomeClass):
"""Some subclass definition"""
def fun(a, b, c):
"""Some docstring
Let's make it span a couple of lines to be interesting...
Note:
* rest
* should
* be
* supported
* or
* ignored...
"""
return "d"
def test_direct_link():
fname = cut_pyc(__file__)
title, link = DirectPaste().getlink(fname, 2, "")
assert title == '%s:%s' % (fname, 2)
assert link == ''
def test_viewvc_link():
vcview = ViewVC("http://codespeak.net/viewvc/")
fname = cut_pyc(__file__)
title, link = vcview.getlink(fname, 0, "")
assert title == '%s:%s' % (fname, 0)
assert link == ('http://codespeak.net/viewvc/py/apigen/rest/'
'testing/test_rest.py?view=markup')
def test_fs_link():
title, link = DirectFS().getlink('/foo/bar/baz.py', 100, "func")
assert title == '/foo/bar/baz.py:100'
assert link == 'file:///foo/bar/baz.py'
class WriterTest(object):
def get_filled_writer(self, writerclass, *args, **kwargs):
dw = writerclass(*args, **kwargs)
dw.write_section('foo', Rest(Paragraph('foo data')))
dw.write_section('bar', Rest(Paragraph('bar data')))
return dw
class TestDirWriter(WriterTest):
def test_write_section(self):
tempdir = temppath.ensure('dirwriter', dir=True)
dw = self.get_filled_writer(DirWriter, tempdir)
fpaths = tempdir.listdir('*.txt')
assert len(fpaths) == 2
assert sorted([f.basename for f in fpaths]) == ['bar.txt', 'foo.txt']
assert _nl(tempdir.join('foo.txt').read()) == 'foo data\n'
assert _nl(tempdir.join('bar.txt').read()) == 'bar data\n'
def test_getlink(self):
dw = DirWriter(temppath.join('dirwriter_getlink'))
link = dw.getlink('function', 'Foo.bar', 'method_foo_bar')
assert link == 'method_foo_bar.html'
class TestFileWriter(WriterTest):
def test_write_section(self):
tempfile = temppath.ensure('filewriter', file=True)
fw = self.get_filled_writer(FileWriter, tempfile)
data = tempfile.read()
assert len(data)
def test_getlink(self):
fw = FileWriter(temppath.join('filewriter_getlink'))
link = fw.getlink('function', 'Foo.bar', 'method_foo_bar')
assert link == '#function-foo-bar'
# only produce the same link target once...
link = fw.getlink('function', 'Foo.bar', 'method_foo_bar')
assert link is None
link = fw.getlink('function', 'Foo.__init__', 'method_foo___init__')
assert link == '#function-foo-init'
class TestPipeWriter(WriterTest):
def test_write_section(self):
s = StringIO()
pw = self.get_filled_writer(PipeWriter, s)
data = s.getvalue()
assert len(data)
def test_getlink(self):
pw = PipeWriter(StringIO())
link = pw.getlink('function', 'Foo.bar', 'method_foo_bar')
assert link == 'method_foo_bar.txt'
class TestHTMLDirWriter(WriterTest):
def test_write_section(self):
tempdir = temppath.ensure('htmldirwriter', dir=1)
hdw = self.get_filled_writer(HTMLDirWriter, HTMLHandler, HTMLHandler,
tempdir)
assert tempdir.join('foo.html').check(file=1)
assert tempdir.join('bar.html').check(file=1)
assert tempdir.join('foo.html').read().startswith('<html>')
class TestRest(object):
def get_filled_docstorage(self):
descs = {'SomeClass': SomeClass,
'SomeSubClass': SomeSubClass,
'fun':fun}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
s1 = SomeClass("a")
fun(1, 2, s1)
s2 = SomeSubClass("b")
s2.method(1,2,3)
fun(1, 3, s2)
t.end_tracing()
return DocStorageAccessor(ds)
def get_filled_docstorage_modules(self):
import somemodule
import someothermodule
descs = {
'somemodule.SomeClass': somemodule.SomeClass,
'someothermodule.SomeSubClass': someothermodule.SomeSubClass,
'someothermodule.fun': someothermodule.fun,
}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
s1 = somemodule.SomeClass("a")
someothermodule.fun(1, 2, s1)
s2 = someothermodule.SomeSubClass("b")
s2.method(1, 2, 3)
someothermodule.fun(1, 3, s2)
t.end_tracing()
return DocStorageAccessor(ds)
def check_rest(self, tempdir):
from py.__.misc import rest
for path in tempdir.listdir('*.txt'):
try:
rest.process(path)
except ImportError:
py.test.skip('skipping rest generation because docutils is '
'not installed (this is a partial skip, the rest '
'of the test was successful)')
py.test.skip("partial skip: find a nice way to re-use pytest_restdoc's genlinkchecks")
# XXX find a nice way check pytest_restdoc's genlinkchecks()
#for path in tempdir.listdir('*.txt'):
# for item, arg1, arg2, arg3 in genlinkchecks(path):
# item(arg1, arg2, arg3)
def test_generation_simple_api(self):
ds = self.get_filled_docstorage()
lg = DirectPaste()
tempdir = temppath.ensure("simple_api", dir=True)
r = RestGen(ds, lg, DirWriter(tempdir))
r.write()
basenames = [p.basename for p in tempdir.listdir('*.txt')]
expected = [
'class_SomeClass.txt',
'class_SomeSubClass.txt',
'function_fun.txt',
'index.txt',
'method_SomeClass.__init__.txt',
'method_SomeClass.method.txt',
'method_SomeSubClass.__init__.txt',
'method_SomeSubClass.method.txt',
'module_Unknown module.txt',
'traceback_SomeClass.__init__.0.txt',
'traceback_SomeSubClass.__init__.0.txt',
'traceback_SomeSubClass.method.0.txt',
'traceback_fun.0.txt',
'traceback_fun.1.txt',
]
print sorted(basenames)
assert sorted(basenames) == expected
# now we check out...
self.check_rest(tempdir)
tempdir = temppath.ensure("simple_api_ps", dir=True)
if 0:
ps = PermaDocStorage(ds)
r = RestGen(ps, lg, DirWriter(tempdir))
r.write()
basenames = [p.basename for p in tempdir.listdir('*.txt')]
assert sorted(basenames) == expected
self.check_rest(tempdir)
pickle.dumps(ps)
def test_generation_modules(self):
ds = self.get_filled_docstorage_modules()
lg = DirectPaste()
tempdir = temppath.ensure('module_api', dir=True)
r = RestGen(ds, lg, DirWriter(tempdir))
r.write()
basenames = [p.basename for p in tempdir.listdir('*.txt')]
expected = [
'class_somemodule.SomeClass.txt',
'class_someothermodule.SomeSubClass.txt',
'function_someothermodule.fun.txt',
'index.txt',
'method_somemodule.SomeClass.__init__.txt',
'method_somemodule.SomeClass.method.txt',
'method_someothermodule.SomeSubClass.__init__.txt',
'method_someothermodule.SomeSubClass.method.txt',
'module_Unknown module.txt',
'module_somemodule.txt',
'module_someothermodule.txt',
'traceback_somemodule.SomeClass.__init__.0.txt',
'traceback_someothermodule.SomeSubClass.__init__.0.txt',
'traceback_someothermodule.SomeSubClass.method.0.txt',
'traceback_someothermodule.fun.0.txt',
'traceback_someothermodule.fun.1.txt',
]
print sorted(basenames)
assert sorted(basenames) == expected
def test_check_internal_links(self):
ds = self.get_filled_docstorage()
lg = DirectFS()
tempdir = temppath.ensure('internal_links', dir=True)
r = RestGen(ds, lg, DirWriter(tempdir))
r.write()
index = tempdir.join('module_Unknown module.txt')
assert index.check(file=True)
data = _nl(index.read())
assert data.find('.. _`fun`: function_fun.html\n') > -1
assert data.find('.. _`fun`: #function-fun\n') == -1
tempfile = temppath.ensure('internal_links.txt',
file=True)
r = RestGen(ds, lg, FileWriter(tempfile))
r.write()
data = _nl(tempfile.read())
assert data.find('.. _`fun`: #function-fun\n') > -1
assert data.find('.. _`fun`: function_fun.html') == -1
tempfile = temppath.ensure("internal_links_ps.txt", file=True)
if 0:
ps = PermaDocStorage(ds)
r = RestGen(ps, lg, FileWriter(tempfile))
r.write()
data = _nl(tempfile.read())
assert data.find('.. _`fun`: #function-fun\n') > -1
assert data.find('.. _`fun`: function_fun.html') == -1
pickle.dumps(ps)
def test_check_section_order(self):
# we use the previous method's data
tempfile = temppath.join('internal_links.txt')
if not tempfile.check():
py.test.skip('depends on previous test, which failed')
data = _nl(tempfile.read())
# index should be above the rest
assert data.find('classes\\:') > -1
assert data.find('classes\\:') < data.find('function\\: fun')
assert data.find('classes\\:') < data.find(
'class\\: SomeClass')
# function definitions should be above class ones
assert data.find('function\\: fun') > data.find('class\\: SomeClass')
# class method definitions should be below the class defs
assert data.find('class\\: SomeClass') < data.find(
'method\\: SomeClass.method')
# __init__ should be above other methods
assert data.find('method\\: SomeClass.\\_\\_init\\_\\_') > -1
assert data.find('method\\: SomeClass.\\_\\_init\\_\\_') < data.find(
'method\\: SomeClass.method')
# base class info
assert py.std.re.search(r'class\\\: SomeSubClass.*'
r'base classes\\\:\n\^+[\n ]+\* `SomeClass`_.*'
r'`SomeSubClass.__init__',
data, py.std.re.S)
def test_som_fun(self):
descs = {'fun_': fun_}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
fun_()
t.end_tracing()
lg = DirectPaste()
tempdir = temppath.ensure("some_fun", dir=True)
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
self.check_rest(tempdir)
def test_function_source(self):
def blah():
a = 3
b = 4
return a + b
descs = {'blah': blah}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
blah()
t.end_tracing()
lg = DirectPaste()
tempdir = temppath.ensure("function_source", dir=True)
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
assert tempdir.join("function_blah.txt").read().find("a = 3") != -1
self.check_rest(tempdir)
ps = DocStorageAccessor(ds)
r = RestGen(ps, lg, DirWriter(tempdir))
r.write()
assert tempdir.join("function_blah.txt").read().find("a = 3") != -1
def test_function_arguments(self):
def blah(a, b, c):
return "axx"
class C:
pass
descs = {'blah':blah}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
blah(3, "x", C())
t.end_tracing()
lg = DirectPaste()
tempdir = temppath.ensure("function_args", dir=True)
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
source = tempdir.join("function_blah.txt").read()
call_point = source.find("call sites\:")
assert call_point != -1
assert source.find("a \:\: <Int>") < call_point
assert source.find("b \:\: <String>") < call_point
assert source.find("c \:\: <Instance of Class C>") < call_point
self.check_rest(tempdir)
def test_class_typedefs(self):
class A(object):
def __init__(self, x):
pass
def a(self):
pass
class B(A):
def __init__(self, y):
pass
def xxx(x):
return x
descs = {'A': A, 'B': B, 'xxx':xxx}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
xxx(A(3))
xxx(B("f"))
t.end_tracing()
lg = DirectPaste()
tempdir = temppath.ensure("classargs", dir=True)
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
source = tempdir.join("function_xxx.txt").read()
call_point = source.find("call sites\:")
assert call_point != -1
print source
assert -1 < source.find("x \:\: <Instance of AnyOf( `Class B`_ , "
"`Class A`_ )>") < call_point
source = tempdir.join('method_B.a.txt').read()
py.test.skip('XXX needs to be fixed, clueless atm though')
assert source.find('**origin** \: `A`_') > -1
self.check_rest(tempdir)
def test_exc_raising(self):
def x():
try:
1/0
except:
pass
descs = {'x':x}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
x()
t.end_tracing()
lg = DirectPaste()
tempdir = temppath.ensure("exc_raising", dir=True)
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
source = tempdir.join('function_x.txt').open().read()
assert source.find('ZeroDivisionError') < source.find('call sites\:')
def test_nonexist_origin(self):
class A:
def method(self):
pass
class B(A):
pass
descs = {'B':B}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
B().method()
t.end_tracing()
lg = DirectPaste()
tempdir = temppath.ensure("nonexit_origin", dir=True)
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
self.check_rest(tempdir)
def test_sourceview(self):
class A:
def method(self):
pass
descs = {'A':A}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
A().method()
t.end_tracing()
lg = SourceView('http://localhost:8000')
tempdir = temppath.ensure("sourceview", dir=True)
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
self.check_rest(tempdir)
assert tempdir.join('traceback_A.method.0.txt').open().read().find(
'.. _`/py/apigen/rest/testing/test\_rest.py\:A.method`: http://localhost:8000/py/apigen/rest/testing/test_rest.py#A.method') != -1
def test_sourceview_fun(self):
def f():
pass
descs = {'f':f}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
f()
t.end_tracing()
tempdir = temppath.ensure("sourceview_fun", dir=True)
lg = SourceView('http://localhost:8000')
r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
r.write()
self.check_rest(tempdir)
assert tempdir.join('function_f.txt').open().read().find(
'.. _`/py/apigen/rest/testing/test\_rest.py\:f`: http://localhost:8000/py/apigen/rest/testing/test_rest.py#f') != -1

0
py/apigen/source/__init__.py
View File

143
py/apigen/source/browser.py
View File

@ -1,143 +0,0 @@
""" source browser using compiler module
WARNING!!!
This is very simple and very silly attempt to make so.
"""
from compiler import parse, ast
import py
from py.__.path.common import PathBase
blockers = [ast.Function, ast.Class]
class BaseElem(object):
def listnames(self):
if getattr(self, 'parent', None):
return self.parent.listnames() + '.' + self.name
return self.name
class Module(BaseElem):
def __init__(self, path, _dict):
self.path = path
self.dict = _dict
def __getattr__(self, attr):
try:
return self.dict[attr]
except KeyError:
raise AttributeError(attr)
def get_children(self):
values = self.dict.values()
all = values[:]
for v in values:
all += v.get_children()
return all
def get_endline(start, lst):
l = lst[::-1]
for i in l:
if i.lineno:
return i.lineno
end_ch = get_endline(None, i.getChildNodes())
if end_ch:
return end_ch
return start
class Function(BaseElem):
def __init__(self, name, parent, firstlineno, endlineno):
self.firstlineno = firstlineno
self.endlineno = endlineno
self.name = name
self.parent = parent
def get_children(self):
return []
class Method(BaseElem):
def __init__(self, name, parent, firstlineno, endlineno):
self.name = name
self.firstlineno = firstlineno
self.endlineno = endlineno
self.parent = parent
def function_from_ast(ast, cls_ast, cls=Function):
startline = ast.lineno
endline = get_endline(startline, ast.getChildNodes())
assert endline
return cls(ast.name, cls_ast, startline, endline)
def class_from_ast(cls_ast):
bases = [i.name for i in cls_ast.bases if isinstance(i, ast.Name)]
# XXX
methods = {}
startline = cls_ast.lineno
name = cls_ast.name
endline = get_endline(startline, cls_ast.getChildNodes())
cls = Class(name, startline, endline, bases, [])
cls.methods = dict([(i.name, function_from_ast(i, cls, Method)) for i in \
cls_ast.code.nodes if isinstance(i, ast.Function)])
return cls
class Class(BaseElem):
def __init__(self, name, firstlineno, endlineno, bases, methods):
self.bases = bases
self.firstlineno = firstlineno
self.endlineno = endlineno
self.name = name
self.methods = methods
def __getattr__(self, attr):
try:
return self.methods[attr]
except KeyError:
raise AttributeError(attr)
def get_children(self):
return self.methods.values()
def dir_nodes(st):
""" List all the subnodes, which are not blockers
"""
res = []
for i in st.getChildNodes():
res.append(i)
if not i.__class__ in blockers:
res += dir_nodes(i)
return res
def update_mod_dict(imp_mod, mod_dict):
# make sure that things that are in mod_dict, and not in imp_mod,
# are not shown
for key, value in mod_dict.items():
if not hasattr(imp_mod, key):
del mod_dict[key]
def parse_path(path):
if not isinstance(path, PathBase):
path = py.path.local(path)
buf = path.open().read()
st = parse(buf)
# first go - we get all functions and classes defined on top-level
nodes = dir_nodes(st)
function_ast = [i for i in nodes if isinstance(i, ast.Function)]
classes_ast = [i for i in nodes if isinstance(i, ast.Class)]
mod_dict = dict([(i.name, function_from_ast(i, None)) for i in function_ast]
+ [(i.name, class_from_ast(i)) for i in classes_ast])
# we check all the elements, if they're really there
try:
mod = path.pyimport()
except (KeyboardInterrupt, SystemExit):
raise
except: # catch all other import problems generically
# XXX some import problem: we probably should not
# pretend to have an empty module
pass
else:
update_mod_dict(mod, mod_dict)
return Module(path, mod_dict)

211
py/apigen/source/color.py
View File

@ -1,211 +0,0 @@
""" simple Python syntax coloring """
import re
class PythonSchema(object):
""" contains information for syntax coloring """
comment = [('#', '\n'), ('#', '$')]
multiline_string = ['"""', "'''"]
string = ['"""', "'''", '"', "'"]
keyword = ['and', 'break', 'continue', 'elif', 'else', 'except',
'finally', 'for', 'if', 'in', 'is', 'not', 'or', 'raise',
'return', 'try', 'while', 'with', 'yield']
alt_keyword = ['as', 'assert', 'class', 'def', 'del', 'exec', 'from',
'global', 'import', 'lambda', 'pass', 'print']
linejoin = r'\\'
def assert_keywords():
from keyword import kwlist
all = PythonSchema.keyword + PythonSchema.alt_keyword
for x in kwlist:
assert x in all
assert_keywords()
class Token(object):
data = None
type = 'unknown'
def __init__(self, data, type='unknown'):
self.data = data
self.type = type
def __repr__(self):
return '<Token type="%s" %r>' % (self.type, self.data)
def __eq__(self, other):
return self.data == other.data and self.type == other.type
def __ne__(self, other):
return not self.__eq__(other)
class Tokenizer(object):
""" when fed lists strings, it will return tokens with type info
very naive tokenizer, state is recorded for multi-line strings, etc.
"""
_re_word = re.compile('[\w_]+', re.U)
_re_space = re.compile('\s+', re.U)
_re_number = re.compile('[\d\.]*\d[\d\.]*l?', re.I | re.U)
# XXX cheating a bit with the quotes
_re_rest = re.compile('[^\w\s\d\'"]+', re.U)
# these will be filled using the schema
_re_strings_full = None
_re_strings_multiline = None
_re_strings_comments = None
def __init__(self, schema):
self.schema = schema
self._inside_multiline = False
self._re_strings_full = []
self._re_strings_multiline = []
self._re_strings_empty = []
for d in schema.string + schema.multiline_string:
self._re_strings_full.append(
re.compile(r'%s[^\\%s]*(\\.[^\\%s]*)+%s' % (d, d, d, d)))
self._re_strings_full.append(
re.compile(r'%s[^\\%s]+(\\.[^\\%s]*)*%s' % (d, d, d, d)))
self._re_strings_empty.append(re.compile('%s%s' % (d, d)))
for d in schema.multiline_string:
self._re_strings_multiline.append((re.compile('(%s).*' % (d,),
re.S),
re.compile('.*?%s' % (d,))))
if schema.linejoin:
j = schema.linejoin
for d in schema.string:
self._re_strings_multiline.append(
(re.compile('(%s).*%s$' % (d, j)),
re.compile('.*?%s' % (d,))))
# no multi-line comments in Python... phew :)
self._re_comments = []
for start, end in schema.comment:
self._re_comments.append(re.compile('%s.*?%s' % (start, end)))
def tokenize(self, data):
if self._inside_multiline:
m = self._inside_multiline.match(data)
if not m:
yield Token(data, 'string')
data = ''
else:
s = m.group(0)
data = data[len(s):]
self._inside_multiline = False
yield Token(s, 'string')
while data:
for f in [self._check_full_strings, self._check_multiline_strings,
self._check_empty_strings, self._check_comments,
self._check_number, self._check_space, self._check_word,
self._check_rest]:
data, t = f(data)
if t:
yield t
break
else:
raise ValueError(
'no token found in %r (bug in tokenizer)' % (data,))
def _check_full_strings(self, data):
token = None
for r in self._re_strings_full:
m = r.match(data)
if m:
s = m.group(0)
data = data[len(s):]
token = Token(s, type='string')
break
return data, token
def _check_multiline_strings(self, data):
token = None
for start, end in self._re_strings_multiline:
m = start.match(data)
if m:
s = m.group(0)
data = ''
# XXX take care of a problem which is hard to fix with regexps:
# '''foo 'bar' baz''' will not match single-line strings
# (because [^"""] matches just a single " already), so let's
# try to catch it here... (quite Python specific issue!)
endm = end.match(s[len(m.group(1)):])
if endm: # see if it ends here already
s = m.group(1) + endm.group(0)
else:
self._inside_multiline = end
token = Token(s, 'string')
break
return data, token
def _check_empty_strings(self, data):
token = None
for r in self._re_strings_empty:
m = r.match(data)
if m:
s = m.group(0)
data = data[len(s):]
token = Token(s, type='string')
break
return data, token
def _check_comments(self, data):
# fortunately we don't have to deal with multi-line comments
token = None
for r in self._re_comments:
m = r.match(data)
if m:
s = m.group(0)
data = data[len(s):]
token = Token(s, 'comment')
break
return data, token
def _check_word(self, data):
m = self._re_word.match(data)
if m:
s = m.group(0)
type = 'word'
if s in self.schema.keyword:
type = 'keyword'
elif s in self.schema.alt_keyword:
type = 'alt_keyword'
return data[len(s):], Token(s, type)
return data, None
def _check_space(self, data):
m = self._re_space.match(data)
if m:
s = m.group(0)
return data[len(s):], Token(s, 'whitespace')
return data, None
def _check_number(self, data):
m = self._re_number.match(data)
if m:
s = m.group(0)
return data[len(s):], Token(s, 'number')
return data, None
def _check_rest(self, data):
m = self._re_rest.match(data)
if m:
s = m.group(0)
return data[len(s):], Token(s, 'unknown')
return data, None
if __name__ == '__main__':
import py, sys
if len(sys.argv) != 2:
print 'usage: %s <filename>'
print ' tokenizes the file and prints the tokens per line'
sys.exit(1)
t = Tokenizer(PythonSchema)
p = py.path.local(sys.argv[1])
assert p.ext == '.py'
for line in p.read().split('\n'):
print repr(line)
print 't in multiline mode:', not not t._inside_multiline
tokens = t.tokenize(line)
print list(tokens)

304
py/apigen/source/html.py
View File

@ -1,304 +0,0 @@
""" html - generating ad-hoc html out of source browser
"""
import py
from py.xml import html, raw
from compiler import ast
import time
from py.__.apigen.source.color import Tokenizer, PythonSchema
from py.__.apigen.source.browser import parse_path
class CompilationException(Exception):
""" raised when something goes wrong while importing a module """
class HtmlEnchanter(object):
def __init__(self, mod):
self.mod = mod
self.create_caches()
def create_caches(self):
mod = self.mod
linecache = {}
for item in mod.get_children():
linecache[item.firstlineno] = item
self.linecache = linecache
def enchant_row(self, num, row):
# add some informations to row, like functions defined in that
# line, etc.
try:
item = self.linecache[num]
# XXX: this should not be assertion, rather check, but we want to
# know if stuff is working
pos = row.find(item.name)
assert pos != -1
end = len(item.name) + pos
chunk = html.a(row[pos:end], href="#" + item.listnames(),
name=item.listnames())
return [row[:pos], chunk, row[end:]]
except KeyError:
return [row] # no more info
def prepare_line(text, tokenizer, encoding):
""" adds html formatting to text items (list)
only processes items if they're of a string type (or unicode)
"""
ret = []
for item in text:
if type(item) in [str, unicode]:
tokens = tokenizer.tokenize(item)
for t in tokens:
if not isinstance(t.data, unicode):
data = unicode(t.data, encoding)
else:
data = t.data
if t.type in ['keyword', 'alt_keyword', 'number',
'string', 'comment']:
ret.append(html.span(data, class_=t.type))
else:
ret.append(data)
else:
ret.append(item)
return ret
def prepare_module(path, tokenizer, encoding):
path = py.path.local(path)
try:
mod = parse_path(path)
except:
# XXX don't try to catch SystemExit: it's actually raised by one
# of the modules in the py lib on import :(
exc, e, tb = py.std.sys.exc_info()
del tb
raise CompilationException('while compiling %s: %s - %s' % (
path, e.__class__.__name__, e))
lines = [unicode(l, encoding) for l in path.readlines()]
enchanter = HtmlEnchanter(mod)
ret = []
for i, line in enumerate(lines):
text = enchanter.enchant_row(i + 1, line)
if text == ['']:
text = [raw('&#xa0;')]
else:
text = prepare_line(text, tokenizer, encoding)
ret.append(text)
return ret
class HTMLDocument(object):
def __init__(self, encoding, tokenizer=None):
self.encoding = encoding
self.html = root = html.html()
self.head = head = self.create_head()
root.append(head)
self.body = body = self.create_body()
root.append(body)
self.table, self.tbody = table, tbody = self.create_table()
body.append(table)
if tokenizer is None:
tokenizer = Tokenizer(PythonSchema)
self.tokenizer = tokenizer
def create_head(self):
return html.head(
html.title('source view'),
html.style("""
body, td {
background-color: #FFF;
color: black;
font-family: monospace, Monaco;
}
table, tr {
margin: 0px;
padding: 0px;
border-width: 0px;
}
a {
color: blue;
font-weight: bold;
text-decoration: none;
}
a:hover {
color: #005;
}
.lineno {
text-align: right;
color: #555;
width: 3em;
padding-right: 1em;
border: 0px solid black;
border-right-width: 1px;
}
.code {
padding-left: 1em;
white-space: pre;
}
.comment {
color: purple;
}
.string {
color: #777;
}
.keyword {
color: blue;
}
.alt_keyword {
color: green;
}
""", type='text/css'),
)
def create_body(self):
return html.body()
def create_table(self):
table = html.table(cellpadding='0', cellspacing='0')
tbody = html.tbody()
table.append(tbody)
return table, tbody
def add_row(self, lineno, text):
if text == ['']:
text = [raw('&#xa0;')]
else:
text = prepare_line(text, self.tokenizer, self.encoding)
self.tbody.append(html.tr(html.td(str(lineno), class_='lineno'),
html.td(class_='code', *text)))
def __unicode__(self):
# XXX don't like to use indent=0 here, but else py.xml's indentation
# messes up the html inside the table cells (which displays formatting)
return self.html.unicode(indent=0)
def create_html(mod):
# out is some kind of stream
#*[html.tr(html.td(i.name)) for i in mod.get_children()]
lines = mod.path.open().readlines()
enchanter = HtmlEnchanter(mod)
enc = get_module_encoding(mod.path)
doc = HTMLDocument(enc)
for i, row in enumerate(lines):
row = enchanter.enchant_row(i + 1, row)
doc.add_row(i + 1, row)
return unicode(doc)
style = html.style("""
body, p, td {
background-color: #FFF;
color: black;
font-family: monospace, Monaco;
}
td.type {
width: 2em;
}
td.name {
width: 30em;
}
td.mtime {
width: 13em;
}
td.size {
text-alignment: right;
}
""")
def create_dir_html(path, href_prefix=''):
h = html.html(
html.head(
html.title('directory listing of %s' % (path,)),
style,
),
)
body = html.body(
html.h1('directory listing of %s' % (path,)),
)
h.append(body)
table = html.table()
body.append(table)
tbody = html.tbody()
table.append(tbody)
items = list(path.listdir())
items.sort(key=lambda p: p.basename)
items.sort(key=lambda p: not p.check(dir=True))
for fpath in items:
tr = html.tr()
tbody.append(tr)
td1 = html.td(fpath.check(dir=True) and 'D' or 'F', class_='type')
tr.append(td1)
href = fpath.basename
if href_prefix:
href = '%s%s' % (href_prefix, href)
if fpath.check(dir=True):
href += '/'
td2 = html.td(html.a(fpath.basename, href=href), class_='name')
tr.append(td2)
td3 = html.td(time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(fpath.mtime())), class_='mtime')
tr.append(td3)
if fpath.check(dir=True):
size = ''
unit = ''
else:
size = fpath.size()
unit = 'B'
for u in ['kB', 'MB', 'GB', 'TB']:
if size > 1024:
size = round(size / 1024.0, 2)
unit = u
td4 = html.td('%s %s' % (size, unit), class_='size')
tr.append(td4)
return unicode(h)
def create_unknown_html(path):
h = html.html(
html.head(
html.title('Can not display page'),
style,
),
html.body(
html.p('The data URL (%s) does not contain Python code.' % (path,))
),
)
return h.unicode()
_reg_enc = py.std.re.compile(r'coding[:=]\s*([-\w.]+)')
def get_module_encoding(path):
if hasattr(path, 'strpath'):
path = path.strpath
if path[-1] in ['c', 'o']:
path = path[:-1]
fpath = py.path.local(path)
fp = fpath.open()
lines = []
try:
# encoding is only allowed in the first two lines
for i in range(2):
lines.append(fp.readline())
finally:
fp.close()
match = _reg_enc.search('\n'.join(lines))
if match:
return match.group(1)
return 'ISO-8859-1'

29
py/apigen/source/index.cgi
View File

@ -1,29 +0,0 @@
#!/usr/bin/python
import cgitb;cgitb.enable()
import path
import py
from py.__.apigen.source.browser import parse_path
from py.__.apigen.source.html import create_html, create_dir_html, \
create_unknown_html
BASE_URL='http://codespeak.net/svn/py/dist'
def cgi_main():
import os
reqpath = os.environ.get('PATH_INFO', '')
path = py.path.svnurl('%s%s' % (BASE_URL, reqpath))
if not path.check():
return create_unknown_html(path)
if path.check(file=True):
return unicode(create_html(parse_path(path)))
elif path.check(dir=True):
prefix = ''
if not reqpath:
prefix = 'index.cgi/'
return create_dir_html(path, href_prefix=prefix)
else:
return create_unknown_html(path)
print 'Content-Type: text/html; charset=UTF-8'
print
print cgi_main()

2
py/apigen/source/path.py
View File

@ -1,2 +0,0 @@
import os, sys
sys.path = ['/'.join(os.path.dirname(__file__).split(os.sep)[:-3])] + sys.path

45
py/apigen/source/server.py
View File

@ -1,45 +0,0 @@
""" web server for displaying source
"""
import py
from pypy.translator.js.examples import server
from py.__.apigen.source.browser import parse_path
from py.__.apigen.source.html import create_html, create_dir_html, create_unknown_html
from py.xml import html
class Handler(server.TestHandler):
BASE_URL='http://codespeak.net/svn/py/dist'
def __getattr__(self, attr):
if attr == 'index':
attr = ''
url = self.BASE_URL + "/" + attr
if url.endswith('_py'):
url = url[:-3] + '.py'
path = py.path.svnurl(url)
if not path.check():
def f(rev=None):
return create_unknown_html(path)
f.exposed = True
f.func_name = attr
return f
def f(rev='HEAD'):
path = py.path.svnurl(url, rev)
# some try.. except.. here
if path.check(file=True):
return unicode(create_html(parse_path(path)))
elif path.check(dir=True):
return create_dir_html(path)
else:
return create_unknown_html(path)
f.exposed = True
f.func_name = attr
return f
def _main():
server.start_server(handler=Handler)
if __name__ == '__main__':
_main()

0
py/apigen/source/testing/__init__.py
View File

80
py/apigen/source/testing/test_browser.py
View File

@ -1,80 +0,0 @@
""" test source browser abilities
"""
from py.__.apigen.source.browser import parse_path, Class, Function, Method
import py
def test_browser():
tmp = py.test.ensuretemp("sourcebrowser")
tmp.ensure("a.py").write(py.code.Source("""
def f():
pass
def g():
pass
class X:
pass
class Z(object):
x = 1
def zzz(self):
1
2
3
4
"""))
mod = parse_path(tmp.join("a.py"))
assert isinstance(mod.g, Function)
assert isinstance(mod.Z, Class)
py.test.raises(AttributeError, "mod.zzz")
assert mod.g.firstlineno == 5
assert mod.g.name == "g"
assert mod.g.endlineno == 6
assert mod.X.firstlineno == 8
assert mod.X.endlineno == 9
assert mod.Z.bases == ["object"]
assert isinstance(mod.Z.zzz, Method)
assert mod.Z.zzz.firstlineno == 13
assert mod.Z.zzz.endlineno == 17
def test_if_browser():
tmp = py.test.ensuretemp("sourcebrowser")
tmp.ensure("b.py").write(py.code.Source("""
if 1:
def f():
pass
if 0:
def g():
pass
"""))
mod = parse_path(tmp.join("b.py"))
assert isinstance(mod.f, Function)
py.test.raises(AttributeError, 'mod.g')
def test_bases():
tmp = py.test.ensuretemp("sourcebrowser")
tmp.ensure("c.py").write(py.code.Source("""
import py
class Dir(py.test.collect.Directory):
pass
"""))
mod = parse_path(tmp.join("c.py"))
# if it does not rise it's ok for now
#
def test_importing_goes_wrong():
tmp = py.test.ensuretemp("sourcebrowserimport")
tmp.ensure("x.py").write(py.code.Source("""
import aslkdjaslkdjasdl
"""))
mod = parse_path(tmp.join("x.py"))
tmp.ensure("y.py").write(py.code.Source("""
raise KeyboardInterrupt
"""))
py.test.raises(KeyboardInterrupt, 'parse_path(tmp.join("y.py"))')
# if it does not rise it's ok for now
#

97
py/apigen/source/testing/test_color.py
View File

@ -1,97 +0,0 @@
import py
from py.__.apigen.source.color import Tokenizer, Token, PythonSchema
class TestTokenizer(object):
def tokens(self, data):
t = Tokenizer(PythonSchema)
return list(t.tokenize(data))
def test_word(self):
assert self.tokens('foo') == [Token('foo', type='word')]
assert self.tokens('_1_word') == [Token('_1_word', type='word')]
def test_keyword(self):
assert 'if' in PythonSchema.keyword
assert self.tokens('see if it works') == [Token('see', type='word'),
Token(' ',
type='whitespace'),
Token('if', type='keyword'),
Token(' ',
type='whitespace'),
Token('it', type='word'),
Token(' ',
type='whitespace'),
Token('works', type='word')]
def test_space(self):
assert self.tokens(' ') == [Token(' ', type='whitespace')]
assert self.tokens(' \n') == [Token(' \n', type='whitespace')]
def test_number(self):
# XXX incomplete
assert self.tokens('1') == [Token('1', type='number')]
assert self.tokens('1.1') == [Token('1.1', type='number')]
assert self.tokens('.1') == [Token('.1', type='number')]
assert self.tokens('1.') == [Token('1.', type='number')]
assert self.tokens('1.1l') == [Token('1.1l', type='number')]
def test_printable(self):
assert self.tokens('.') == [Token('.', 'unknown')]
assert self.tokens(';#$@\n') == [Token(';#$@', type='unknown'),
Token('\n', type='whitespace')]
def test_comment(self):
assert self.tokens('# foo\n') == [Token('# foo\n', type='comment')]
assert self.tokens('foo # bar\n') == [Token('foo', type='word'),
Token(' ', type='whitespace'),
Token('# bar\n', type='comment')]
assert self.tokens("# foo 'bar\n") == [Token("# foo 'bar\n",
type='comment')]
assert self.tokens('# foo') == [Token('# foo', type='comment')]
def test_string_simple(self):
assert self.tokens('""') == [Token('""', type='string')]
assert self.tokens('"foo"') == [Token('"foo"', type='string')]
assert self.tokens('"foo"\'bar\'') == [Token('"foo"', type='string'),
Token("'bar'", type='string')]
def test_string_escape(self):
assert self.tokens('"foo \\" bar"') == [Token('"foo \\" bar"',
type='string')]
def test_string_multiline(self):
t = Tokenizer(PythonSchema)
res = list(t.tokenize('"""foo\n'))
assert res == [Token('"""foo\n', type='string')]
res = list(t.tokenize('bar\n'))
assert res == [Token('bar\n', type='string')]
res = list(t.tokenize('"""\n'))
assert res == [Token('"""', type='string'),
Token('\n', type='whitespace')]
# tricky problem: the following line must not put the tokenizer in
# 'multiline state'...
res = list(t.tokenize('"""foo"""'))
assert res == [Token('"""foo"""', type='string')]
res = list(t.tokenize('bar'))
assert res == [Token('bar', type='word')]
def test_string_multiline_slash(self):
t = Tokenizer(PythonSchema)
res = list(t.tokenize("'foo\\"))
assert res == [Token("'foo\\", type='string')]
res = list(t.tokenize("bar'"))
assert res == [Token("bar'", type='string')]
res = list(t.tokenize("bar"))
assert res == [Token('bar', type='word')]
res = list(t.tokenize('"foo\\bar"'))
assert res == [Token('"foo\\bar"', type="string")]
def test_string_following_printable(self):
assert self.tokens('."foo"') == [Token('.', type='unknown'),
Token('"foo"', type='string')]
def test_something_strange(self):
t = Tokenizer(PythonSchema)
tokens = list(t.tokenize('"""foo "bar" baz"""'))
assert not t._inside_multiline

190
py/apigen/source/testing/test_html.py
View File

@ -1,190 +0,0 @@
# -*- coding: UTF-8 -*-
""" test of html generation
"""
from py.__.apigen.source.html import prepare_line, create_html, HTMLDocument, \
get_module_encoding
from py.__.apigen.source.browser import parse_path
from py.__.apigen.source.color import Tokenizer, PythonSchema
from py.xml import html
import py
import os
def create_html_and_show(path):
mod = parse_path(path)
html = create_html(mod)
testfile = py.test.ensuretemp("htmloutput").ensure("test.html")
testfile.write(unicode(html))
return testfile
def test_basic():
tmp = py.test.ensuretemp("sourcehtml")
inp = tmp.ensure("one.py")
inp.write(py.code.Source("""
def func_one():
pass
def func_two(x, y):
x = 1
y = 2
return x + y
class B:
pass
class A(B):
def meth1(self):
pass
def meth2(self):
pass
"""))
testfile = create_html_and_show(inp)
data = testfile.open().read()
assert data.find('<a href="#func_one"') != -1
assert data.find('<a href="#func_two"') != -1
assert data.find('<a href="#B"') != -1
assert data.find('<a href="#A"') != -1
assert data.find('<a href="#A.meth1"') != -1
class _HTMLDocument(HTMLDocument):
def __init__(self):
self.encoding = 'ascii'
class TestHTMLDocument(object):
def test_head(self):
doc = _HTMLDocument()
head = doc.create_head()
assert isinstance(head, html.head)
rendered = unicode(head)
assert rendered.find('<title>source view</title>') > -1
assert py.std.re.search('<style type="text/css">[^<]+</style>',
rendered)
def test_body(self):
doc = _HTMLDocument()
body = doc.create_body()
assert unicode(body) == '<body></body>'
def test_table(self):
doc = _HTMLDocument()
table, tbody = doc.create_table()
assert isinstance(table, html.table)
assert isinstance(tbody, html.tbody)
assert tbody == table[0]
def test_add_row(self):
doc = HTMLDocument('ascii')
doc.add_row(1, ['""" this is a foo implementation """'])
doc.add_row(2, [''])
doc.add_row(3, ['class ', html.a('Foo', name='Foo'), ':'])
doc.add_row(4, [' pass'])
tbody = doc.tbody
assert len(tbody) == 4
assert unicode(tbody[0][0]) == '<td class="lineno">1</td>'
assert unicode(tbody[0][1]) == ('<td class="code">'
'<span class="string">'
'&quot;&quot;&quot; '
'this is a foo implementation '
'&quot;&quot;&quot;'
'</span></td>')
assert unicode(tbody[1][1]) == '<td class="code">&#xa0;</td>'
assert unicode(tbody[2][1]) == ('<td class="code">'
'<span class="alt_keyword">class'
'</span> '
'<a name="Foo">Foo</a>:</td>')
assert unicode(tbody[3][1]) == ('<td class="code"> '
'<span class="alt_keyword">pass'
'</span></td>')
def test_unicode(self):
doc = HTMLDocument('ascii')
h = unicode(doc)
print h
assert py.std.re.match(r'<html>\s*<head>\s*<title>[^<]+</title>'
'.*</body>\w*</html>$', h, py.std.re.S)
def prepare_line_helper(line, tokenizer=None, encoding='ascii'):
if tokenizer is None:
tokenizer = Tokenizer(PythonSchema)
l = prepare_line(line, tokenizer, encoding)
return ''.join([unicode(i) for i in l])
def test_prepare_line_basic():
result = prepare_line_helper(['see if this works'])
assert result == 'see <span class="keyword">if</span> this works'
result = prepare_line_helper(['see if this ',
html.a('works', name='works'),' too'])
assert result == ('see <span class="keyword">if</span> this '
'<a name="works">works</a> too')
result = prepare_line_helper(['see if something else works'])
assert result == ('see <span class="keyword">if</span> something '
'<span class="keyword">else</span> works')
result = prepare_line_helper(['see if something ',
html.a('else', name='else'), ' works too'])
assert result == ('see <span class="keyword">if</span> something '
'<a name="else">else</a> works too')
def test_prepare_line_strings():
result = prepare_line_helper(['foo = "bar"'])
assert result == 'foo = <span class="string">&quot;bar&quot;</span>'
result = prepare_line_helper(['"spam"'])
assert result == '<span class="string">&quot;spam&quot;</span>'
def test_prepare_line_multiline_strings():
# test multiline strings
t = Tokenizer(PythonSchema)
result = prepare_line_helper(['"""start of multiline'], t)
assert result == ('<span class="string">&quot;&quot;&quot;start of '
'multiline</span>')
result = prepare_line_helper(['see if it doesn\'t touch this'], t)
assert result == ('<span class="string">see if it doesn&apos;t touch '
'this</span>')
result = prepare_line_helper(['"""'], t)
assert result == '<span class="string">&quot;&quot;&quot;</span>'
result = prepare_line_helper(['see if it colours this again'], t)
assert result == ('see <span class="keyword">if</span> it colours '
'this again')
def test_prepare_line_nonascii():
result = prepare_line_helper(['"föö"'], encoding='UTF-8')
assert (result ==
unicode('<span class="string">&quot;föö&quot;</span>', 'UTF-8'))
def test_get_encoding_ascii():
temp = py.test.ensuretemp('test_get_encoding')
fpath = temp.join('ascii.py')
fpath.write(str(py.code.Source("""\
def foo():
return 'foo'
""")))
# XXX I think the specs say we have to assume latin-1 here...
assert get_module_encoding(fpath.strpath) == 'ISO-8859-1'
def test_get_encoding_for_real():
temp = py.test.ensuretemp('test_get_encoding')
fpath = temp.join('utf-8.py')
fpath.write(str(py.code.Source("""\
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
def foo():
return 'föö'
""")))
assert get_module_encoding(fpath.strpath) == 'UTF-8'
def test_get_encoding_matching_pattern_elsewhere():
temp = py.test.ensuretemp('test_get_encoding')
fpath = temp.join('matching_pattern.py')
fpath.write(str(py.code.Source("""\
#!/usr/bin/env python
def foo(coding=None):
pass
""")))
assert get_module_encoding(fpath.strpath) == 'ISO-8859-1'

137
py/apigen/style.css
View File

@ -1,137 +0,0 @@
#apigen-content {
font-size: 0.8em;
}
#logo {
position: relative;
position: fixed;
}
div.sidebar {
font-family: Verdana, Helvetica, Arial, sans-serif;
font-size: 0.9em;
width: 155px;
vertical-align: top;
position: absolute;
top: 130px;
left: 4px;
bottom: 4px;
overflow: auto;
}
/* trick to not make IE ignore something (>) */
body > .sidebar {
position: fixed;
}
div.sidebar a, div.sidebar a:visited, div.sidebar a:hover {
color: blue;
text-decoration: none;
}
div.sidebar .selected a, div.sidebar .selected a:visited,
div.sidebar .selected a:hover {
color: white;
background-color: #3ba6ec;
}
#content {
border: 0px;
height: 95%;
}
ul {
padding-left: 0em;
margin-top: 0px;
}
ul li {
list-style-type: none;
}
h2 {
padding-top: 0.5em;
}
h2.funcdef {
color: blue;
}
h2.funcdef:hover {
text-decoration: underline;
}
.codeblock {
margin-top: 0.5em;
margin-bottom: 0.5em;
}
.code a {
color: blue;
font-weight: bold;
text-decoration: none;
}
.lineno {
height: 1.4em;
text-align: right;
padding: 0px;
margin: 0px;
color: #555;
width: 3em;
padding-right: 1em;
border: 0px solid black;
border-right-width: 1px;
}
.codecell {
height: 1.4em;
padding: 0px;
margin: 0px;
padding-left: 1em;
}
pre.code {
line-height: 1.3em;
background-color: white;
margin: 0px;
padding: 0px;
border: 0px;
font-family: monospace, Monaco;
}
.comment {
color: purple;
}
.string {
color: #777;
}
.keyword {
color: blue;
}
.alt_keyword {
color: green;
}
.funcdocinfo {
border: 1px solid black;
color: black;
padding: 1em;
background-color: white;
}
.callstackitem {
border: 1px solid black;
margin-bottom: 1em;
}
td.lineno {
line-height: 1.1em;
}
td.code {
line-height: 1.1em;
}

0
py/apigen/testing/__init__.py
View File

475
py/apigen/testing/test_apigen_example.py
View File

@ -1,475 +0,0 @@
# -*- coding: UTF-8 -*-
import py
html = py.xml.html
from py.__.apigen.linker import TempLinker
from py.__.apigen.htmlgen import *
from py.__.apigen.tracer.docstorage import DocStorage, DocStorageAccessor
from py.__.apigen.tracer.tracer import Tracer
from py.__.apigen.layout import LayoutPage
from py.__.apigen.project import Project
from py.__.test.web import webcheck
from py.__.path.svn.testing.svntestbase import make_test_repo
def run_string_sequence_test(data, seq):
currpos = -1
for s in seq:
newpos = data.find(s)
if currpos >= newpos:
if newpos == -1:
message = 'not found'
else:
message = 'unexpected position: %s' % (newpos,)
py.test.fail('string %r: %s' % (s, message))
currpos = newpos
def setup_fs_project(temp):
temp.ensure("pkg/func.py").write(py.code.Source("""\
def func(arg1):
"docstring"
"""))
temp.ensure('pkg/someclass.py').write(py.code.Source("""\
class SomeClass(object):
" docstring someclass "
def __init__(self, somevar):
self.somevar = somevar
def get_somevar(self):
" get_somevar docstring "
return self.somevar
SomeInstance = SomeClass(10)
class SomeHiddenClass(object):
" docstring somehiddenclass "
__apigen_hide_from_nav__ = True # hide it from the navigation
"""))
temp.ensure('pkg/somesubclass.py').write(py.code.Source("""\
from someclass import SomeClass
class SomeSubClass(SomeClass):
" docstring somesubclass "
def get_somevar(self):
return self.somevar + 1
"""))
temp.ensure('pkg/somenamespace.py').write(py.code.Source("""\
def foo():
return 'bar'
def baz(qux):
return qux
"""))
temp.ensure("pkg/__init__.py").write(py.code.Source("""\
from py.initpkg import initpkg
initpkg(__name__, exportdefs = {
'main.sub.func': ("./func.py", "func"),
'main.SomeClass': ('./someclass.py', 'SomeClass'),
'main.SomeInstance': ('./someclass.py', 'SomeInstance'),
'main.SomeSubClass': ('./somesubclass.py', 'SomeSubClass'),
'main.SomeSubClass': ('./somesubclass.py', 'SomeSubClass'),
'main.SomeHiddenClass': ('./someclass.py', 'SomeHiddenClass'),
'other': ('./somenamespace.py', '*'),
'_test': ('./somenamespace.py', '*'),
})
"""))
return temp, 'pkg'
def get_dsa(fsroot, pkgname):
py.std.sys.path.insert(0, str(fsroot))
pkg = __import__(pkgname)
ds = DocStorage()
ds.from_pkg(pkg)
dsa = DocStorageAccessor(ds)
return ds, dsa
def _checkhtml(htmlstring):
if isinstance(htmlstring, unicode):
htmlstring = htmlstring.encode('UTF-8', 'replace')
assert isinstance(htmlstring, str)
if py.test.config.option.webcheck:
webcheck.check_html(htmlstring)
else:
py.test.skip("pass --webcheck to validate html produced in tests "
"(partial skip: the test has succeeded up until here)")
def _checkhtmlsnippet(htmlstring):
# XXX wrap page around snippet and validate
pass
#newstring = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
#"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n""" + unicode(h)
#_checkhtml(newstring)
class LayoutTestPage(LayoutPage):
def get_relpath(self):
return '../'
class AbstractBuilderTest(object):
def setup_class(cls):
temp = py.test.ensuretemp('apigen_example')
cls.fs_root, cls.pkg_name = setup_fs_project(temp)
cls.ds, cls.dsa = get_dsa(cls.fs_root, cls.pkg_name)
cls.project = Project()
def setup_method(self, meth):
self.base = base = py.test.ensuretemp('%s_%s' % (
self.__class__.__name__, meth.im_func.func_name))
self.linker = linker = TempLinker()
namespace_tree = create_namespace_tree(['main.sub',
'main.sub.func',
'main.SomeClass',
'main.SomeSubClass',
'main.SomeInstance',
'main.SomeHiddenClass',
'other.foo',
'other.baz',
'_test'])
self.namespace_tree = namespace_tree
self.apb = ApiPageBuilder(base, linker, self.dsa,
self.fs_root.join(self.pkg_name),
namespace_tree, self.project)
self.spb = SourcePageBuilder(base, linker,
self.fs_root.join(self.pkg_name),
self.project)
self.apb.pageclass = self.spb.pageclass = LayoutTestPage
class TestApiPageBuilder(AbstractBuilderTest):
def test_build_callable_view(self):
ds, dsa = get_dsa(self.fs_root, self.pkg_name)
t = Tracer(ds)
t.start_tracing()
pkg = __import__(self.pkg_name)
pkg.main.sub.func(10)
pkg.main.sub.func(pkg.main.SomeClass(10))
t.end_tracing()
apb = ApiPageBuilder(self.base, self.linker, dsa, self.fs_root,
self.namespace_tree, self.project)
snippet = apb.build_callable_view('main.sub.func')
html = snippet.unicode()
print html
# XXX somewhat grokky tests because the order of the items may change
assert 'arg1: AnyOf(' in html
pos1 = html.find('arg1: AnyOf(')
assert pos1 > -1
pos2 = html.find('href="', pos1)
assert pos2 > pos1
pos3 = html.find('Class SomeClass', pos2)
assert pos3 > pos2
pos4 = html.find('Int&gt;', pos1)
assert pos4 > pos1
pos5 = html.find('return value:', pos4)
assert pos5 > pos4 and pos5 > pos3
pos6 = html.find('&lt;None&gt;', pos5)
assert pos6 > pos5
sourcefile = self.fs_root.join('pkg/func.py')
pos7 = html.find('source: %s' % (get_rel_sourcepath(apb.projpath,
sourcefile),),
pos6)
assert pos7 > pos6
_checkhtmlsnippet(html)
def test_build_function_pages(self):
self.apb.build_function_pages(['main.sub.func'])
funcfile = self.base.join('api/main.sub.func.html')
assert funcfile.check()
html = funcfile.read()
_checkhtml(html)
def test_build_class_view(self):
snippet = self.apb.build_class_view('main.SomeClass')
html = snippet.unicode()
_checkhtmlsnippet(html)
def test_build_class_pages(self):
self.apb.build_class_pages(['main.SomeClass', 'main.SomeSubClass'])
clsfile = self.base.join('api/main.SomeClass.html')
assert clsfile.check()
html = clsfile.read()
_checkhtml(html)
def test_build_class_pages_instance(self):
self.apb.build_class_pages(['main.SomeClass',
'main.SomeSubClass',
'main.SomeInstance'])
clsfile = self.base.join('api/main.SomeInstance.html')
assert clsfile.check()
html = clsfile.read()
print html
run_string_sequence_test(html, [
'instance of SomeClass()',
])
def test_build_class_pages_nav_links(self):
self.apb.build_class_pages(['main.SomeSubClass',
'main.SomeClass'])
self.apb.build_namespace_pages()
self.linker.replace_dirpath(self.base, False)
clsfile = self.base.join('api/main.SomeClass.html')
assert clsfile.check()
html = clsfile.read()
run_string_sequence_test(html, [
'href="../style.css"',
'href="../apigen_style.css"',
'src="../api.js"',
'href="index.html">pkg',
'href="main.html">main',
'href="main.SomeClass.html">SomeClass',
'href="main.SomeSubClass.html">SomeSubClass',
])
assert 'href="main.sub.func.html"' not in html
assert 'href="_test' not in html
assert 'href="main.sub.html">sub' in html
_checkhtml(html)
def test_build_class_pages_base_link(self):
self.apb.build_class_pages(['main.SomeSubClass',
'main.SomeClass'])
self.linker.replace_dirpath(self.base, False)
clsfile = self.base.join('api/main.SomeSubClass.html')
assert clsfile.check()
html = clsfile.read()
print html
run_string_sequence_test(html, [
'href="../style.css"',
'href="main.SomeClass.html">main.SomeClass',
])
_checkhtml(html)
def test_source_links(self):
self.apb.build_class_pages(['main.SomeSubClass', 'main.SomeClass'])
self.spb.build_pages(self.fs_root)
self.linker.replace_dirpath(self.base, False)
funchtml = self.base.join('api/main.SomeClass.html').read()
print funchtml
#assert funchtml.find('href="../source/pkg/someclass.py.html#SomeClass"') > -1
assert funchtml.find('href="../source/pkg/someclass.py.html"') > -1
_checkhtml(funchtml)
def test_build_namespace_pages(self):
self.apb.build_namespace_pages()
mainfile = self.base.join('api/main.html')
assert mainfile.check()
html = mainfile.read()
print html
run_string_sequence_test(html, [
'index of main',
])
otherfile = self.base.join('api/other.html')
assert otherfile.check()
otherhtml = otherfile.read()
print otherhtml
run_string_sequence_test(otherhtml, [
'index of other',
])
_checkhtml(html)
_checkhtml(otherhtml)
def test_build_namespace_pages_index(self):
self.apb.build_namespace_pages()
pkgfile = self.base.join('api/index.html')
assert pkgfile.check()
html = pkgfile.read()
assert 'index of pkg' in html
_checkhtml(html)
def test_build_namespace_pages_subnamespace(self):
self.apb.build_namespace_pages()
subfile = self.base.join('api/main.sub.html')
assert subfile.check()
html = subfile.read()
_checkhtml(html)
def test_build_function_api_pages_nav(self):
self.linker.set_link('main.sub', 'api/main.sub.html')
self.linker.set_link('', 'api/index.html')
self.linker.set_link('main', 'api/main.html')
self.apb.build_function_pages(['main.sub.func'])
self.linker.replace_dirpath(self.base, False)
funcfile = self.base.join('api/main.sub.func.html')
html = funcfile.read()
print html
run_string_sequence_test(html, [
'<a href="index.html">',
'<a href="main.html">',
'<a href="main.sub.html">',
'<a href="main.sub.func.html">',
])
_checkhtml(html)
def test_build_function_navigation(self):
self.apb.build_namespace_pages()
self.apb.build_function_pages(['main.sub.func'])
self.apb.build_class_pages(['main.SomeClass',
'main.SomeSubClass',
'main.SomeInstance',
'main.SomeHiddenClass'])
self.linker.replace_dirpath(self.base, False)
html = self.base.join('api/main.sub.func.html').read()
print html
# XXX NOTE: do not mess with the string below, the spaces between the
# <div> and <a> are actually UTF-8 \xa0 characters (non-breaking
# spaces)!
assert """\
<div class="sidebar">
<div class="selected"><a href="index.html">pkg</a></div>
<div class="selected">  <a href="main.html">main</a></div>
<div>    <a href="main.SomeClass.html">SomeClass</a></div>
<div>    <a href="main.SomeInstance.html">SomeInstance</a></div>
<div>    <a href="main.SomeSubClass.html">SomeSubClass</a></div>
<div class="selected">    <a href="main.sub.html">sub</a></div>
<div class="selected">      <a href="main.sub.func.html">func</a></div>
<div>  <a href="other.html">other</a></div></div>
""" in html
def test_build_root_namespace_view(self):
self.apb.build_namespace_pages()
self.linker.replace_dirpath(self.base, False)
rootfile = self.base.join('api/index.html')
assert rootfile.check()
html = rootfile.read()
assert '<a href="main.html">' in html
_checkhtml(html)
def test_get_revision(self):
py.test.skip('XXX changed implementation (temporarily?)')
if py.std.sys.platform.startswith('win'):
py.test.skip('broken on win32 for some reason (svn caching?), '
'skipping')
# XXX a lot of setup required for this one... more like a functional
# test I fear
# create test repo and checkout
repo = make_test_repo('test_get_revision_api_repo')
wc = py.path.svnwc(py.test.ensuretemp('test_get_revision_api_wc'))
wc.checkout(repo.url)
assert wc.status().rev == '0'
# create a temp package inside the working copy
fs_root, pkg_name = setup_fs_project(wc)
ds, dsa = get_dsa(self.fs_root, self.pkg_name)
wc.commit('test get revision commit')
wc.update()
# clear cache
py.__.apigen.htmlgen._get_obj_cache = {}
# fiddle about a bit with paths so that our package is picked up :|
old_path = py.std.sys.path
try:
py.std.sys.path.insert(0, fs_root.strpath)
pkgkeys = [k for k in py.std.sys.modules.keys() if
k == 'pkg' or k.startswith('pkg.')]
# remove modules from sys.modules
for key in pkgkeys:
del py.std.sys.modules[key]
# now create a new apb that uses the wc pkg
apb = ApiPageBuilder(self.base, self.linker, dsa,
fs_root.join(pkg_name),
self.namespace_tree, self.project)
apb._revcache = {} # clear cache, this is on class level!!
pkg = wc.join('pkg')
assert pkg.check(versioned=True)
assert pkg.info().created_rev == 1
funcpath = pkg.join('func.py')
classpath = pkg.join('someclass.py')
assert funcpath.check(versioned=True)
assert classpath.check(versioned=True)
assert apb.get_revision('main.sub.func') == 1
assert apb.get_revision('main.SomeClass') == 1
assert apb.get_revision('') == 1
assert apb.get_revision('main.sub') == 1
funcpath.write(funcpath.read() + '\n')
funcpath.commit('updated func')
wc.update()
apb._revcache = {} # clear cache
assert apb.get_revision('main.sub.func') == 2
assert apb.get_revision('') == 1
assert apb.get_revision('main.SomeClass') == 1
finally:
py.std.sys.path = old_path
# clear caches again
py.__.apigen.htmlgen._get_obj_cache = {}
apb._revcache = {}
class TestSourcePageBuilder(AbstractBuilderTest):
def test_build_pages(self):
self.spb.build_pages(self.fs_root)
somesource = self.base.join('source/pkg/func.py.html').read()
_checkhtml(somesource)
def test_build_pages_nav(self):
self.spb.build_pages(self.fs_root)
self.linker.replace_dirpath(self.base, False)
funcsource = self.base.join('source/pkg/func.py.html')
assert funcsource.check(file=True)
html = funcsource.read()
print html
run_string_sequence_test(html, [
'href="../style.css"',
'<a href="index.html">pkg</a>',
'<a href="someclass.py.html">someclass.py</a>',
'<a href="somesubclass.py.html">somesubclass.py</a>',
])
def test_build_dir_page(self):
self.spb.build_pages(self.fs_root)
self.linker.replace_dirpath(self.base, False)
pkgindex = self.base.join('source/pkg/index.html')
assert pkgindex.check(file=True)
html = pkgindex.read()
print html
run_string_sequence_test(html, [
'href="../style.css"',
'<a href="index.html">pkg</a>',
'<a href="func.py.html">func.py</a>',
'<a href="someclass.py.html">someclass.py</a>',
'<a href="somesubclass.py.html">somesubclass.py</a>',
'<h2>directories</h2>',
'<h2>files</h2>'])
_checkhtml(html)
def test_build_source_page(self):
self.spb.build_pages(self.fs_root)
self.linker.replace_dirpath(self.base, False)
funcsource = self.base.join('source/pkg/func.py.html')
assert funcsource.check(file=True)
html = funcsource.read()
print html
assert ('<span class="alt_keyword">def</span> '
'<a href="#func" name="func">func</a>(arg1)') in html
def test_build_navigation_root(self):
self.spb.build_pages(self.fs_root)
self.linker.replace_dirpath(self.base)
html = self.base.join('source/pkg/index.html').read()
print html
run_string_sequence_test(html, [
'href="index.html">pkg',
'href="func.py.html">func.py',
'href="someclass.py.html">someclass.py',
'href="somesubclass.py.html">somesubclass.py',
])
def test_get_revision(self):
py.test.skip('XXX changed implementation (temporarily?)')
if py.std.sys.platform.startswith('win'):
py.test.skip('broken on win32 for some reason (svn caching?), '
'skipping')
repo = make_test_repo('test_get_revision_source_repo')
wc = py.path.svnwc(py.test.ensuretemp('test_get_revision_source_wc'))
wc.checkout(repo.url)
dir = wc.ensure('dir', dir=True)
file = dir.ensure('file.py', file=True)
wc.commit('added dir and file')
wc.update()
assert file.check(versioned=True)
assert wc.status().rev == '1'
assert self.spb.get_revision(dir) == 1
assert self.spb.get_revision(file) == 1
file.write('while 1:\n print "py lib is cool\n"')
file.commit('added some code')
assert file.status().rev == '2'
self.spb._revcache = {}
assert self.spb.get_revision(file) == 2
assert self.spb.get_revision(dir) == 1

179
py/apigen/testing/test_apigen_functional.py
View File

@ -1,179 +0,0 @@
""" functional test for apigen.py
script to build api + source docs from py.test
"""
import py
from py.__.apigen import apigen
py.test.skip("Apigen functionality temporarily disabled")
def setup_module(mod):
if py.std.sys.platform == "win32":
py.test.skip("not supported with win32 yet")
def setup_fs_project(name):
temp = py.test.ensuretemp(name)
assert temp.listdir() == []
temp.ensure("pak/func.py").write(py.code.Source("""\
def func(arg1):
"docstring"
def func_2(arg1, arg2):
return arg1(arg2)
"""))
temp.ensure('pak/sometestclass.py').write(py.code.Source("""\
class SomeTestClass(object):
" docstring sometestclass "
someattr = 'somevalue'
def __init__(self, somevar):
self.somevar = somevar
def get_somevar(self):
" get_somevar docstring "
return self.somevar
def get_some_source(self):
ret = py.code.Source('''\\
def foo():
return 'bar'
''')
return ret
"""))
temp.ensure('pak/sometestsubclass.py').write(py.code.Source("""\
from sometestclass import SomeTestClass
class SomeTestSubClass(SomeTestClass):
" docstring sometestsubclass "
def get_somevar(self):
return self.somevar + 1
"""))
temp.ensure('pak/somenamespace.py').write(py.code.Source("""\
def foo():
return 'bar'
def baz(qux):
return qux
def _hidden():
return 'quux'
"""))
temp.ensure("pak/__init__.py").write(py.code.Source("""\
'''pkg docstring'''
from py.initpkg import initpkg
initpkg(__name__,
long_description=globals()['__doc__'],
exportdefs={'main.sub.func': ("./func.py", "func"),
'main.func': ("./func.py", "func_2"),
'main.SomeTestClass': ('./sometestclass.py',
'SomeTestClass'),
'main.SomeTestSubClass': ('./sometestsubclass.py',
'SomeTestSubClass'),
'somenamespace': ('./somenamespace.py', '*')})
"""))
temp.ensure('apigen.py').write(py.code.Source("""\
import py
py.std.sys.path.insert(0,
py.magic.autopath().dirpath().dirpath().dirpath().strpath)
from py.__.apigen.apigen import build, \
get_documentable_items_pkgdir as get_documentable_items
"""))
temp.ensure('pak/test/test_pak.py').write(py.code.Source("""\
import py
py.std.sys.path.insert(0,
py.magic.autopath().dirpath().dirpath().dirpath().strpath)
import pak
# this mainly exists to provide some data to the tracer
def test_pak():
s = pak.main.SomeTestClass(10)
assert s.get_somevar() == 10
s = pak.main.SomeTestClass('10')
assert s.get_somevar() == '10'
s = pak.main.SomeTestSubClass(10)
assert s.get_somevar() == 11
s = pak.main.SomeTestSubClass('10')
py.test.raises(TypeError, 's.get_somevar()')
assert pak.main.sub.func(10) is None
assert pak.main.sub.func(20) is None
s = pak.main.func(pak.main.SomeTestClass, 10)
assert isinstance(s, pak.main.SomeTestClass)
# some nice things to confuse the tracer/storage
source = py.code.Source('''\
pak.main.sub.func(10)
''')
c = compile(str(source), '<test>', 'exec')
exec c in globals()
assert pak.somenamespace._hidden() == 'quux'
# this just to see a multi-level stack in the docs
def foo():
return pak.main.sub.func(10)
assert foo() is None
"""))
return temp, 'pak'
def test_get_documentable_items():
fs_root, package_name = setup_fs_project('test_get_documentable_items')
pkgname, documentable = apigen.get_documentable_items_pkgdir(
fs_root.join(package_name))
assert pkgname == 'pak'
keys = documentable.keys()
keys.sort()
assert keys == [
'main.SomeTestClass', 'main.SomeTestSubClass', 'main.func',
'main.sub.func', 'somenamespace.baz', 'somenamespace.foo']
def test_apigen_functional():
#if py.std.sys.platform == "win32":
# py.test.skip("XXX test fails on windows")
fs_root, package_name = setup_fs_project('test_apigen_functional')
tempdir = py.test.ensuretemp('test_apigen_functional_results')
pydir = py.magic.autopath().dirpath().dirpath().dirpath()
pakdir = fs_root.join('pak')
if py.std.sys.platform == 'win32':
cmd = ('set APIGENPATH=%s && set PYTHONPATH=%s && '
'python "%s/bin/py.test"') % (tempdir, fs_root, pydir)
else:
cmd = ('APIGENPATH="%s" PYTHONPATH="%s" '
'python "%s/bin/py.test"') % (tempdir, fs_root, pydir)
try:
output = py.process.cmdexec('%s --apigen="%s/apigen.py" "%s"' % (
cmd, fs_root, pakdir))
except py.error.Error, e:
print e.out
raise
assert output.lower().find('traceback') == -1
# just some quick content checks
apidir = tempdir.join('api')
assert apidir.check(dir=True)
sometestclass_api = apidir.join('main.SomeTestClass.html')
assert sometestclass_api.check(file=True)
html = sometestclass_api.read()
print html
assert '<a href="main.SomeTestClass.html">SomeTestClass</a>' in html
assert 'someattr: <em>somevalue</em>' in html
namespace_api = apidir.join('main.html')
assert namespace_api.check(file=True)
html = namespace_api.read()
assert '<a href="main.SomeTestClass.html">SomeTestClass</a>' in html
index = apidir.join('index.html')
assert index.check(file=True)
html = index.read()
assert 'pkg docstring' in html
sourcedir = tempdir.join('source')
assert sourcedir.check(dir=True)
sometestclass_source = sourcedir.join('sometestclass.py.html')
assert sometestclass_source.check(file=True)
html = sometestclass_source.read()
assert '<div class="project_title">sources for sometestclass.py</div>' in html
index = sourcedir.join('index.html')
assert index.check(file=True)
html = index.read()
print html
assert '<a href="test/index.html">test</a>' in html
assert 'href="../../py/doc/home.html"'

194
py/apigen/testing/test_htmlgen.py
View File

@ -1,194 +0,0 @@
import py
from py.__.apigen import htmlgen
from py.__.apigen.linker import Linker
def assert_eq_string(string1, string2):
if string1 == string2:
return
__tracebackhide__ = True
for i, (c1, c2) in py.builtin.enumerate(zip(string1, string2)):
if c1 != c2:
start = max(0, i-20)
end = i + 20
py.test.fail("strings not equal in position i=%d\n"
"string1[%d:%d] = %r\n"
"string2[%d:%d] = %r\n"
"string1 = %r\n"
"string2 = %r\n"
% (i,
start, end, string1[start:end],
start, end, string2[start:end],
string1, string2
))
def test_create_namespace_tree():
tree = htmlgen.create_namespace_tree(['foo.bar.baz'])
assert tree == {'': ['foo'],
'foo': ['foo.bar'],
'foo.bar': ['foo.bar.baz']}
tree = htmlgen.create_namespace_tree(['foo.bar.baz', 'foo.bar.qux'])
assert tree == {'': ['foo'],
'foo': ['foo.bar'],
'foo.bar': ['foo.bar.baz', 'foo.bar.qux']}
tree = htmlgen.create_namespace_tree(['pkg.sub.func',
'pkg.SomeClass',
'pkg.SomeSubClass'])
assert tree == {'': ['pkg'],
'pkg.sub': ['pkg.sub.func'],
'pkg': ['pkg.sub', 'pkg.SomeClass',
'pkg.SomeSubClass']}
def test_source_dirs_files():
temp = py.test.ensuretemp('test_source_dirs_files')
temp.join('dir').ensure(dir=True)
temp.join('dir/file1.py').ensure(file=True)
temp.join('dir/file2.pyc').ensure(file=True)
temp.join('dir/file3.c').ensure(file=True)
temp.join('dir/.hidden_file').ensure(file=True)
temp.join('dir/sub').ensure(dir=True)
temp.join('dir/.hidden_dir').ensure(dir=True)
dirs, files = htmlgen.source_dirs_files(temp.join('dir'))
dirnames = py.builtin.sorted([d.basename for d in dirs])
filenames = py.builtin.sorted([f.basename for f in files])
assert dirnames == ['sub']
assert filenames == ['file1.py', 'file3.c']
def test_deindent():
assert htmlgen.deindent('foo\n\n bar\n ') == 'foo\n\nbar\n'
assert htmlgen.deindent(' foo\n\n bar\n ') == 'foo\n\nbar\n'
assert htmlgen.deindent('foo\n\n bar\n baz') == 'foo\n\nbar\nbaz\n'
assert htmlgen.deindent(' foo\n\n bar\n baz\n') == (
'foo\n\nbar\n baz\n')
assert htmlgen.deindent('foo\n\n bar\n baz\n') == (
'foo\n\n bar\nbaz\n')
def test_enumerate_and_color():
colored = htmlgen.enumerate_and_color(['def foo():', ' print "bar"'], 0,
'ascii')
div = py.xml.html.div(colored).unicode(indent=0)
print repr(div)
assert_eq_string(div,
u'<div>'
'<table class="codeblock">'
'<tbody>'
'<tr>'
'<td style="width: 1%">'
'<table>'
'<tbody>'
'<tr><td class="lineno">1</td></tr>'
'<tr><td class="lineno">2</td></tr>'
'</tbody>'
'</table>'
'</td>'
'<td>'
'<table>'
'<tbody>'
'<tr><td class="codecell">'
'<pre class="code">'
'<span class="alt_keyword">def</span> foo():'
'</pre>'
'</td></tr>'
'<tr><td class="codecell">'
'<pre class="code">'
' <span class="alt_keyword">print</span>'
' <span class="string">&quot;bar&quot;</span>'
'</pre>'
'</td></tr>'
'</tbody>'
'</table>'
'</td>'
'</tr>'
'</tbody>'
'</table>'
'</div>')
def test_enumerate_and_color_multiline():
colored = htmlgen.enumerate_and_color(['code = """\\', 'foo bar', '"""'],
0, 'ascii')
div = py.xml.html.div(colored).unicode(indent=0)
print repr(div)
assert_eq_string (div,
u'<div>'
'<table class="codeblock">'
'<tbody>'
'<tr>'
'<td style="width: 1%">'
'<table>'
'<tbody>'
'<tr><td class="lineno">1</td></tr>'
'<tr><td class="lineno">2</td></tr>'
'<tr><td class="lineno">3</td></tr>'
'</tbody>'
'</table>'
'</td>'
'<td>'
'<table>'
'<tbody>'
'<tr><td class="codecell">'
'<pre class="code">'
'code = <span class="string">&quot;&quot;&quot;\\</span>'
'</pre>'
'</td></tr>'
'<tr><td class="codecell">'
'<pre class="code">'
'<span class="string">foo bar</span>'
'</pre>'
'</td></tr>'
'<tr><td class="codecell">'
'<pre class="code">'
'<span class="string">&quot;&quot;&quot;</span>'
'</pre>'
'</td></tr>'
'</tbody>'
'</table>'
'</td>'
'</tr>'
'</tbody>'
'</table>'
'</div>')
def test_show_property():
assert htmlgen.show_property('foo')
assert not htmlgen.show_property('_foo')
assert htmlgen.show_property('__foo__')
assert not htmlgen.show_property('__doc__')
assert not htmlgen.show_property('__dict__')
assert not htmlgen.show_property('__name__')
assert not htmlgen.show_property('__class__')
def test_get_rel_sourcepath():
projpath = py.path.local('/proj')
assert (htmlgen.get_rel_sourcepath(projpath, py.path.local('/proj/foo')) ==
'foo')
assert (htmlgen.get_rel_sourcepath(projpath, py.path.local('/foo')) is
None)
assert (htmlgen.get_rel_sourcepath(projpath, py.path.local('<string>')) is
None)
def test_find_method_origin():
class Foo(object):
def foo(self):
pass
class Bar(Foo):
def bar(self):
pass
class Baz(Bar):
pass
assert htmlgen.find_method_origin(Baz.bar) is Bar
assert htmlgen.find_method_origin(Baz.foo) is Foo
assert htmlgen.find_method_origin(Bar.bar) is Bar
assert htmlgen.find_method_origin(Baz.__init__) is None
def test_find_method_origin_old_style():
class Foo:
def foo(self):
pass
class Bar(Foo):
def bar(self):
pass
class Baz(Bar):
pass
assert htmlgen.find_method_origin(Baz.bar) is Bar
assert htmlgen.find_method_origin(Baz.foo) is Foo
assert htmlgen.find_method_origin(Bar.bar) is Bar

68
py/apigen/testing/test_linker.py
View File

@ -1,68 +0,0 @@
import py
from py.__.apigen.linker import Linker, TempLinker, getrelfspath, relpath
class TestLinker(object):
def test_get_target(self):
linker = Linker()
lazyhref = linker.get_lazyhref('py.path.local')
linker.set_link('py.path.local', 'py/path/local.html')
relpath = linker.get_target('py.path.local')
assert relpath == 'py/path/local.html'
def test_target_relative(self):
linker = Linker()
lazyhref = linker.get_lazyhref('py.path.local')
linker.set_link('py.path.local', 'py/path/local.html')
relpath = linker.call_withbase('py/index.html',
linker.get_target, 'py.path.local')
assert relpath == 'path/local.html'
testspec = [
'a a/b a/b /',
'/a /a/b a/b /',
'a b b /',
'/a /b b /',
'a/b c/d ../c/d /',
'/a/b /c/d ../c/d /',
'a/b a ../a /',
'/a/b /a ../a /',
'c:\\foo\\bar c:\\foo ../foo \\',
]
class TestTempLinker(object):
def test_get_target(self):
linker = TempLinker()
temphref = linker.get_lazyhref('py.path.local')
linker.set_link('py.path.local', 'py/path/local.html')
relpath = linker.get_target(temphref)
assert relpath == 'py/path/local.html'
def test_functional(self):
temp = py.test.ensuretemp('TestTempLinker.test_functional')
l = TempLinker()
bar = temp.ensure('foo/bar.html', file=True)
baz = temp.ensure('foo/baz.html', file=True)
l.set_link(baz.strpath, baz.relto(temp))
bar.write('<a href="%s">baz</a>' % (l.get_lazyhref(baz.strpath),))
l.replace_dirpath(temp)
assert bar.read() == '<a href="baz.html">baz</a>'
def test_with_anchor(self):
py.test.skip("get_lazyhref needs fixing?")
linker = TempLinker()
temphref = linker.get_lazyhref('py.path.local', 'LocalPath.join')
linker.set_link('py.path.local', 'py/path/local.html')
relpath = linker.get_target(temphref)
assert relpath == 'py/path/local.html#LocalPath.join'
def gen_check(frompath, topath, sep, expected):
result = relpath(frompath, topath, sep=sep)
assert result == expected
def test_gen_check():
for line in testspec:
frompath, topath, expected, sep = line.split()
yield gen_check, frompath, topath, sep, expected
def test_check_incompatible():
py.test.raises(ValueError, "relpath('/a', 'b')")

16
py/apigen/todo-apigen.txt
View File

@ -1,16 +0,0 @@
* use "inherited" doc strings, i.e. for
class A:
def meth(self):
"doc1"
class B(A):
def meth(self):
pass
B.meth should display the A.meth docstring, probably
with special formatting (italics or so).
NOT YET DONE (later?)
* add warning about py.test possibly not covering the whole API

61
py/apigen/todo.txt
View File

@ -1,61 +0,0 @@
* source page headers should read:
py/apigen sources [rev XXX]
DONE (XXX: i don't see it as done, are you sure?
* and "namespace" pages:
builtin namespace index [rev XXX]
DONE, except they're now called 'index of <dotted_name> [rev. XXX]'
(XXX: strange, i also don't see this, am i doing something wrong?)
* get konqueror to display indents in source code better?
(currently it doesn't look like more than a single space)
Hrmph, fonts look just fine to me :( what machine is this (new laptop btw?)?
you seem to have a font problem... If you really want me to fix it for your
machine, please give me access...
I also made sure IE looks (somewhat) good...
* function view:
def __init__(self, rawcode):
docstring-in-grey-and-not-in-a-box
and the "show/hide funcinfo" link could be underyling
the full "def __init__(self, rawcode)" or be a link right after
(or maybe before) it.
goal: consume less vertical space and have the functions
be "sticking" out (the show/hide info link IMO disrupts this
and it's not visually clear it belongs to the function above it)
DONE, but please review if you like it like this...
XXX it's nice but can you keep the docstring visible when
more information is displayed/toggled?
DONE too
* linking from docs to apigen and back:
XXX holger thinks that apigen needs a doc_relpath
(symettric to py/doc/conftest needing a apigen_relpath)
if you can't find a way to provide this as a command line param,
then we probably need to hardcode it.
note that both relpath's are related to how we map docs and
apigen into the URL namespace.
Currently handled by using an env var (APIGEN_DOCRELPATH), since
to make it possible to run py.test --apigen on the full py lib _and_
set the option, it would have to be global (yuck), and apigen used
an env var already anyway... Of course it can easily be changed to an
option if you like that better...
There's now also a script bin/_docgen.py that runs all the tests
and builds the py lib docs + api ones in one go.

0
py/apigen/tracer/__init__.py
View File

365
py/apigen/tracer/description.py
View File

@ -1,365 +0,0 @@
import py
from py.__.apigen.tracer import model
from py.__.code.source import getsource
import types
import inspect
import copy
MAX_CALL_SITES = 20
set = py.builtin.set
def is_private(name):
return name.startswith('_') and not name.startswith('__')
class CallFrame(object):
def __init__(self, frame):
self.filename = frame.code.raw.co_filename
self.lineno = frame.lineno
self.firstlineno = frame.code.firstlineno
try:
self.source = getsource(frame.code.raw)
except IOError:
self.source = "could not get to source"
def _getval(self):
return (self.filename, self.lineno)
def __hash__(self):
return hash(self._getval())
def __eq__(self, other):
return self._getval() == other._getval()
def __ne__(self, other):
return not self == other
class CallStack(object):
def __init__(self, tb):
#if isinstance(tb, py.code.Traceback):
# self.tb = tb
#else:
# self.tb = py.code.Traceback(tb)
self.tb = [CallFrame(frame) for frame in tb]
#def _getval(self):
# return [(frame.code.raw.co_filename, frame.lineno+1) for frame
# in self]
def __hash__(self):
return hash(tuple(self.tb))
def __eq__(self, other):
return self.tb == other.tb
def __ne__(self, other):
return not self == other
#def __getattr__(self, attr):
# return getattr(self.tb, attr)
def __iter__(self):
return iter(self.tb)
def __getitem__(self, item):
return self.tb[item]
def __len__(self):
return len(self.tb)
def __cmp__(self, other):
return cmp(self.tb, other.tb)
def cut_stack(stack, frame, upward_frame=None):
if hasattr(frame, 'raw'):
frame = frame.raw
if upward_frame:
if hasattr(upward_frame, 'raw'):
upward_frame = upward_frame.raw
lst = [py.code.Frame(i) for i in stack[stack.index(frame):\
stack.index(upward_frame)+1]]
if len(lst) > 1:
return CallStack(lst[:-1])
return CallStack(lst)
return CallStack([py.code.Frame(i) for i in stack[stack.index(frame):]])
##class CallSite(object):
## def __init__(self, filename, lineno):
## self.filename = filename
## self.lineno = lineno
##
## def get_tuple(self):
## return self.filename, self.lineno
##
## def __hash__(self):
## return hash((self.filename, self.lineno))
##
## def __eq__(self, other):
## return (self.filename, self.lineno) == (other.filename, other.lineno)
##
## def __ne__(self, other):
## return not self == other
##
## def __cmp__(self, other):
## if self.filename < other.filename:
## return -1
## if self.filename > other.filename:
## return 1
## if self.lineno < other.lineno:
## return -1
## if self.lineno > other.lineno:
## return 1
## return 0
class NonHashableObject(object):
def __init__(self, cls):
self.cls = cls
def __hash__(self):
raise NotImplementedError("Object of type %s are unhashable" % self.cls)
class Desc(object):
def __init__(self, name, pyobj, **kwargs):
self.pyobj = pyobj
self.is_degenerated = False
self.name = name
if type(self) is Desc:
# do not override property...
self.code = NonHashableObject(self.__class__) # dummy think that makes code unhashable
# we make new base class instead of using pypy's one because
# of type restrictions of pypy descs
def __hash__(self):
return hash(self.code)
def __eq__(self, other):
if isinstance(other, Desc):
return self.code == other.code
if isinstance(other, types.CodeType):
return self.code == other
if isinstance(other, tuple) and len(other) == 2:
return self.code == other
return False
def __ne__(self, other):
return not self == other
# This set of functions will not work on Desc, because we need to
# define code somehow
class FunctionDesc(Desc):
def __init__(self, *args, **kwargs):
super(FunctionDesc, self).__init__(*args, **kwargs)
self.inputcells = [model.s_ImpossibleValue for i in xrange(self.\
code.co_argcount)]
self.call_sites = {}
self.keep_frames = kwargs.get('keep_frames', False)
self.frame_copier = kwargs.get('frame_copier', lambda x:x)
self.retval = model.s_ImpossibleValue
self.exceptions = {}
def consider_call(self, inputcells):
for cell_num, cell in enumerate(inputcells):
self.inputcells[cell_num] = model.unionof(cell, self.inputcells[cell_num])
def consider_call_site(self, frame, cut_frame):
if len(self.call_sites) > MAX_CALL_SITES:
return
stack = [i[0] for i in inspect.stack()]
cs = cut_stack(stack, frame, cut_frame)
self.call_sites[cs] = cs
def consider_exception(self, exc, value):
self.exceptions[exc] = True
def get_call_sites(self):
# convinient accessor for various data which we keep there
if not self.keep_frames:
return [(key, val) for key, val in self.call_sites.iteritems()]
else:
lst = []
for key, val in self.call_sites.iteritems():
for frame in val:
lst.append((key, frame))
return lst
def consider_return(self, arg):
self.retval = model.unionof(arg, self.retval)
def consider_start_locals(self, frame):
pass
def consider_end_locals(self, frame):
pass
def getcode(self):
return self.pyobj.func_code
code = property(getcode)
def get_local_changes(self):
return {}
class ClassDesc(Desc):
def __init__(self, *args, **kwargs):
super(ClassDesc, self).__init__(*args, **kwargs)
self.fields = {}
# we'll gather informations about methods and possibly
# other variables encountered here
def getcode(self):
# This is a hack. We're trying to return as much close to __init__
# of us as possible, but still hashable object
if hasattr(self.pyobj, '__init__'):
if hasattr(self.pyobj.__init__, 'im_func') and \
hasattr(self.pyobj.__init__.im_func, 'func_code'):
result = self.pyobj.__init__.im_func.func_code
else:
result = self.pyobj.__init__
else:
result = self.pyobj
try:
hash(result)
except KeyboardInterrupt, SystemExit:
raise
except: # XXX UUuuuu bare except here. What can it really rise???
try:
hash(self.pyobj)
result = self.pyobj
except:
result = self
return result
code = property(getcode)
def consider_call(self, inputcells):
if '__init__' in self.fields:
md = self.fields['__init__']
else:
md = MethodDesc(self.name + '.__init__', self.pyobj.__init__)
self.fields['__init__'] = md
md.consider_call(inputcells)
def consider_return(self, arg):
pass # we *know* what return value we do have
def consider_exception(self, exc, value):
if '__init__' in self.fields:
md = self.fields['__init__']
else:
md = MethodDesc(self.name + '.__init__', self.pyobj.__init__)
self.fields['__init__'] = md
md.consider_exception(exc, value)
def consider_start_locals(self, frame):
if '__init__' in self.fields:
md = self.fields['__init__']
md.consider_start_locals(frame)
def consider_end_locals(self, frame):
if '__init__' in self.fields:
md = self.fields['__init__']
md.consider_end_locals(frame)
def consider_call_site(self, frame, cut_frame):
self.fields['__init__'].consider_call_site(frame, cut_frame)
def add_method_desc(self, name, methoddesc):
self.fields[name] = methoddesc
def getfields(self):
# return fields of values that has been used
l = [i for i, v in self.fields.iteritems() if not is_private(i)]
return l
def getbases(self):
bases = []
tovisit = [self.pyobj]
while tovisit:
current = tovisit.pop()
if current is not self.pyobj:
bases.append(current)
tovisit += [b for b in current.__bases__ if b not in bases]
return bases
bases = property(getbases)
## def has_code(self, code):
## # check __init__ method
## return self.pyobj.__init__.im_func.func_code is code
##
## def consider_call(self, inputcells):
## # special thing, make MethodDesc for __init__
##
##
class MethodDesc(FunctionDesc):
def __init__(self, *args, **kwargs):
super(MethodDesc, self).__init__(*args, **kwargs)
self.old_dict = {}
self.changeset = {}
# right now it's not different than method desc, only code is different
def getcode(self):
return self.pyobj.im_func.func_code
code = property(getcode)
## def has_code(self, code):
## return self.pyobj.im_func.func_code is code
def __hash__(self):
return hash((self.code, self.pyobj.im_class))
def __eq__(self, other):
if isinstance(other, tuple):
return self.code is other[0] and self.pyobj.im_class is other[1]
if isinstance(other, MethodDesc):
return self.pyobj is other.pyobj
return False
def consider_start_locals(self, frame):
# XXX recursion issues?
obj = frame.f_locals[self.pyobj.im_func.func_code.co_varnames[0]]
try:
if not obj:
# static method
return
except AttributeError:
return
self.old_dict = self.perform_dict_copy(obj.__dict__)
def perform_dict_copy(self, d):
if d is None:
return {}
return d.copy()
def consider_end_locals(self, frame):
obj = frame.f_locals[self.pyobj.im_func.func_code.co_varnames[0]]
try:
if not obj:
# static method
return
except AttributeError:
return
# store the local changes
# update self.changeset
self.update_changeset(obj.__dict__)
def get_local_changes(self):
return self.changeset
def set_changeset(changeset, key, value):
if key not in changeset:
changeset[key] = set([value])
else:
changeset[key].add(value)
set_changeset = staticmethod(set_changeset)
def update_changeset(self, new_dict):
changeset = self.changeset
for k, v in self.old_dict.iteritems():
if k not in new_dict:
self.set_changeset(changeset, k, "deleted")
elif new_dict[k] != v:
self.set_changeset(changeset, k, "changed")
for k, v in new_dict.iteritems():
if k not in self.old_dict:
self.set_changeset(changeset, k, "created")
return changeset

356
py/apigen/tracer/docstorage.py
View File

@ -1,356 +0,0 @@
""" This module is keeping track about API informations as well as
providing some interface to easily access stored data
"""
import py
import sys
import types
import inspect
from py.__.apigen.tracer.description import FunctionDesc, ClassDesc, \
MethodDesc, Desc
from py.__.apigen.tracer import model
sorted = py.builtin.sorted
def pkg_to_dict(module):
defs = module.__pkg__.exportdefs
d = {}
for key, value in defs.iteritems():
chain = key.split('.')
base = module
# XXX generalize this:
# a bit of special casing for greenlets which are
# not available on all the platforms that python/py
# lib runs
try:
for elem in chain:
base = getattr(base, elem)
except RuntimeError, exc:
if elem == "greenlet":
print exc.__class__.__name__, exc
print "Greenlets not supported on this platform. Skipping apigen doc for this module"
continue
else:
raise
if value[1] == '*':
d.update(get_star_import_tree(base, key))
else:
d[key] = base
return d
def get_star_import_tree(module, modname):
""" deal with '*' entries in an initpkg situation """
ret = {}
modpath = py.path.local(inspect.getsourcefile(module))
pkgpath = module.__pkg__.getpath()
for objname in dir(module):
if objname.startswith('_'):
continue # also skip __*__ attributes
obj = getattr(module, objname)
if (isinstance(obj, types.ClassType) or
isinstance(obj, types.ObjectType)):
try:
sourcefile_object = py.path.local(
inspect.getsourcefile(obj))
except TypeError:
continue
else:
if sourcefile_object.strpath != modpath.strpath:
# not in this package
continue
dotted_name = '%s.%s' % (modname, objname)
ret[dotted_name] = obj
return ret
class DocStorage(object):
""" Class storing info about API
"""
def __init__(self):
self.module_name = None
def consider_call(self, frame, caller_frame, upward_cut_frame=None):
assert isinstance(frame, py.code.Frame)
desc = self.find_desc(frame.code, frame.raw.f_locals)
if desc:
self.generalize_args(desc, frame)
desc.consider_call_site(caller_frame, upward_cut_frame)
desc.consider_start_locals(frame)
def generalize_args(self, desc, frame):
args = [arg for key, arg in frame.getargs()]
#self.call_stack.append((desc, args))
desc.consider_call([model.guess_type(arg) for arg in args])
def generalize_retval(self, desc, arg):
desc.consider_return(model.guess_type(arg))
def consider_return(self, frame, arg):
assert isinstance(frame, py.code.Frame)
desc = self.find_desc(frame.code, frame.raw.f_locals)
if desc:
self.generalize_retval(desc, arg)
desc.consider_end_locals(frame)
def consider_exception(self, frame, arg):
desc = self.find_desc(frame.code, frame.raw.f_locals)
if desc:
exc_class, value, _ = arg
desc.consider_exception(exc_class, value)
def find_desc(self, code, locals):
try:
# argh, very fragile specialcasing
return self.desc_cache[(code.raw,
locals[code.raw.co_varnames[0]].__class__)]
except (KeyError, IndexError, AttributeError): # XXX hrmph
return self.desc_cache.get(code.raw, None)
#for desc in self.descs.values():
# if desc.has_code(frame.code.raw):
# return desc
#return None
def make_cache(self):
self.desc_cache = {}
for key, desc in self.descs.iteritems():
self.desc_cache[desc] = desc
def from_dict(self, _dict, keep_frames=False, module_name=None):
self.module_name = module_name
self.descs = {}
for key, val in _dict.iteritems():
to_key, to_val = self.make_desc(key, val)
if to_key:
self.descs[to_key] = to_val
self.make_cache()
# XXX
return self
# XXX: This function becomes slowly outdated and even might go away at some
# point. The question is whether we want to use tracer.magic or not
# at all
def add_desc(self, name, value, **kwargs):
key = name
count = 1
while key in self.descs:
key = "%s_%d" % (name, count)
count += 1
key, desc = self.make_desc(key, value, **kwargs)
if key:
self.descs[key] = desc
self.desc_cache[desc] = desc
return desc
else:
return None
def make_desc(self, key, value, add_desc=True, **kwargs):
if isinstance(value, types.FunctionType):
desc = FunctionDesc(key, value, **kwargs)
elif isinstance(value, (types.ObjectType, types.ClassType)):
desc = ClassDesc(key, value, **kwargs)
# XXX: This is the special case when we do not have __init__
# in dir(value) for uknown reason. Need to investigate it
for name in dir(value) + ['__init__']:
field = getattr(value, name, None)
if isinstance(field, types.MethodType) and \
isinstance(field.im_func, types.FunctionType):
real_name = key + '.' + name
md = MethodDesc(real_name, field)
if add_desc: # XXX hack
self.descs[real_name] = md
desc.add_method_desc(name, md)
# Some other fields as well?
elif isinstance(value, types.MethodType):
desc = MethodDesc(key, value, **kwargs)
else:
desc = Desc(value)
return (key, desc) # How to do it better? I want a desc to be a key
# value, but I cannot get full object if I do a lookup
def from_pkg(self, module, keep_frames=False):
self.module = module
self.from_dict(pkg_to_dict(module), keep_frames, module.__name__)
# XXX
return self
def from_module(self, func):
raise NotImplementedError("From module")
class AbstractDocStorageAccessor(object):
def __init__(self):
raise NotImplementedError("Purely virtual object")
def get_function_names(self):
""" Returning names of all functions
"""
def get_class_names(self):
""" Returning names of all classess
"""
def get_doc(self, name):
""" Returning __doc__ of a function
"""
def get_function_definition(self, name):
""" Returns definition of a function (source)
"""
def get_function_signature(self, name):
""" Returns types of a function
"""
def get_function_callpoints(self, name):
""" Returns list of all callpoints
"""
def get_module_name(self):
pass
def get_class_methods(self, name):
""" Returns all methods of a class
"""
#def get_object_info(self, key):
#
def get_module_info(self):
""" Returns module information
"""
class DocStorageAccessor(AbstractDocStorageAccessor):
""" Set of helper functions to access DocStorage, separated in different
class to keep abstraction
"""
def __init__(self, ds):
self.ds = ds
def _get_names(self, filter):
return [i for i, desc in self.ds.descs.iteritems() if filter(i, desc)]
def get_function_names(self):
return sorted(self._get_names(lambda i, desc: type(desc) is
FunctionDesc))
def get_class_names(self):
return sorted(self._get_names(lambda i, desc: isinstance(desc,
ClassDesc)))
#def get_function(self, name):
# return self.ds.descs[name].pyobj
def get_doc(self, name):
return self.ds.descs[name].pyobj.__doc__ or "*Not documented*"
def get_function_definition(self, name):
desc = self.ds.descs[name]
assert isinstance(desc, FunctionDesc)
code = py.code.Code(desc.code)
return code.fullsource[code.firstlineno]
def get_function_signature(self, name):
desc = self.ds.descs[name]
# we return pairs of (name, type) here
names = desc.pyobj.func_code.co_varnames[
:desc.pyobj.func_code.co_argcount]
types = desc.inputcells
return zip(names, types), desc.retval
def get_function_source(self, name):
desc = self.ds.descs[name]
try:
return str(py.code.Source(desc.pyobj))
except IOError:
return "Cannot get source"
def get_function_callpoints(self, name):
# return list of tuple (filename, fileline, frame)
return self.ds.descs[name].get_call_sites()
def get_function_local_changes(self, name):
return self.ds.descs[name].get_local_changes()
def get_function_exceptions(self, name):
return sorted([i.__name__ for i in self.ds.descs[name].exceptions.keys()])
def get_module_name(self):
if self.ds.module_name is not None:
return self.ds.module_name
elif hasattr(self.ds, 'module'):
return self.ds.module.__name__
return "Unknown module"
def get_class_methods(self, name):
desc = self.ds.descs[name]
assert isinstance(desc, ClassDesc)
return sorted(desc.getfields())
def get_module_info(self):
module = getattr(self.ds, 'module', None)
if module is None:
return "Lack of module info"
try:
retval = module.__doc__ or "*undocumented*"
retval = module.__pkg__.description
retval = module.__pkg__.long_description
except AttributeError:
pass
return retval
def get_type_desc(self, _type):
# XXX We provide only classes here
if not isinstance(_type, model.SomeClass):
return None
# XXX we might want to cache it at some point
for key, desc in self.ds.descs.iteritems():
if desc.pyobj == _type.cls:
return key, 'class', desc.is_degenerated
return None
def get_method_origin(self, name):
method = self.ds.descs[name].pyobj
cls = method.im_class
if not cls.__bases__:
return self.desc_from_pyobj(cls, cls.__name__)
curr = cls
while curr:
for base in curr.__bases__:
basefunc = getattr(base, method.im_func.func_name, None)
if (basefunc is not None and hasattr(basefunc, 'im_func') and
hasattr(basefunc.im_func, 'func_code') and
basefunc.im_func.func_code is
method.im_func.func_code):
curr = base
break
else:
break
return self.desc_from_pyobj(curr, curr.__name__)
def get_possible_base_classes(self, name):
cls = self.ds.descs[name].pyobj
if not hasattr(cls, '__bases__'):
return []
retval = []
for base in cls.__bases__:
desc = self.desc_from_pyobj(base, base.__name__)
if desc is not None:
retval.append(desc)
return retval
def desc_from_pyobj(self, pyobj, name):
for desc in self.ds.descs.values():
if isinstance(desc, ClassDesc) and desc.pyobj is pyobj:
return desc
# otherwise create empty desc
key, desc = self.ds.make_desc(name, pyobj, False)
#self.ds.descs[key] = desc
desc.is_degenerated = True
# and make sure we'll not try to link to it directly
return desc
def get_obj(self, name):
return self.ds.descs[name].pyobj

63
py/apigen/tracer/magic.py
View File

@ -1,63 +0,0 @@
""" magic - some operations which helps to extend PDB with some magic data.
Actually there is only explicit tracking of data, might be extended to
automatic at some point.
"""
# some magic stuff to have singleton of DocStorage, but initialised explicitely
import weakref
import py
from py.__.apigen.tracer.docstorage import DocStorage
from py.__.apigen.tracer.tracer import Tracer
import sys
class DocStorageKeeper(object):
doc_storage = DocStorage()
doc_storage.tracer = Tracer(doc_storage)
doc_storage.from_dict({})
def set_storage(cl, ds):
cl.doc_storage = ds
cl.doc_storage.tracer = Tracer(ds)
set_storage = classmethod(set_storage)
def get_storage():
return DocStorageKeeper.doc_storage
def stack_copier(frame):
# copy all stack, not only frame
num = 0
gather = False
stack = []
try:
while 1:
if gather:
stack.append(py.code.Frame(sys._getframe(num)))
else:
if sys._getframe(num) is frame.raw:
gather = True
num += 1
except ValueError:
pass
return stack
def trace(keep_frames=False, frame_copier=lambda x:x):
def decorator(fun):
ds = get_storage()
# in case we do not have this function inside doc storage, we
# want to have it
desc = ds.find_desc(py.code.Code(fun.func_code))
if desc is None:
desc = ds.add_desc(fun.func_name, fun, keep_frames=keep_frames,
frame_copier=frame_copier)
def wrapper(*args, **kwargs):
ds.tracer.start_tracing()
retval = fun(*args, **kwargs)
ds.tracer.end_tracing()
return retval
return wrapper
return decorator

331
py/apigen/tracer/model.py
View File

@ -1,331 +0,0 @@
""" model - type system model for apigen
"""
# we implement all the types which are in the types.*, naming
# scheme after pypy's
import py
import types
set = py.builtin.set
# __extend__ and pairtype?
class SomeObject(object):
typedef = types.ObjectType
def __repr__(self):
return "<%s>" % self.__class__.__name__[4:]
return str(self.typedef)[7:-2]
def unionof(self, other):
if isinstance(other, SomeImpossibleValue):
return self
if isinstance(other, SomeUnion):
return other.unionof(self)
if self == other:
return self
return SomeUnion([self, other])
def gettypedef(self):
return self.typedef
def __hash__(self):
return hash(self.__class__)
def __eq__(self, other):
return self.__class__ == other.__class__
def __ne__(self, other):
return not self == other
# this is to provide possibility of eventually linking some stuff
def striter(self):
yield str(self)
class SomeUnion(object):
# empty typedef
def __init__(self, possibilities):
self.possibilities = set(possibilities)
def unionof(self, other):
if isinstance(other, SomeUnion):
return SomeUnion(self.possibilities.union(other.possibilities))
return SomeUnion(list(self.possibilities) + [other])
def __eq__(self, other):
if type(other) is not SomeUnion:
return False
return self.possibilities == other.possibilities
def __ne__(self, other):
return not self == other
def __repr__(self):
return "AnyOf(%s)" % ", ".join([str(i) for i in list(self.possibilities)])
def gettypedef(self):
return (None, None)
def striter(self):
yield "AnyOf("
for num, i in enumerate(self.possibilities):
yield i
if num != len(self.possibilities) - 1:
yield ", "
yield ")"
class SomeBoolean(SomeObject):
typedef = types.BooleanType
class SomeBuffer(SomeObject):
typedef = types.BufferType
class SomeBuiltinFunction(SomeObject):
typedef = types.BuiltinFunctionType
#class SomeBuiltinMethod(SomeObject):
# typedef = types.BuiltinMethodType
class SomeClass(SomeObject):
typedef = types.ClassType
def __init__(self, cls):
self.cls = cls
self.name = cls.__name__
self.id = id(cls)
def __getstate__(self):
return (self.name, self.id)
def __setstate__(self, state):
self.name, self.id = state
self.cls = None
def __hash__(self):
return hash("Class") ^ hash(self.id)
def __eq__(self, other):
if type(other) is not SomeClass:
return False
return self.id == other.id
def unionof(self, other):
if type(other) is not SomeClass or self.id is not other.id:
return super(SomeClass, self).unionof(other)
return self
def __repr__(self):
return "Class %s" % self.name
class SomeCode(SomeObject):
typedef = types.CodeType
class SomeComplex(SomeObject):
typedef = types.ComplexType
class SomeDictProxy(SomeObject):
typedef = types.DictProxyType
class SomeDict(SomeObject):
typedef = types.DictType
class SomeEllipsis(SomeObject):
typedef = types.EllipsisType
class SomeFile(SomeObject):
typedef = types.FileType
class SomeFloat(SomeObject):
typedef = types.FloatType
class SomeFrame(SomeObject):
typedef = types.FrameType
class SomeFunction(SomeObject):
typedef = types.FunctionType
class SomeGenerator(SomeObject):
typedef = types.GeneratorType
class SomeInstance(SomeObject):
def __init__(self, classdef):
self.classdef = classdef
def __hash__(self):
return hash("SomeInstance") ^ hash(self.classdef)
def __eq__(self, other):
if type(other) is not SomeInstance:
return False
return other.classdef == self.classdef
def unionof(self, other):
if type(other) is not SomeInstance:
return super(SomeInstance, self).unionof(other)
if self.classdef == other.classdef:
return self
return SomeInstance(unionof(self.classdef, other.classdef))
def __repr__(self):
return "<Instance of %s>" % str(self.classdef)
def striter(self):
yield "<Instance of "
yield self.classdef
yield ">"
typedef = types.InstanceType
class SomeInt(SomeObject):
typedef = types.IntType
class SomeLambda(SomeObject):
typedef = types.LambdaType
class SomeList(SomeObject):
typedef = types.ListType
class SomeLong(SomeObject):
typedef = types.LongType
class SomeMethod(SomeObject):
typedef = types.MethodType
class SomeModule(SomeObject):
typedef = types.ModuleType
class SomeNone(SomeObject):
typedef = types.NoneType
class SomeNotImplemented(SomeObject):
typedef = types.NotImplementedType
class SomeObject(SomeObject):
typedef = types.ObjectType
class SomeSlice(SomeObject):
typedef = types.SliceType
class SomeString(SomeObject):
typedef = types.StringType
class SomeTraceback(SomeObject):
typedef = types.TracebackType
class SomeTuple(SomeObject):
typedef = types.TupleType
class SomeType(SomeObject):
typedef = types.TypeType
class SomeUnboundMethod(SomeObject):
typedef = types.UnboundMethodType
class SomeUnicode(SomeObject):
typedef = types.UnicodeType
class SomeXRange(SomeObject):
typedef = types.XRangeType
class SomeImpossibleValue(SomeObject):
def unionof(self, other):
return other
def __repr__(self):
return "<UNKNOWN>"
s_ImpossibleValue = SomeImpossibleValue()
s_None = SomeNone()
s_Ellipsis = SomeEllipsis()
def guess_type(x):
# this is mostly copy of immutablevalue
if hasattr(x, 'im_self') and x.im_self is None:
x = x.im_func
assert not hasattr(x, 'im_self')
tp = type(x)
if tp is bool:
result = SomeBoolean()
elif tp is int:
result = SomeInt()
elif issubclass(tp, str):
result = SomeString()
elif tp is unicode:
result = SomeUnicode()
elif tp is tuple:
result = SomeTuple()
#result = SomeTuple(items = [self.immutablevalue(e, need_const) for e in x])
elif tp is float:
result = SomeFloat()
elif tp is list:
#else:
# listdef = ListDef(self, s_ImpossibleValue)
# for e in x:
# listdef.generalize(self.annotation_from_example(e))
result = SomeList()
elif tp is dict:
## dictdef = DictDef(self,
## s_ImpossibleValue,
## s_ImpossibleValue,
## is_r_dict = tp is r_dict)
## if tp is r_dict:
## s_eqfn = self.immutablevalue(x.key_eq)
## s_hashfn = self.immutablevalue(x.key_hash)
## dictdef.dictkey.update_rdict_annotations(s_eqfn,
## s_hashfn)
## for ek, ev in x.iteritems():
## dictdef.generalize_key(self.annotation_from_example(ek))
## dictdef.generalize_value(self.annotation_from_example(ev))
result = SomeDict()
elif tp is types.ModuleType:
result = SomeModule()
elif callable(x):
#if hasattr(x, '__self__') and x.__self__ is not None:
# # for cases like 'l.append' where 'l' is a global constant list
# s_self = self.immutablevalue(x.__self__, need_const)
# result = s_self.find_method(x.__name__)
# if result is None:
# result = SomeObject()
#elif hasattr(x, 'im_self') and hasattr(x, 'im_func'):
# # on top of PyPy, for cases like 'l.append' where 'l' is a
# # global constant list, the find_method() returns non-None
# s_self = self.immutablevalue(x.im_self, need_const)
# result = s_self.find_method(x.im_func.__name__)
#else:
# result = None
#if result is None:
# if (self.annotator.policy.allow_someobjects
# and getattr(x, '__module__', None) == '__builtin__'
# # XXX note that the print support functions are __builtin__
# and tp not in (types.FunctionType, types.MethodType)):
## result = SomeObject()
# result.knowntype = tp # at least for types this needs to be correct
# else:
# result = SomePBC([self.getdesc(x)])
if tp is types.BuiltinFunctionType or tp is types.BuiltinMethodType:
result = SomeBuiltinFunction()
elif hasattr(x, 'im_func'):
result = SomeMethod()
elif hasattr(x, 'func_code'):
result = SomeFunction()
elif hasattr(x, '__class__'):
if x.__class__ is type:
result = SomeClass(x)
else:
result = SomeInstance(SomeClass(x.__class__))
elif tp is types.ClassType:
result = SomeClass(x)
elif x is None:
return s_None
elif hasattr(x, '__class__'):
result = SomeInstance(SomeClass(x.__class__))
else:
result = SomeObject()
# XXX here we might want to consider stuff like
# buffer, slice, etc. etc. Let's leave it for now
return result
def unionof(first, other):
return first.unionof(other)

106
py/apigen/tracer/permastore.py
View File

@ -1,106 +0,0 @@
import py
class DescPlaceholder(object):
pass
class ClassPlaceholder(object):
pass
class SerialisableClassDesc(object):
def __init__(self, original_desc):
self.is_degenerated = original_desc.is_degenerated
self.name = original_desc.name
class PermaDocStorage(object):
""" Picklable version of docstorageaccessor
"""
function_fields = ['source', 'signature', 'definition', 'callpoints',
'local_changes', 'exceptions']
def __init__(self, dsa):
""" Initialise from original doc storage accessor
"""
self.names = {}
self.module_info = dsa.get_module_info()
self.module_name = dsa.get_module_name()
self._save_functions(dsa)
self._save_classes(dsa)
def _save_functions(self, dsa):
names = dsa.get_function_names()
self.function_names = names
for name in names:
self._save_function(dsa, name)
def _save_function(self, dsa, name):
ph = DescPlaceholder()
ph.__doc__ = dsa.get_doc(name)
for field in self.function_fields:
setattr(ph, field, getattr(dsa, 'get_function_%s' % field)(name))
self.names[name] = ph
return ph
def _save_classes(self, dsa):
names = dsa.get_class_names()
self.class_names = names
for name in names:
ph = ClassPlaceholder()
ph.__doc__ = dsa.get_doc(name)
methods = dsa.get_class_methods(name)
ph.methods = methods
ph.base_classes = [SerialisableClassDesc(i) for i in
dsa.get_possible_base_classes(name)]
for method in methods:
method_name = name + "." + method
mh = self._save_function(dsa, name + "." + method)
mh.origin = SerialisableClassDesc(dsa.get_method_origin(
method_name))
self.names[name] = ph
def get_class_methods(self, name):
desc = self.names[name]
assert isinstance(desc, ClassPlaceholder)
return desc.methods
def get_doc(self, name):
return self.names[name].__doc__
def get_module_info(self):
return self.module_info
def get_module_name(self):
return self.module_name
def get_class_names(self):
return self.class_names
def get_function_names(self):
return self.function_names
def get_method_origin(self, name):
# returns a DESCRIPTION of a method origin, to make sure where we
# write it
return self.names[name].origin
def get_possible_base_classes(self, name):
# returns list of descs of base classes
return self.names[name].base_classes
# This are placeholders to provide something more reliable
def get_type_desc(self, _type):
return None
#def get_obj(self, name):
# This is quite hairy, get rid of it soon
# # returns a pyobj
# pass
for field in PermaDocStorage.function_fields:
d = {"field": field}
func_name = "get_function_%s" % (field, )
exec py.code.Source("""
def %s(self, name, field=field):
return getattr(self.names[name], field)
""" % (func_name, )).compile() in d
setattr(PermaDocStorage, func_name, d[func_name])

0
py/apigen/tracer/testing/__init__.py
View File

0
py/apigen/tracer/testing/package/__init__.py
View File

13
py/apigen/tracer/testing/package/submodule/__init__.py
View File

@ -1,13 +0,0 @@
import py
from py.__.initpkg import initpkg
initpkg(__name__,
description="test package",
exportdefs = {
'pak.mod.one': ('./pak/mod.py', 'one'),
'pak.mod.two': ('./pak/mod.py', 'nottwo'),
'notpak.notmod.notclass': ('./pak/mod.py', 'cls'),
'somenamespace': ('./pak/somenamespace.py', '*'),
})

0
py/apigen/tracer/testing/package/submodule/pak/__init__.py
View File

10
py/apigen/tracer/testing/package/submodule/pak/mod.py
View File

@ -1,10 +0,0 @@
class cls(object):
def __init__(self, x):
self.x = x
def one(x):
return x+3
def nottwo():
pass

5
py/apigen/tracer/testing/package/submodule/pak/somenamespace.py
View File

@ -1,5 +0,0 @@
def foo(x):
return x + 1
def bar(x):
return x + 2

5
py/apigen/tracer/testing/runtest.py
View File

@ -1,5 +0,0 @@
def cut_pyc(f_name):
if f_name.endswith('.pyc'):
return f_name[:-1]
return f_name

28
py/apigen/tracer/testing/test_desc.py
View File

@ -1,28 +0,0 @@
""" Some additional tests about descriptions
"""
from py.__.apigen.tracer.description import *
class A:
pass
class B(object):
def __init__(self):
pass
class C(object):
pass
class D:
def __init__(self):
pass
def test_getcode():
assert hash(ClassDesc("a", A).code)
assert hash(ClassDesc("b", B).code)
assert hash(ClassDesc("c", C).code)
assert hash(ClassDesc("d", D).code)
def test_eq():
assert ClassDesc('a', A) == ClassDesc('a', A)

454
py/apigen/tracer/testing/test_docgen.py
View File

@ -1,454 +0,0 @@
""" test doc generation
"""
import py
import sys
#try:
from py.__.apigen.tracer.tracer import Tracer
from py.__.apigen.tracer.docstorage import DocStorageAccessor, DocStorage, \
get_star_import_tree, pkg_to_dict
from py.__.apigen.tracer.testing.runtest import cut_pyc
from py.__.apigen.tracer.description import FunctionDesc
from py.__.apigen.tracer import model
from py.__.apigen.tracer.permastore import PermaDocStorage
def setup_module(mod):
if py.std.sys.platform == "win32":
py.test.skip("tracing on win32 not supported")
# XXX: Perma doc storage disabled a bit
sorted = py.builtin.sorted
set = py.builtin.set
def fun(a, b, c):
"Some docstring"
return "d"
def test_basic():
descs = {"fun":fun}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
fun(1, ("g", 3), 8)
fun(2., ("a", 1.), "a")
t.end_tracing()
desc = ds.descs['fun']
inputcells = desc.inputcells
assert len(inputcells) == 3
assert isinstance(inputcells[0], model.SomeUnion)
assert isinstance(inputcells[1], model.SomeTuple)
assert isinstance(inputcells[2], model.SomeUnion)
assert isinstance(desc.retval, model.SomeString)
cs = sorted(desc.call_sites.keys())
assert len(cs) == 2
f_name = cut_pyc(__file__)
assert len(cs[0]) == 1
assert len(cs[1]) == 1
assert cs[1][0].filename == f_name
# lines are counted from 0
num = test_basic.func_code.co_firstlineno
assert cs[1][0].lineno == num + 4 or cs[1][0].lineno == num + 5
assert cs[0][0].filename == f_name
assert cs[0][0].lineno == num + 5 or cs[0][0].lineno == num + 4
if 0:
pds = PermaDocStorage(DocStorageAccessor(ds))
assert pds.get_function_names() == ['fun']
sig = pds.get_function_signature('fun')
assert sig[0][0][0] == 'a'
assert isinstance(sig[0][0][1], model.SomeUnion)
assert len(pds.get_function_callpoints('fun')) == 2
class AClass(object):
""" Class docstring
"""
def __init__(self, b="blah"):
pass
def exposed_method(self, a, b, c):
""" method docstring
"""
return self._hidden_method()
def _hidden_method(self):
""" should not appear
"""
return "z"
class ANotherClass(AClass):
def another_exposed_method(self, a):
# no docstring
return a
def test_class():
descs = {'AClass':AClass}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
s = AClass()
s.exposed_method(1, 2., [1,2,3])
t.end_tracing()
desc = ds.descs['AClass']
inputcells = desc.fields['__init__'].inputcells
assert len(inputcells) == 2
assert isinstance(inputcells[0], model.SomeInstance)
#assert inputcells[0].classdef.classdesc.pyobj is SomeClass
# XXX: should work
assert isinstance(inputcells[1], model.SomeString)
f_name = __file__
if f_name.endswith('.pyc'):
f_name = f_name[:-1]
cs = sorted(desc.fields['__init__'].call_sites.keys())
assert len(cs) == 1
assert len(cs[0]) == 1
assert cs[0][0].filename == f_name
assert cs[0][0].lineno == test_class.func_code.co_firstlineno + 4
# method check
assert sorted(desc.getfields()) == ['__init__', 'exposed_method']
inputcells = desc.fields['exposed_method'].inputcells
assert len(inputcells) == 4
assert isinstance(inputcells[0], model.SomeInstance)
#assert inputcells[0].classdef.classdesc.pyobj is SomeClass
# XXX should work
assert isinstance(inputcells[1], model.SomeInt)
assert isinstance(inputcells[2], model.SomeFloat)
assert isinstance(inputcells[3], model.SomeList)
assert isinstance(desc.fields['exposed_method'].retval, model.SomeString)
if 0:
pds = PermaDocStorage(DocStorageAccessor(ds))
assert pds.get_class_names() == ['AClass']
assert len(pds.get_function_signature('AClass.exposed_method')[0]) == 4
def other_fun():
pass
def test_add_desc():
ds = DocStorage().from_dict({})
ds.add_desc("one", fun)
ds.add_desc("one", other_fun)
assert sorted(ds.descs.keys()) == ["one", "one_1"]
assert isinstance(ds.descs["one"], FunctionDesc)
assert isinstance(ds.descs["one_1"], FunctionDesc)
assert ds.descs["one"].pyobj is fun
assert ds.descs["one_1"].pyobj is other_fun
assert ds.desc_cache[ds.descs["one"]] is ds.descs["one"]
assert ds.desc_cache[ds.descs["one_1"]] is ds.descs["one_1"]
def test_while_call():
ds = DocStorage().from_dict({"other_fun":other_fun})
t = Tracer(ds)
t.start_tracing()
for x in xrange(8):
other_fun()
t.end_tracing()
desc = ds.descs["other_fun"]
assert len(desc.call_sites.keys()) == 1
#assert isinstance(desc.call_sites.values()[0][0], py.code.Frame)
if 0:
pds = PermaDocStorage(DocStorageAccessor(ds))
assert len(pds.get_function_callpoints("other_fun")) == 1
class A(object):
def method(self, x):
self.x = x
class B:
def method(self, x):
self.x = x
def test_without_init():
ds = DocStorage().from_dict({'A':A, 'B':B})
t = Tracer(ds)
t.start_tracing()
x = A()
y = B()
x.method(3)
y.method(4)
t.end_tracing()
assert isinstance(ds.descs['A'].fields['method'].inputcells[1],
model.SomeInt)
assert isinstance(ds.descs['B'].fields['method'].inputcells[1],
model.SomeInt)
if 0:
pds = PermaDocStorage(DocStorageAccessor(ds))
def test_local_changes():
class testclass(object):
def __init__(self):
self.foo = 0
def bar(self, x):
self.foo = x
ds = DocStorage().from_dict({'testclass': testclass})
t = Tracer(ds)
t.start_tracing()
c = testclass()
c.bar(1)
t.end_tracing()
desc = ds.descs['testclass']
methdesc = desc.fields['bar']
#assert methdesc.old_dict != methdesc.new_dict
assert methdesc.get_local_changes() == {'foo': set(['changed'])}
return ds
def test_local_changes_nochange():
class testclass(object):
def __init__(self):
self.foo = 0
def bar(self, x):
self.foo = x
ds = DocStorage().from_dict({'testclass': testclass})
t = Tracer(ds)
t.start_tracing()
c = testclass()
t.end_tracing()
desc = ds.descs['testclass']
methdesc = desc.fields['bar']
assert methdesc.get_local_changes() == {}
return ds
def test_multiple_classes_with_same_init():
class A:
def __init__(self, x):
self.x = x
class B(A):
pass
ds = DocStorage().from_dict({'A':A, 'B':B})
t = Tracer(ds)
t.start_tracing()
c = A(3)
d = B(4)
t.end_tracing()
assert len(ds.descs['A'].fields['__init__'].call_sites) == 1
assert len(ds.descs['B'].fields['__init__'].call_sites) == 1
return ds
def test_exception_raise():
def x():
1/0
def y():
try:
x()
except ZeroDivisionError:
pass
def z():
y()
ds = DocStorage().from_dict({'x':x, 'y':y, 'z':z})
t = Tracer(ds)
t.start_tracing()
z()
t.end_tracing()
assert ds.descs['x'].exceptions.keys() == [ZeroDivisionError]
assert ds.descs['y'].exceptions.keys() == [ZeroDivisionError]
assert ds.descs['z'].exceptions.keys() == []
return ds
def test_subclass():
descs = {'ANotherClass': ANotherClass}
ds = DocStorage().from_dict(descs)
t = Tracer(ds)
t.start_tracing()
s = ANotherClass('blah blah')
s.another_exposed_method(1)
t.end_tracing()
desc = ds.descs['ANotherClass']
assert len(desc.fields) == 4
inputcells = desc.fields['__init__'].inputcells
assert len(inputcells) == 2
inputcells = desc.fields['another_exposed_method'].inputcells
assert len(inputcells) == 2
bases = desc.bases
assert len(bases) == 2
return ds
def test_bases():
class A:
pass
class B:
pass
class C(A,B):
pass
ds = DocStorage().from_dict({'C':C, 'B':B})
dsa = DocStorageAccessor(ds)
for desc in dsa.get_possible_base_classes('C'):
assert desc is ds.descs['B'] or desc.is_degenerated
return ds
def test_desc_from_pyobj():
class A:
pass
class B(A):
pass
ds = DocStorage().from_dict({'A': A, 'B': B})
dsa = DocStorageAccessor(ds)
assert dsa.desc_from_pyobj(A, 'A') is ds.descs['A']
return ds
def test_method_origin():
class A:
def foo(self):
pass
class B(A):
def bar(self):
pass
class C(B):
pass
ds = DocStorage().from_dict({'C': C, 'B': B})
dsa = DocStorageAccessor(ds)
origin = dsa.get_method_origin('C.bar')
assert origin is ds.descs['B']
return ds
def test_multiple_methods():
class A(object):
def meth(self):
pass
class B(A):
pass
class C(A):
pass
ds = DocStorage().from_dict({'C':C, 'B':B})
dsa = DocStorageAccessor(ds)
t = Tracer(ds)
t.start_tracing()
B().meth()
C().meth()
t.end_tracing()
assert len(ds.descs['B'].fields['meth'].call_sites) == 1
assert len(ds.descs['C'].fields['meth'].call_sites) == 1
return ds
def test_is_private():
# XXX implicit test, but so are the rest :|
class Foo(object):
def foo(self):
pass
def _foo(self):
pass
def __foo(self):
pass
def trigger__foo(self):
self.__foo()
def __foo__(self):
pass
ds = DocStorage().from_dict({'Foo': Foo})
dsa = DocStorageAccessor(ds)
t = Tracer(ds)
t.start_tracing()
f = Foo()
f.foo()
f._foo()
f.trigger__foo()
f.__foo__()
t.end_tracing()
assert sorted(ds.descs['Foo'].getfields()) == ['__foo__', 'foo',
'trigger__foo']
def setup_fs_project():
temp = py.test.ensuretemp('test_get_initpkg_star_items')
temp.ensure("pkg/func.py").write(py.code.Source("""\
def func(arg1):
"docstring"
"""))
temp.ensure('pkg/someclass.py').write(py.code.Source("""\
class SomeClass(object):
" docstring someclass "
def __init__(self, somevar):
self.somevar = somevar
def get_somevar(self):
" get_somevar docstring "
return self.somevar
SomeInstance = SomeClass(10)
"""))
temp.ensure('pkg/somesubclass.py').write(py.code.Source("""\
from someclass import SomeClass
class SomeSubClass(SomeClass):
" docstring somesubclass "
#def get_somevar(self):
# return self.somevar + 1
"""))
temp.ensure('pkg/somenamespace.py').write(py.code.Source("""\
from pkg.main.sub import func
import py
def foo():
return 'bar'
def baz(qux):
return qux
quux = py.code.Source('print "foo"')
"""))
temp.ensure("pkg/__init__.py").write(py.code.Source("""\
from py.initpkg import initpkg
initpkg(__name__, exportdefs = {
'main.sub.func': ("./func.py", "func"),
'main.SomeClass': ('./someclass.py', 'SomeClass'),
#'main.SomeInstance': ('./someclass.py', 'SomeInstance'),
'main.SomeSubClass': ('./somesubclass.py', 'SomeSubClass'),
'other': ('./somenamespace.py', '*'),
})
"""))
return temp, 'pkg'
def setup_pkg_docstorage():
pkgdir, pkgname = setup_fs_project()
py.std.sys.path.insert(0, str(pkgdir))
# XXX test_get_initpkg_star_items depends on package not
# being imported already
for key in py.std.sys.modules.keys():
if key == pkgname or key.startswith(pkgname + "."):
del py.std.sys.modules[key]
pkg = __import__(pkgname)
ds = DocStorage().from_pkg(pkg)
return pkg, ds
def test_get_initpkg_star_items():
pkg, ds = setup_pkg_docstorage()
sit = get_star_import_tree(pkg.other, 'pkg.other')
assert sorted(sit.keys()) == ['pkg.other.baz', 'pkg.other.foo']
t = Tracer(ds)
t.start_tracing()
pkg.main.sub.func("a1")
pkg.main.SomeClass(3).get_somevar()
pkg.main.SomeSubClass(4).get_somevar()
t.end_tracing()
assert isinstance(ds.descs['main.sub.func'].inputcells[0], model.SomeString)
desc = ds.descs['main.SomeClass']
assert ds.descs['main.SomeClass.get_somevar'] is desc.fields['get_somevar']
cell = desc.fields['get_somevar'].inputcells[0]
assert isinstance(cell, model.SomeInstance)
assert cell.classdef.cls is desc.pyobj
desc = ds.descs['main.SomeSubClass']
assert ds.descs['main.SomeSubClass.get_somevar'] is desc.fields['get_somevar']
cell = desc.fields['get_somevar'].inputcells[0]
assert isinstance(cell, model.SomeInstance)
assert cell.classdef.cls is desc.pyobj
def test_pkg_to_dict():
pkg, ds = setup_pkg_docstorage()
assert sorted(pkg_to_dict(pkg).keys()) == ['main.SomeClass',
'main.SomeSubClass',
'main.sub.func',
'other.baz',
'other.foo']

112
py/apigen/tracer/testing/test_model.py
View File

@ -1,112 +0,0 @@
""" test_model - our (very simple) type system
model tests
"""
from py.__.apigen.tracer.model import *
import types
import py
def check_guess(val, t):
assert isinstance(guess_type(val), t)
def test_basic():
""" This tests checks every object that we might want
to track
"""
check_guess(3, SomeInt)
check_guess(3., SomeFloat)
check_guess(True, SomeBoolean)
check_guess(lambda x: None, SomeFunction)
class A:
pass
check_guess(A, SomeClass)
check_guess(A(), SomeInstance)
class B(object):
def meth(self):
pass
class C(object):
def __call__(self):
pass
check_guess(B, SomeClass)
check_guess(B.meth, SomeFunction)
check_guess(B(), SomeInstance)
check_guess(B().meth, SomeMethod)
check_guess([1], SomeList)
check_guess(None, SomeNone)
check_guess((1,), SomeTuple)
check_guess(C(), SomeInstance)
import sys
check_guess(sys, SomeModule)
check_guess({}, SomeDict)
check_guess(sys.exc_info, SomeBuiltinFunction)
def test_anyof():
def check_lst(lst):
a = guess_type(lst[0])
for i in lst[1:]:
a = unionof(a, guess_type(i))
d = dict([(i, True) for i in a.possibilities])
assert len(a.possibilities) == len(d)
for i in a.possibilities:
assert not isinstance(i, SomeUnion)
return a
class C(object):
pass
ret = check_lst([3, 4, 3., "aa"])
assert len(ret.possibilities) == 3
ret = check_lst([3, 4, 3.])
ret2 = check_lst([1, "aa"])
ret3 = unionof(ret, ret2)
assert len(ret3.possibilities) == 3
ret = check_lst([3, 1.])
ret = unionof(ret, guess_type("aa"))
ret = unionof(guess_type("aa"), ret)
ret = unionof(guess_type(C()), ret)
ret = unionof(ret, guess_type("aa"))
ret = unionof(ret, guess_type(C()))
assert len(ret.possibilities) == 4
def test_union():
class A(object):
pass
class B(object):
pass
f = guess_type(A).unionof(guess_type(A))
assert isinstance(f, SomeClass)
assert f.cls is A
f = guess_type(A).unionof(guess_type(B)).unionof(guess_type(A))
assert isinstance(f, SomeUnion)
assert len(f.possibilities) == 2
f = guess_type(A()).unionof(guess_type(A()))
assert isinstance(f, SomeInstance)
assert isinstance(f.classdef, SomeClass)
assert f.classdef.cls is A
f = guess_type(B()).unionof(guess_type(A())).unionof(guess_type(B()))
assert isinstance(f, SomeInstance)
assert isinstance(f.classdef, SomeUnion)
assert len(f.classdef.possibilities) == 2
def test_striter():
class A(object):
pass
class B(object):
pass
g = guess_type(A).unionof(guess_type(A()))
l = py.builtin.sorted(list(g.striter()))
assert l[4] == "AnyOf("
assert isinstance(l[0], SomeClass)
assert l[3] == ", "
assert isinstance(l[1], SomeInstance)

54
py/apigen/tracer/testing/test_package.py
View File

@ -1,54 +0,0 @@
""" some tests for from_package
"""
from py.__.apigen.tracer.docstorage import DocStorage
from py.__.apigen.tracer.tracer import Tracer
from py.__.apigen.tracer import model
import sys
import py
def setup_module(mod):
sys.path.insert(0, str(py.path.local(__file__).dirpath().join("package")))
import submodule
mod.submodule = submodule
def teardown_module(mod):
sys.path = sys.path[1:]
class TestFullModule(object):
def setup_class(cls):
cls.ds = DocStorage().from_pkg(submodule)
cls.tracer = Tracer(cls.ds)
def test_init(self):
ds = self.ds
print py.builtin.sorted(ds.descs.keys())
if sys.platform == "win32":
py.test.skip("not sure why, but this fails with 4 == 6")
assert len(ds.descs) == 6
assert py.builtin.sorted(ds.descs.keys()) == [
'notpak.notmod.notclass', 'notpak.notmod.notclass.__init__',
'pak.mod.one', 'pak.mod.two', 'somenamespace.bar',
'somenamespace.foo']
def test_simple_call(self):
ds = self.ds
self.tracer.start_tracing()
submodule.pak.mod.one(3)
self.tracer.end_tracing()
desc = self.ds.descs['pak.mod.one']
assert isinstance(desc.retval, model.SomeInt)
assert isinstance(desc.inputcells[0], model.SomeInt)
def test_call_class(self):
ds = self.ds
self.tracer.start_tracing()
c = submodule.notpak.notmod.notclass(3)
self.tracer.end_tracing()
desc = self.ds.descs['notpak.notmod.notclass']
methdesc = desc.fields['__init__']
assert isinstance(methdesc.inputcells[0], model.SomeInstance)
assert isinstance(methdesc.inputcells[1], model.SomeInt)

55
py/apigen/tracer/tracer.py
View File

@ -1,55 +0,0 @@
""" simple tracer for API generation
"""
import py
import sys
import types
from py.__.apigen.tracer.description import FunctionDesc
from py.__.apigen.tracer.docstorage import DocStorage
class UnionError(Exception):
pass
class NoValue(object):
pass
class Tracer(object):
""" Basic tracer object, used for gathering additional info
about API functions
"""
def __init__(self, docstorage):
self.docstorage = docstorage
self.tracing = False
_locals = {}
def _tracer(self, frame, event, arg):
# perform actuall tracing
frame = py.code.Frame(frame)
if event == 'call':
assert arg is None
try:
self.docstorage.consider_call(frame,
py.code.Frame(sys._getframe(2)),
self.frame)
except ValueError:
self.docstorage.consider_call(frame, None, self.frame)
elif event == 'return':
self.docstorage.consider_return(frame, arg)
elif event == 'exception':
self.docstorage.consider_exception(frame, arg)
return self._tracer
def start_tracing(self):
if self.tracing:
return
self.tracing = True
self.frame = py.code.Frame(sys._getframe(1))
sys.settrace(self._tracer)
def end_tracing(self):
self.tracing = False
sys.settrace(None)

56
py/bin/gendoc.py
View File

@ -1,56 +0,0 @@
#!/usr/bin/env python
"""
build the 'py' documentation and api docs in a specified
directory, defaulting to 'html'. You need to be a directory
where your "py" package is.
This script generates API documentation and static
documentation. The API documentation is generated by using
the "apigen" facility of the py lib which currently only works
on windows.
"""
import sys
sys.path.insert(0, '.')
import py
import os
try:
import subprocess
except ImportError:
from py.__.compat import subprocess
def sysexec(cmd):
print "executing", cmd
p = subprocess.Popen(cmd, shell=True)
os.waitpid(p.pid, 0)
if __name__ == '__main__':
pydir = py.path.local().join("py")
assert pydir.check(dir=1), "py directory not found"
pypath = py.path.local(py.__file__).dirpath()
assert pydir == pypath, "directory %s and %s differ" %(pydir, pypath)
args = sys.argv[1:]
if not args:
htmldir = py.path.local('html')
else:
htmldir = py.path.local(sys.argv.pop(0))
print "generating docs into", htmldir
print "pypath", pypath
pytest = pypath.join("bin/py.test")
assert pytest.check()
print
print "*" * 30, "apigen", "*"*30
apigendir = htmldir.join("apigen")
env = 'DOCPATH="%s" APIGENPATH="%s"' %(htmldir, apigendir)
if apigendir.check():
print apigendir, "exists, not re-generating - remove to trigger regeneration"
else:
sysexec('%(env)s %(pytest)s py' % locals())
print
print "*" * 30, "static generation", "*" * 30
sysexec('%(env)s %(pytest)s --forcegen %(pypath)s/doc' % locals())

284
py/doc/apigen.txt
View File

@ -1,284 +0,0 @@
===========================================
apigen - API documentation generation tool
===========================================
What is it?
===========
Apigen is a tool for automatically generating API reference documentation for
Python projects. It works by examining code at runtime rather than at compile
time. This way it is capable of displaying information about the code base
after initialization. A drawback is that you cannot easily document source code
that automatically starts server processes or has some other irreversible
effects upon getting imported.
The apigen functionality is normally triggered from :api:`py.test`, and while
running the tests it gathers information such as code paths, arguments and
return values of callables, and exceptions that can be raised while the code
runs (XXX not yet!) to include in the documentation. It's also possible to
run the tracer (which collects the data) in other code if your project
does not use :api:`py.test` but still wants to collect the runtime information
and build the docs.
Apigen is written for the :api:`py` lib, but can be used to build documentation
for any project: there are hooks in py.test to, by providing a simple script,
build api documentation for the tested project when running py.test. Of course
this does imply :api:`py.test` is actually used: if little or no tests are
actually ran, the additional information (code paths, arguments and return
values and exceptions) can not be gathered and thus there will be less of an
advantage of apigen compared to other solutions.
Features
========
Some features were mentioned above already, but here's a complete list of all
the niceties apigen has to offer:
* source documents
Apigen not only builds the API documentation, but also a tree of
syntax-colored source files, with links from the API docs to the source
files.
* abundance of information
compared to other documentation generation tools, apigen produces an
abundant amount of information: it provides syntax-colored code snippets,
code path traces, etc.
* linking
besides links to the source files, apigen provides links all across the
documentation: callable arguments and return values link to their
definition (if part of the documented code), class definition to their
base classes (again, if they're part of the documented code), and
everywhere are links to the source files (including in traces)
* (hopefully) improves testing
because the documentation is built partially from test results, developers
may (especially if they're using the documentation themselves) be more
aware of untested parts of the code, or parts can use more tests or need
attention
Using apigen
============
To trigger apigen, all you need to do is run the :source:`py/bin/py.test` tool
with an --apigen argument, as such::
$ py.test --apigen=<path>
where <path> is a path to a script containing some special hooks to build
the documents (see below). The script to build the documents for the :api:`py`
lib can be found in :source:`py/apigen/apigen.py`, so building those documents
can be done by cd'ing to the 'py' directory, and executing::
$ py.test --apigen=apigen/apigen.py
The documents will by default be built in the *parent directory* of the
*package dir* (in this case the 'py' directory). Be careful that you don't
overwrite anything!
Other projects
==============
To use apigen from another project, there are three things that you need to do:
Use :api:`py.test` for unit tests
---------------------------------
This is a good idea anyway... ;) The more tests, the more tracing information
and such can be built, so it makes sense to have good test coverage when using
this tool.
Provide :api:`py.test` hooks
----------------------------
To hook into the unit testing framework, you will need to write a script with
two functions. The first should be called 'get_documentable_items', gets a
package dir (the root of the project) as argument, and should return a tuple
with the package name as first element, and a dict as second. The dict should
contain, for all the to-be-documented items, a dotted name as key and a
reference to the item as value.
The second function should be called 'build', and gets also the package dir as
argument, but also a reference to a DocStorageAcessor, which contains
information gathered by the tracer, and a reference to a
:api:`py.io.StdCaptureFD` instance that is used to capture stdout and stderr,
and allows writing to them, when the docs are built.
This 'build' function is responsible for actually building the documentation,
and, depending on your needs, can be used to control each aspect of it. In most
situations you will just copy the code from :source:`py/apigen/apigen.py`'s
build() function, but if you want you can choose to build entirely different
output formats by directly accessing the DocStorageAccessor class.
Provide layout
--------------
For the :api:`py` lib tests, the 'LayoutPage' class found in
:source:`py/apigen/layout.py` is used, which produces HTML specific for that
particular library (with a menubar, etc.). To customize this, you will need to
provide a similar class, most probably using the Page base class from
:source:`py/doc/confrest.py`. Note that this step depends on how heavy the
customization in the previous step is done: if you decide to directly use the
DocStorageAccessor rather than let the code in :source:`py/apigen/htmlgen.py`
build HTML for you, this can be skipped.
Using apigen from code
======================
If you want to avoid using :api:`py.test`, or have an other idea of how to best
collect information while running code, the apigen functionality can be
directly accessed. The most important classes are the Tracer class found in
:source:`py/apigen/tracer/tracer.py`, which holds the information gathered
during the tests, and the DocStorage and DocStorageAccessor classes from
:source:`py/apigen/tracer/docstorage.py`, which (respectively) store the data,
and make it accessible.
Gathering information
---------------------
To gather information about documentation, you will first need to tell the tool
what objects it should investigate. Only information for registered objects
will be stored. An example::
>>> import py
>>> from py.__.apigen.tracer.docstorage import DocStorage, DocStorageAccessor
>>> from py.__.apigen.tracer.tracer import Tracer
>>> toregister = {'py.path.local': py.path.local,
... 'py.path.svnwc': py.path.svnwc}
>>> ds = DocStorage().from_dict(toregister)
>>> t = Tracer(ds)
>>> t.start_tracing()
>>> p = py.path.local('.')
>>> p.check(dir=True)
True
>>> t.end_tracing()
Now the 'ds' variable should contain all kinds of information about both the
:api:`py.path.local` and the :api:`py.path.svnwc` classes, and things like call
stacks, possible argument types, etc. as additional information about
:api:`py.path.local.check()` (since it was called from the traced code).
Using the information
---------------------
To use the information, we need to get a DocStorageAccessor instance to
provide access to the data stored in the DocStorage object::
>>> dsa = DocStorageAccessor(ds)
Currently there is no API reference available for this object, so you'll have
to read the source (:source:`py/apigen/tracer/docstorage.py`) to see what
functionality it offers.
Comparison with other documentation generation tools
====================================================
Apigen is of course not the only documentation generation tool available for
Python. Although we knew in advance that our tool had certain features the
others do not offer, we decided to investigate a bit so that we could do a
proper comparison.
Tools examined
--------------
After some 'googling around', it turned out that the amount of documentation
generation tools available was surprisingly low. There were only 5 packages
I could find, of which 1 (called 'HappyDoc') seems dead (last release 2001),
one (called 'Pudge') not yet born (perhaps DOA even? most of the links on the
website are dead), and one (called 'Endo') specific to the Enthought suite.
The remaining two were Epydoc, which is widely used [1]_, and PyDoctor, which is
used only by (and written for) the Twisted project, but can be used seperately.
Epydoc
~~~~~~
http://epydoc.sourceforge.net/
Epydoc is the best known, and most widely used, documentation generation tool
for Python. It builds a documentation tree by inspecting imported modules and
using Python's introspection features. This way it can display information like
containment, inheritance, and docstrings.
The tool is relatively sophisticated, with support for generating HTML and PDF,
choosing different styles (CSS), generating graphs using Graphviz, etc. Also
it allows using markup (which can be ReST, JavaDoc, or their own 'epytext'
format) inside docstrings for displaying rich text in the result.
Quick overview:
* builds docs from object tree
* displays relatively little information, just inheritance trees, API and
docstrings
* supports some markup (ReST, 'epytext', JavaDoc) in docstrings
PyDoctor
~~~~~~~~
http://codespeak.net/~mwh/pydoctor/
This tool is written by Michael Hudson for the Twisted project. The major
difference between this and Epydoc is that it browses the AST (Abstract Syntax
Tree) instead of using 'live' objects, which means that code that uses special
import mechanisms, or depends on other code that is not available can still be
inspected. On the other hand, code that, for example, puts bound methods into a
module namespace is not documented.
The tool is relatively simple and doesn't support the more advanced features
that Epydoc offers. It was written for Twisted and there are no current plans to
promote its use for unrelated projects.
Quick overview:
* inspects AST rather than object tree
* again not a lot of information, the usual API docstrings, class inheritance
and module structure, but that's it
* rather heavy dependencies (depends on Twisted/Nevow (trunk version))
* written for Twisted, but quite nice output with other applications
Quick overview lists of the other tools
---------------------------------------
HappyDoc
~~~~~~~~
http://happydoc.sourceforge.net/
* dead
* inspects AST
* quite flexible, different output formats (HTML, XML, SGML, PDF)
* pluggable docstring parsers
Pudge
~~~~~
http://pudge.lesscode.org/
* immature, dead?
* builds docs from live object tree (I think?)
* supports ReST
* uses Kid templates
Endo
~~~~
https://svn.enthought.com/enthought/wiki/EndoHowTo
* inspects object tree (I think?)
* 'traits' aware (see https://svn.enthought.com/enthought/wiki/Traits)
* customizable HTML output with custom templating engine
* little documentation, seems like it's written for Enthought's own use
mostly
* heavy dependencies
.. [1] Epydoc doesn't seem to be developed anymore, either, but it's so
widely used it can not be ignored...
Questions, remarks, etc.
========================
For more information, questions, remarks, etc. see http://codespeak.net/py.
This website also contains links to mailing list and IRC channel.

57
py/doc/confrest.py
View File

@ -1,7 +1,6 @@
import py
from py.__.misc.rest import convert_rest_html, strip_html_header
from py.__.misc.difftime import worded_time
from py.__.apigen.linker import relpath
html = py.xml.html
@ -187,3 +186,59 @@ class Project:
page.contentspace.append(py.xml.raw(content))
outputpath.ensure().write(page.unicode().encode(encoding))
# XXX this function comes from apigen/linker.py, put it
# somewhere in py lib
import os
def relpath(p1, p2, sep=os.path.sep, back='..', normalize=True):
""" create a relative path from p1 to p2
sep is the seperator used for input and (depending
on the setting of 'normalize', see below) output
back is the string used to indicate the parent directory
when 'normalize' is True, any backslashes (\) in the path
will be replaced with forward slashes, resulting in a consistent
output on Windows and the rest of the world
paths to directories must end on a / (URL style)
"""
if normalize:
p1 = p1.replace(sep, '/')
p2 = p2.replace(sep, '/')
sep = '/'
# XXX would be cool to be able to do long filename
# expansion and drive
# letter fixes here, and such... iow: windows sucks :(
if (p1.startswith(sep) ^ p2.startswith(sep)):
raise ValueError("mixed absolute relative path: %r -> %r" %(p1, p2))
fromlist = p1.split(sep)
tolist = p2.split(sep)
# AA
# AA BB -> AA/BB
#
# AA BB
# AA CC -> CC
#
# AA BB
# AA -> ../AA
diffindex = 0
for x1, x2 in zip(fromlist, tolist):
if x1 != x2:
break
diffindex += 1
commonindex = diffindex - 1
fromlist_diff = fromlist[diffindex:]
tolist_diff = tolist[diffindex:]
if not fromlist_diff:
return sep.join(tolist[commonindex:])
backcount = len(fromlist_diff)
if tolist_diff:
return sep.join([back,]*(backcount-1) + tolist_diff)
return sep.join([back,]*(backcount) + tolist[commonindex:])

2
py/doc/index.txt
View File

@ -26,7 +26,6 @@ Main tools and API
`py lib scripts`_ describe the scripts contained in the ``py/bin`` directory.
`apigen`_: a new way to generate rich Python API documentation
support functionality
---------------------------------
@ -50,7 +49,6 @@ Background and Motivation information
.. _`py-dev at codespeak net`: http://codespeak.net/mailman/listinfo/py-dev
.. _`py.execnet`: execnet.html
.. _`py.magic.greenlet`: greenlet.html
.. _`apigen`: apigen.html
.. _`py.log`: log.html
.. _`py.io`: io.html
.. _`py.path`: path.html