test_ok1/py/apigen/htmlgen.py

745 lines
29 KiB
Python

import py
import os
import inspect
from py.__.apigen.layout import LayoutPage
from py.__.apigen.source import browser as source_browser
from py.__.apigen.source import html as source_html
from py.__.apigen.source import color as source_color
from py.__.apigen.tracer.description import is_private
from py.__.apigen.rest.genrest import split_of_last_part
from py.__.apigen.linker import relpath
sorted = py.builtin.sorted
html = py.xml.html
raw = py.xml.raw
def deindent(str, linesep='\n'):
""" de-indent string
can be used to de-indent Python docstrings, it de-indents the first
line to the side always, and determines the indentation of the rest
of the text by taking that of the least indented (filled) line
"""
lines = str.strip().split(linesep)
normalized = []
deindent = None
normalized.append(lines[0].strip())
# replace tabs with spaces, empty lines that contain spaces only, and
# find out what the smallest indentation is
for line in lines[1:]:
line = line.replace('\t', ' ' * 4)
stripped = line.strip()
if not stripped:
normalized.append('')
else:
rstripped = line.rstrip()
indent = len(rstripped) - len(stripped)
if deindent is None or indent < deindent:
deindent = indent
normalized.append(line)
ret = [normalized[0]]
for line in normalized[1:]:
if not line:
ret.append(line)
else:
ret.append(line[deindent:])
return '%s\n' % (linesep.join(ret),)
# HTML related stuff
class H(html):
class Content(html.div):
style = html.Style(margin_left='15em')
class Description(html.div):
pass
class NamespaceDescription(Description):
pass
class NamespaceItem(html.div):
pass
class NamespaceDef(html.h1):
pass
class ClassDescription(Description):
pass
class ClassDef(html.h1):
pass
class MethodDescription(Description):
pass
class MethodDef(html.h2):
pass
class FunctionDescription(Description):
pass
class FunctionDef(html.h2):
pass
class ParameterDescription(html.div):
pass
class Docstring(html.pre):
#style = html.Style(white_space='pre', min_height='3em')
pass
class Navigation(html.div):
style = html.Style(min_height='99%', float='left', margin_top='1.2em',
overflow='auto', width='15em', white_space='nowrap')
class NavigationItem(html.div):
pass
class BaseDescription(html.a):
pass
class SourceDef(html.div):
pass
class NonPythonSource(html.pre):
style = html.Style(margin_left='15em')
class DirList(html.div):
style = html.Style(margin_left='15em')
class DirListItem(html.div):
pass
class ValueDescList(html.ul):
def __init__(self, *args, **kwargs):
super(H.ValueDescList, self).__init__(*args, **kwargs)
class ValueDescItem(html.li):
pass
class CallStackDescription(Description):
pass
class CallStackItem(html.div):
class_ = 'callstackitem'
def get_param_htmldesc(linker, func):
""" get the html for the parameters of a function """
import inspect
# XXX copy and modify formatargspec to produce html
return H.em(inspect.formatargspec(*inspect.getargspec(func)))
def build_navitem_html(linker, name, linkid, indent, selected):
href = linker.get_lazyhref(linkid)
navitem = H.NavigationItem((indent * 2 * u'\xa0'), H.a(name, href=href))
if selected:
navitem.attr.class_ = 'selected'
return navitem
# some helper functionality
def source_dirs_files(fspath):
""" returns a tuple (dirs, files) for fspath
dirs are all the subdirs, files are the files which are interesting
in building source documentation for a Python code tree (basically all
normal files excluding .pyc and .pyo ones)
all files and dirs that have a name starting with . are considered
hidden
"""
dirs = []
files = []
for child in fspath.listdir():
if child.basename.startswith('.'):
continue
if child.check(dir=True):
dirs.append(child)
elif child.check(file=True):
if child.ext in ['.pyc', '.pyo']:
continue
files.append(child)
return sorted(dirs), sorted(files)
def create_namespace_tree(dotted_names):
""" creates a tree (in dict form) from a set of dotted names
"""
ret = {}
for dn in dotted_names:
path = dn.split('.')
for i in xrange(len(path)):
ns = '.'.join(path[:i])
itempath = '.'.join(path[:i + 1])
if ns not in ret:
ret[ns] = []
if itempath not in ret[ns]:
ret[ns].append(itempath)
return ret
def wrap_page(project, title, contentel, navel, outputpath, stylesheeturl,
scripturls):
page = LayoutPage(project, title, nav=navel, encoding='UTF-8',
stylesheeturl=stylesheeturl, scripturls=scripturls)
page.set_content(contentel)
here = py.magic.autopath().dirpath()
style = here.join(stylesheeturl.split('/')[-1]).read()
outputpath.join(stylesheeturl.split('/')[-1]).write(style)
for spath in scripturls:
sname = spath.split('/')[-1]
sdata = here.join(sname).read()
outputpath.join(sname).write(sdata)
return page
# the PageBuilder classes take care of producing the docs (using the stuff
# above)
class AbstractPageBuilder(object):
def write_page(self, title, reltargetpath, project, tag, nav):
targetpath = self.base.join(reltargetpath)
stylesheeturl = relpath('%s%s' % (targetpath.dirpath(), os.path.sep),
self.base.join('style.css').strpath)
scripturls = [relpath('%s%s' % (targetpath.dirpath(), os.path.sep),
self.base.join('api.js').strpath)]
page = wrap_page(project, title,
tag, nav, self.base, stylesheeturl, scripturls)
content = self.linker.call_withbase(reltargetpath, page.unicode)
targetpath.ensure()
targetpath.write(content.encode("utf8"))
class SourcePageBuilder(AbstractPageBuilder):
""" builds the html for a source docs page """
def __init__(self, base, linker, projroot):
self.base = base
self.linker = linker
self.projroot = projroot
def build_navigation(self, fspath):
nav = H.Navigation()
relpath = fspath.relto(self.projroot)
path = relpath.split(os.path.sep)
indent = 0
# build links to parents
if relpath != '':
for i in xrange(len(path)):
dirpath = os.path.sep.join(path[:i])
abspath = self.projroot.join(dirpath).strpath
if i == 0:
text = self.projroot.basename
else:
text = path[i-1]
nav.append(build_navitem_html(self.linker, text, abspath,
indent, False))
indent += 1
# build siblings or children and self
if fspath.check(dir=True):
# we're a dir, build ourselves and our children
dirpath = fspath
nav.append(build_navitem_html(self.linker, dirpath.basename,
dirpath.strpath, indent, True))
indent += 1
elif fspath.strpath == self.projroot.strpath:
dirpath = fspath
else:
# we're a file, build our parent's children only
dirpath = fspath.dirpath()
diritems, fileitems = source_dirs_files(dirpath)
for dir in diritems:
nav.append(build_navitem_html(self.linker, dir.basename,
dir.strpath, indent, False))
for file in fileitems:
selected = (fspath.check(file=True) and
file.basename == fspath.basename)
nav.append(build_navitem_html(self.linker, file.basename,
file.strpath, indent, selected))
return nav
re = py.std.re
_reg_body = re.compile(r'<body[^>]*>(.*)</body>', re.S)
def build_python_page(self, fspath):
mod = source_browser.parse_path(fspath)
# XXX let's cheat a bit here... there should be a different function
# using the linker, and returning a proper py.xml.html element,
# at some point
html = source_html.create_html(mod)
snippet = self._reg_body.search(html).group(1)
tag = H.SourceDef(raw(snippet))
nav = self.build_navigation(fspath)
return tag, nav
def build_dir_page(self, fspath):
tag = H.DirList()
dirs, files = source_dirs_files(fspath)
tag.append(H.h2('directories'))
for path in dirs:
tag.append(H.DirListItem(H.a(path.basename,
href=self.linker.get_lazyhref(str(path)))))
tag.append(H.h2('files'))
for path in files:
tag.append(H.DirListItem(H.a(path.basename,
href=self.linker.get_lazyhref(str(path)))))
nav = self.build_navigation(fspath)
return tag, nav
def build_nonpython_page(self, fspath):
try:
tag = H.NonPythonSource(unicode(fspath.read(), 'utf-8'))
except UnicodeError:
# XXX we should fix non-ascii support here!!
tag = H.NonPythonSource('no source available (binary file?)')
nav = self.build_navigation(fspath)
return tag, nav
def prepare_pages(self, base):
passed = []
for fspath in [base] + list(base.visit()):
if fspath.ext in ['.pyc', '.pyo']:
continue
relfspath = fspath.relto(base)
if relfspath.find('%s.' % (os.path.sep,)) > -1:
# skip hidden dirs and files
continue
elif fspath.check(dir=True):
if relfspath != '':
relfspath += os.path.sep
reloutputpath = 'source%s%sindex.html' % (os.path.sep,
relfspath)
else:
reloutputpath = "source%s%s.html" % (os.path.sep, relfspath)
reloutputpath = reloutputpath.replace(os.path.sep, '/')
outputpath = self.base.join(reloutputpath)
self.linker.set_link(str(fspath), reloutputpath)
passed.append((fspath, outputpath))
return passed
def build_pages(self, data, project, base):
""" build syntax-colored source views """
for fspath, outputpath in data:
if fspath.check(ext='.py'):
try:
tag, nav = self.build_python_page(fspath)
except (KeyboardInterrupt, SystemError):
raise
except: # XXX strange stuff going wrong at times... need to fix
exc, e, tb = py.std.sys.exc_info()
print '%s - %s' % (exc, e)
print
print ''.join(py.std.traceback.format_tb(tb))
print '-' * 79
del tb
tag, nav = self.build_nonpython_page(fspath)
elif fspath.check(dir=True):
tag, nav = self.build_dir_page(fspath)
else:
tag, nav = self.build_nonpython_page(fspath)
title = 'sources for %s' % (fspath.basename,)
reltargetpath = outputpath.relto(self.base).replace(os.path.sep, '/')
self.write_page(title, reltargetpath, project, tag, nav)
def enumerate_and_color(codelines, firstlineno, enc):
tokenizer = source_color.Tokenizer(source_color.PythonSchema)
colored = []
for i, line in enumerate(codelines):
try:
colored.append(H.span('%04s: ' % (i + firstlineno + 1)))
colored.append(source_html.prepare_line([line], tokenizer, enc))
colored.append('\n')
except py.error.ENOENT:
# error reading source code, giving up
colored = org
break
return colored
class ApiPageBuilder(AbstractPageBuilder):
""" builds the html for an api docs page """
def __init__(self, base, linker, dsa, projroot, namespace_tree):
self.base = base
self.linker = linker
self.dsa = dsa
self.projroot = projroot
self.projpath = py.path.local(projroot)
self.namespace_tree = namespace_tree
def build_callable_view(self, dotted_name):
""" build the html for a class method """
# XXX we may want to have seperate
func = self.dsa.get_obj(dotted_name)
docstring = func.__doc__
if docstring:
docstring = deindent(docstring)
localname = func.__name__
argdesc = get_param_htmldesc(self.linker, func)
valuedesc = self.build_callable_signature_description(dotted_name)
sourcefile = inspect.getsourcefile(func)
callable_source = self.dsa.get_function_source(dotted_name)
# i assume they're both either available or unavailable(XXX ?)
is_in_pkg = self.is_in_pkg(sourcefile)
if sourcefile and callable_source:
enc = source_html.get_module_encoding(sourcefile)
tokenizer = source_color.Tokenizer(source_color.PythonSchema)
firstlineno = func.func_code.co_firstlineno
org = callable_source.split('\n')
colored = enumerate_and_color(org, firstlineno, enc)
if is_in_pkg:
slink = H.a('source: %s' % (sourcefile,),
href=self.linker.get_lazyhref(sourcefile))
else:
slink = H.em('source: %s' % (sourcefile,))
csource = H.div(H.br(), slink, H.br(),
H.SourceDef(H.div(class_='code', *colored)))
else:
csource = H.SourceDef('could not get source file')
csdiv = H.div(style='display: none')
for cs, _ in self.dsa.get_function_callpoints(dotted_name):
csdiv.append(self.build_callsite(dotted_name, cs))
callstack = H.CallStackDescription(
H.a('show/hide call sites',
href='#',
onclick='showhideel(getnextsibling(this)); return false;'),
csdiv,
)
snippet = H.FunctionDescription(
H.FunctionDef('def %s' % (localname,), argdesc),
H.Docstring(docstring or '*no docstring available*'),
H.div(H.a('show/hide info',
href='#',
onclick=('showhideel(getnextsibling(this));'
'return false;')),
H.div(valuedesc, csource, callstack, style='display: none',
class_='funcinfo')),
)
return snippet
def build_class_view(self, dotted_name):
""" build the html for a class """
cls = self.dsa.get_obj(dotted_name)
# XXX is this a safe check?
try:
sourcefile = inspect.getsourcefile(cls)
except TypeError:
sourcelink = 'builtin file, no source available'
else:
if sourcefile is None:
sourcelink = H.div('no source available')
else:
if sourcefile[-1] in ['o', 'c']:
sourcefile = sourcefile[:-1]
sourcelink = H.div(H.a('view source',
href=self.linker.get_lazyhref(sourcefile)))
docstring = cls.__doc__
if docstring:
docstring = deindent(docstring)
methods = self.dsa.get_class_methods(dotted_name)
basehtml = []
bases = self.dsa.get_possible_base_classes(dotted_name)
for base in bases:
try:
obj = self.dsa.get_obj(base.name)
except KeyError:
basehtml.append(base.name)
else:
href = self.linker.get_lazyhref(base.name)
basehtml.append(H.BaseDescription(base.name, href=href))
basehtml.append(',')
if basehtml:
basehtml.pop()
basehtml.append('):')
if not hasattr(cls, '__name__'):
clsname = 'instance of %s' % (cls.__class__.__name__,)
else:
clsname = cls.__name__
snippet = H.ClassDescription(
# XXX bases HTML
H.ClassDef('%s(' % (clsname,), *basehtml),
H.Docstring(docstring or '*no docstring available*'),
sourcelink,
)
if methods:
snippet.append(H.h2('methods:'))
for method in methods:
snippet += self.build_callable_view('%s.%s' % (dotted_name,
method))
# XXX properties
return snippet
def build_namespace_view(self, namespace_dotted_name, item_dotted_names):
""" build the html for a namespace (module) """
try:
obj = self.dsa.get_obj(namespace_dotted_name)
except KeyError:
docstring = None
else:
docstring = obj.__doc__
if docstring:
docstring = deindent(docstring)
snippet = H.NamespaceDescription(
H.NamespaceDef(namespace_dotted_name),
H.Docstring(docstring or '*no docstring available*')
)
for dotted_name in sorted(item_dotted_names):
itemname = dotted_name.split('.')[-1]
if is_private(itemname):
continue
snippet.append(
H.NamespaceItem(
H.a(itemname,
href=self.linker.get_lazyhref(dotted_name)
)
)
)
return snippet
def prepare_class_pages(self, classes_dotted_names):
passed = []
for dotted_name in sorted(classes_dotted_names):
parent_dotted_name, _ = split_of_last_part(dotted_name)
try:
sibling_dotted_names = self.namespace_tree[parent_dotted_name]
except KeyError:
# no siblings (built-in module or sth)
sibling_dotted_names = []
tag = H.Content(self.build_class_view(dotted_name))
nav = self.build_navigation(dotted_name, False)
reltargetpath = "api/%s.html" % (dotted_name,)
self.linker.set_link(dotted_name, reltargetpath)
passed.append((dotted_name, tag, nav, reltargetpath))
return passed
def build_class_pages(self, data, project):
""" build the full api pages for a set of classes """
for dotted_name, tag, nav, reltargetpath in data:
title = 'api documentation for %s' % (dotted_name,)
self.write_page(title, reltargetpath, project, tag, nav)
def prepare_method_pages(self, method_dotted_names):
# XXX note that even though these pages are still built, there's no nav
# pointing to them anymore...
passed = []
for dotted_name in sorted(method_dotted_names):
parent_dotted_name, _ = split_of_last_part(dotted_name)
module_dotted_name, _ = split_of_last_part(parent_dotted_name)
sibling_dotted_names = self.namespace_tree[module_dotted_name]
tag = self.build_callable_view(dotted_name)
nav = self.build_navigation(dotted_name, False)
reltargetpath = "api/%s.html" % (dotted_name,)
self.linker.set_link(dotted_name, reltargetpath)
passed.append((dotted_name, tag, nav, reltargetpath))
return passed
def build_method_pages(self, data, project):
for dotted_name, tag, nav, reltargetpath in data:
title = 'api documentation for %s' % (dotted_name,)
self.write_page(title, reltargetpath, project, tag, nav)
def prepare_function_pages(self, method_dotted_names):
passed = []
for dotted_name in sorted(method_dotted_names):
# XXX should we create a build_function_view instead?
parent_dotted_name, _ = split_of_last_part(dotted_name)
sibling_dotted_names = self.namespace_tree[parent_dotted_name]
tag = H.Content(self.build_callable_view(dotted_name))
nav = self.build_navigation(dotted_name, False)
reltargetpath = "api/%s.html" % (dotted_name,)
self.linker.set_link(dotted_name, reltargetpath)
passed.append((dotted_name, tag, nav, reltargetpath))
return passed
def build_function_pages(self, data, project):
for dotted_name, tag, nav, reltargetpath in data:
title = 'api documentation for %s' % (dotted_name,)
self.write_page(title, reltargetpath, project, tag, nav)
def prepare_namespace_pages(self):
passed = []
module_name = self.dsa.get_module_name().split('/')[-1]
names = self.namespace_tree.keys()
names.sort()
function_names = self.dsa.get_function_names()
class_names = self.dsa.get_class_names()
for dotted_name in sorted(names):
if dotted_name in function_names or dotted_name in class_names:
continue
subitem_dotted_names = self.namespace_tree[dotted_name]
tag = H.Content(self.build_namespace_view(dotted_name,
subitem_dotted_names))
nav = self.build_navigation(dotted_name, True)
if dotted_name == '':
reltargetpath = 'api/index.html'
else:
reltargetpath = 'api/%s.html' % (dotted_name,)
self.linker.set_link(dotted_name, reltargetpath)
passed.append((dotted_name, tag, nav, reltargetpath))
return passed
def build_namespace_pages(self, data, project):
for dotted_name, tag, nav, reltargetpath in data:
if dotted_name == '':
dotted_name = self.dsa.get_module_name().split('/')[-1]
title = 'index of %s namespace' % (dotted_name,)
self.write_page(title, reltargetpath, project, tag, nav)
def build_navigation(self, dotted_name, build_children=True):
navitems = []
# top namespace, index.html
module_name = self.dsa.get_module_name().split('/')[-1]
navitems.append(build_navitem_html(self.linker, module_name, '', 0,
True))
def build_nav_level(dotted_name, depth=1):
navitems = []
path = dotted_name.split('.')[:depth]
siblings = self.namespace_tree.get('.'.join(path[:-1]))
for dn in sorted(siblings):
selected = dn == '.'.join(path)
sibpath = dn.split('.')
sibname = sibpath[-1]
if is_private(sibname):
continue
navitems.append(build_navitem_html(self.linker, sibname,
dn, depth, selected))
if selected:
lastlevel = dn.count('.') == dotted_name.count('.')
if not lastlevel:
navitems += build_nav_level(dotted_name, depth+1)
elif lastlevel and build_children:
# XXX hack
navitems += build_nav_level('%s.' % (dotted_name,),
depth+1)
return navitems
navitems += build_nav_level(dotted_name)
return H.Navigation(*navitems)
navitems = []
# top namespace, index.html
module_name = self.dsa.get_module_name().split('/')[-1]
navitems.append(build_navitem_html(self.linker, module_name, '', 0,
(selection == '')))
indent = 1
path = dotted_name.split('.')
if dotted_name != '':
# build html for each item in path to dotted_name item
for i in xrange(len(path)):
name = path[i]
item_dotted_name = '.'.join(path[:i+1])
selected = (selection == item_dotted_name)
navitems.append(build_navitem_html(self.linker, name,
item_dotted_name, indent,
selected))
indent += 1
# build sub items of dotted_name item
for item_dotted_name in py.builtin.sorted(item_dotted_names):
itemname = item_dotted_name.split('.')[-1]
if is_private(itemname):
continue
selected = (item_dotted_name == selection)
navitems.append(build_navitem_html(self.linker, itemname,
item_dotted_name, indent,
selected))
return H.Navigation(*navitems)
def build_callable_signature_description(self, dotted_name):
args, retval = self.dsa.get_function_signature(dotted_name)
valuedesc = H.ValueDescList()
for name, _type in args:
valuedesc.append(self.build_sig_value_description(name, _type))
if retval:
retval = self.process_type_link(retval)
ret = H.div(H.div('arguments:'), valuedesc, H.div('return value:'),
retval or 'None')
return ret
def build_sig_value_description(self, name, _type):
l = self.process_type_link(_type)
items = []
next = "%s: " % name
for item in l:
if isinstance(item, str):
next += item
else:
if next:
items.append(next)
next = ""
items.append(item)
if next:
items.append(next)
return H.ValueDescItem(*items)
def process_type_link(self, _type):
# now we do simple type dispatching and provide a link in this case
lst = []
data = self.dsa.get_type_desc(_type)
if not data:
for i in _type.striter():
if isinstance(i, str):
lst.append(i)
else:
lst += self.process_type_link(i)
return lst
name, _desc_type, is_degenerated = data
if not is_degenerated:
linktarget = self.linker.get_lazyhref(name)
lst.append(H.a(str(_type), href=linktarget))
else:
raise IOError('do not think we ever get here?')
# we should provide here some way of linking to sourcegen directly
lst.append(name)
return lst
def is_in_pkg(self, sourcefile):
return py.path.local(sourcefile).relto(self.projpath)
def build_callsite(self, functionname, call_site):
tbtag = self.gen_traceback(functionname, reversed(call_site))
tag = H.CallStackItem(
H.a("show/hide stack trace %s - line %s" % (
call_site[0].filename, call_site[0].lineno + 1),
href='#',
onclick="showhideel(getnextsibling(this)); return false;"),
H.div(tbtag, style='display: none', class_='callstackitem'),
)
return tag
_reg_source = py.std.re.compile(r'([^>]*)<(.*)>')
def gen_traceback(self, funcname, call_site):
tbdiv = H.div()
for frame in call_site:
lineno = frame.lineno - frame.firstlineno
source = frame.source
sourcefile = frame.filename
tokenizer = source_color.Tokenizer(source_color.PythonSchema)
mangled = []
for i, sline in enumerate(str(source).split('\n')):
if i == lineno:
l = '-> %s' % (sline,)
else:
l = ' %s' % (sline,)
mangled.append(l)
if sourcefile:
linktext = '%s - line %s' % (sourcefile, frame.lineno + 1)
# skip py.code.Source objects and source files outside of the
# package
is_code_source = self._reg_source.match(sourcefile)
if (not is_code_source and self.is_in_pkg(sourcefile)):
enc = source_html.get_module_encoding(sourcefile)
href = self.linker.get_lazyhref(sourcefile)
sourcelink = H.a(linktext, href=href)
else:
enc = 'latin-1'
sourcelink = H.div(linktext)
colored = enumerate_and_color(mangled, frame.firstlineno, enc)
else:
sourcelink = H.div('source unknown')
tbdiv.append(sourcelink)
tbdiv.append(H.div(class_='code', *colored))
return tbdiv