[svn r37913] Some more cleanups in HTML generation, fixed support for docstrings in

namespaces, in order to do this I had to change the way objects are retrieved:
instead of getting them from the DSA I now walk the package tree, small change
in apigen.py: to allow re-using the get_documentable_items function I split it
up in a generic and a specific part.

--HG--
branch : trunk

py/apigen/apigen.py

@@ -12,13 +12,23 @@ from py.__.apigen import linker
 from py.__.apigen import project
 from py.__.apigen.tracer.docstorage import pkg_to_dict
 
-def get_documentable_items(pkgdir):
+def get_documentable_items_pkgdir(pkgdir):
+    """ get all documentable items from an initpkg pkgdir
+    
+        this is a generic implementation, import as 'get_documentable_items'
+        from your module when using initpkg to get all public stuff in the
+        package documented
+    """
     sys.path.insert(0, str(pkgdir.dirpath()))
     rootmod = __import__(pkgdir.basename)
     d = pkg_to_dict(rootmod)
+    return pkgdir.basename, d
+
+def get_documentable_items(pkgdir):
+    pkgname, pkgdict = get_documentable_items_pkgdir(pkgdir)
     from py.__.execnet.channel import Channel
-    #d['execnet.Channel'] = Channel  # XXX doesn't work 
+    # pkgdict['execnet.Channel'] = Channel  # XXX doesn't work 
-    return 'py', d
+    return pkgname, pkgdict
 
 def build(pkgdir, dsa, capture):
     l = linker.Linker()

py/apigen/html.py

@@ -82,16 +82,25 @@ class H(html):
                 link, H.div(class_='code', *sourceels))
     
     class SourceDef(html.div):
-        pass
+        def __init__(self, *sourceels):
+            super(H.SourceDef, self).__init__(
+                H.div(class_='code', *sourceels))
 
     class NonPythonSource(html.pre):
         pass # style = html.Style(margin_left='15em')
 
     class DirList(html.div):
-        pass # style = html.Style(margin_left='15em')
+        def __init__(self, dirs, files):
+            dirs = [H.DirListItem(text, href) for (text, href) in dirs]
+            files = [H.DirListItem(text, href) for (text, href) in files]
+            super(H.DirList, self).__init__(
+                H.h2('directories'), dirs,
+                H.h2('files'), files,
+            )
 
     class DirListItem(html.div):
-        pass
+        def __init__(self, text, href):
+            super(H.DirListItem, self).__init__(H.a(text, href=href))
 
     class ValueDescList(html.ul):
         def __init__(self, *args, **kwargs):

py/apigen/htmlgen.py

@@ -14,6 +14,9 @@ sorted = py.builtin.sorted
 html = py.xml.html
 raw = py.xml.raw
 
+def is_navigateable(name):
+    return (not is_private(name) and name != '__doc__')
+
 def deindent(str, linesep='\n'):
     """ de-indent string
 
@@ -46,6 +49,16 @@ def deindent(str, linesep='\n'):
             ret.append(line[deindent:])
     return '%s\n' % (linesep.join(ret),)
 
+def get_linesep(s, default='\n'):
+    """ return the line seperator of a string
+
+        returns 'default' if no seperator can be found
+    """
+    for sep in ('\r\n', '\r', '\n'):
+        if sep in s:
+            return sep
+    return default
+
 def get_param_htmldesc(linker, func):
     """ get the html for the parameters of a function """
     import inspect
@@ -161,27 +174,21 @@ class SourcePageBuilder(AbstractPageBuilder):
     re = py.std.re
     _reg_body = re.compile(r'<body[^>]*>(.*)</body>', re.S)
     def build_python_page(self, fspath):
-        mod = source_browser.parse_path(fspath)
+        # XXX two reads of the same file here... not very bad (disk caches
-        # XXX let's cheat a bit here... there should be a different function 
+        # and such) but also not very nice...
-        # using the linker, and returning a proper py.xml.html element,
+        enc = source_html.get_module_encoding(fspath.strpath)
-        # at some point
+        source = fspath.read()
-        html = source_html.create_html(mod)
+        sep = get_linesep(source)
-        snippet = self._reg_body.search(html).group(1)
+        colored = enumerate_and_color(source.split(sep), 0, enc)
-        tag = H.SourceDef(raw(snippet))
+        tag = H.SourceDef(*colored)
         nav = self.build_navigation(fspath)
         return tag, nav
 
     def build_dir_page(self, fspath):
-        tag = H.DirList()
         dirs, files = source_dirs_files(fspath)
-        tag.append(H.h2('directories'))
+        dirs = [(p.basename, self.linker.get_lazyhref(str(p))) for p in dirs]
-        for path in dirs:
+        files = [(p.basename, self.linker.get_lazyhref(str(p))) for p in files]
-            tag.append(H.DirListItem(H.a(path.basename,
+        tag = H.DirList(dirs, files)
-                            href=self.linker.get_lazyhref(str(path)))))
-        tag.append(H.h2('files'))
-        for path in files:
-            tag.append(H.DirListItem(H.a(path.basename,
-                            href=self.linker.get_lazyhref(str(path)))))
         nav = self.build_navigation(fspath)
         return tag, nav
 
@@ -238,7 +245,8 @@ class SourcePageBuilder(AbstractPageBuilder):
             else:
                 tag, nav = self.build_nonpython_page(fspath)
             title = 'sources for %s' % (fspath.basename,)
-            reltargetpath = outputpath.relto(self.base).replace(os.path.sep, '/')
+            reltargetpath = outputpath.relto(self.base).replace(os.path.sep,
+                                                                '/')
             self.write_page(title, reltargetpath, project, tag, nav)
 
 def enumerate_and_color(codelines, firstlineno, enc):
@@ -255,6 +263,20 @@ def enumerate_and_color(codelines, firstlineno, enc):
             break
     return colored
 
+def get_obj(pkg, dotted_name):
+    full_dotted_name = '%s.%s' % (pkg.__name__, dotted_name)
+    if dotted_name == '':
+        return pkg
+    path = dotted_name.split('.')
+    ret = pkg
+    for item in path:
+        marker = []
+        ret = getattr(ret, item, marker)
+        if ret is marker:
+            raise NameError('can not access %s in %s' % (item,
+                                                         full_dotted_name))
+    return ret
+
 class ApiPageBuilder(AbstractPageBuilder):
     """ builds the html for an api docs page """
     def __init__(self, base, linker, dsa, projroot, namespace_tree,
@@ -267,10 +289,13 @@ class ApiPageBuilder(AbstractPageBuilder):
         self.namespace_tree = namespace_tree
         self.capture = capture
 
+        pkgname = self.dsa.get_module_name().split('/')[-1]
+        self.pkg = __import__(pkgname)
+
     def build_callable_view(self, dotted_name):
         """ build the html for a class method """
         # XXX we may want to have seperate
-        func = self.dsa.get_obj(dotted_name)
+        func = get_obj(self.pkg, dotted_name)
         docstring = func.__doc__ 
         if docstring:
             docstring = deindent(docstring)
@@ -289,7 +314,8 @@ class ApiPageBuilder(AbstractPageBuilder):
             enc = source_html.get_module_encoding(sourcefile)
             tokenizer = source_color.Tokenizer(source_color.PythonSchema)
             firstlineno = func.func_code.co_firstlineno
-            org = callable_source.split('\n')
+            sep = get_linesep(callable_source)
+            org = callable_source.split(sep)
             colored = enumerate_and_color(org, firstlineno, enc)
             text = 'source: %s' % (sourcefile,)
             if is_in_pkg:
@@ -307,7 +333,7 @@ class ApiPageBuilder(AbstractPageBuilder):
 
     def build_class_view(self, dotted_name):
         """ build the html for a class """
-        cls = self.dsa.get_obj(dotted_name)
+        cls = get_obj(self.pkg, dotted_name)
         # XXX is this a safe check?
         try:
             sourcefile = inspect.getsourcefile(cls)
@@ -360,21 +386,15 @@ class ApiPageBuilder(AbstractPageBuilder):
 
     def build_namespace_view(self, namespace_dotted_name, item_dotted_names):
         """ build the html for a namespace (module) """
-        try:
+        obj = get_obj(self.pkg, namespace_dotted_name)
-            obj = self.dsa.get_obj(namespace_dotted_name)
-        except KeyError:
-            docstring = None
-        else:
         docstring = obj.__doc__
-            if docstring:
-                docstring = deindent(docstring)
         snippet = H.NamespaceDescription(
             H.NamespaceDef(namespace_dotted_name),
             H.Docstring(docstring or '*no docstring available*')
         )
         for dotted_name in sorted(item_dotted_names):
             itemname = dotted_name.split('.')[-1]
-            if is_private(itemname):
+            if not is_navigateable(itemname):
                 continue
             snippet.append(
                 H.NamespaceItem(
@@ -490,7 +510,7 @@ class ApiPageBuilder(AbstractPageBuilder):
                 selected = dn == '.'.join(path)
                 sibpath = dn.split('.')
                 sibname = sibpath[-1]
-                if is_private(sibname):
+                if not is_navigateable(sibname):
                     continue
                 navitems.append(H.NavigationItem(self.linker, dn, sibname,
                                                  depth, selected))
@@ -560,7 +580,6 @@ class ApiPageBuilder(AbstractPageBuilder):
         return py.path.local(sourcefile).relto(self.projpath)
 
     def build_callsite(self, functionname, call_site):
-        print 'building callsite for', functionname
         tbtag = self.gen_traceback(functionname, reversed(call_site))
         return H.CallStackItem(call_site[0].filename, call_site[0].lineno + 1,
                                tbtag)
@@ -575,7 +594,10 @@ class ApiPageBuilder(AbstractPageBuilder):
 
             tokenizer = source_color.Tokenizer(source_color.PythonSchema)
             mangled = []
-            for i, sline in enumerate(str(source).split('\n')):
+
+            source = str(source)
+            sep = get_linesep(source)
+            for i, sline in enumerate(source.split(sep)):
                 if i == lineno:
                     l = '-> %s' % (sline,)
                 else:

py/apigen/testing/test_apigen_example.py

@@ -116,7 +116,7 @@ class AbstractBuilderTest(object):
         self.namespace_tree = namespace_tree
         self.apb = ApiPageBuilder(base, linker, self.dsa,
                                   self.fs_root.join(self.pkg_name),
-                                  namespace_tree)
+                                  namespace_tree, 'root docstring')
         self.spb = SourcePageBuilder(base, linker,
                                      self.fs_root.join(self.pkg_name))
 
@@ -130,7 +130,7 @@ class TestApiPageBuilder(AbstractBuilderTest):
         pkg.main.sub.func(pkg.main.SomeClass(10))
         t.end_tracing()
         apb = ApiPageBuilder(self.base, self.linker, dsa, self.fs_root,
-                             self.namespace_tree)
+                             self.namespace_tree, 'root docstring')
         snippet = apb.build_callable_view('main.sub.func')
         html = snippet.unicode()
         print html
@@ -371,8 +371,7 @@ class TestSourcePageBuilder(AbstractBuilderTest):
         assert funcsource.check(file=True)
         html = funcsource.read()
         print html
-        assert ('<span class="alt_keyword">def</span> '
+        assert ('<span class="alt_keyword">def</span> func(arg1)') in html
-                '<a href="#func" name="func">func</a>(arg1):') in html
 
     def test_build_navigation_root(self):
         self.spb.prepare_pages(self.fs_root)

py/apigen/testing/test_apigen_functional.py

@@ -42,15 +42,24 @@ def setup_fs_project(name):
             return 'quux'
     """))
     temp.ensure("pak/__init__.py").write(py.code.Source("""\
+        '''pkg docstring'''
         from py.initpkg import initpkg
-        initpkg(__name__, exportdefs = {
+        initpkg(__name__,
-            'main.sub.func': ("./func.py", "func"),
+                long_description=globals()['__doc__'],
+                exportdefs={'main.sub.func': ("./func.py", "func"),
                             'main.func': ("./func.py", "func_2"),
-            'main.SomeTestClass': ('./sometestclass.py', 'SomeTestClass'),
+                            'main.SomeTestClass': ('./sometestclass.py',
+                                                   'SomeTestClass'),
                             'main.SomeTestSubClass': ('./sometestsubclass.py',
                                                       'SomeTestSubClass'),
-            'somenamespace': ('./somenamespace.py', '*'),
+                            'somenamespace': ('./somenamespace.py', '*')})
-        })
+    """))
+    temp.ensure('apigen.py').write(py.code.Source("""\
+        import py
+        py.std.sys.path.insert(0,
+            py.magic.autopath().dirpath().dirpath().dirpath().strpath)
+        from py.__.apigen.apigen import build, \
+            get_documentable_items_pkgdir as get_documentable_items
     """))
     temp.ensure('pak/test/test_pak.py').write(py.code.Source("""\
         import py
@@ -88,7 +97,7 @@ def test_get_documentable_items():
     fs_root, package_name = setup_fs_project('test_get_documentable_items')
     pkgname, documentable = apigen.get_documentable_items(
                                                fs_root.join(package_name))
-    assert pkgname == 'py'
+    assert pkgname == 'pak'
     assert sorted(documentable.keys()) ==  [
         'main.SomeTestClass', 'main.SomeTestSubClass', 'main.func',
         'main.sub.func', 'somenamespace.baz', 'somenamespace.foo']
@@ -101,14 +110,14 @@ def test_apigen_functional():
     pydir = py.magic.autopath().dirpath().dirpath().dirpath()
     pakdir = fs_root.join('pak')
     if py.std.sys.platform == 'win32':
-        cmd = 'set APIGEN_TARGET=%s && python "%s/bin/py.test"' % (tempdir,
+        cmd = ('set APIGEN_TARGET=%s && set PYTHONPATH=%s && '
-                                                                   pydir)
+               'python "%s/bin/py.test"') % (tempdir, fs_root, pydir)
     else:
-        cmd = 'APIGEN_TARGET="%s" "%s/bin/py.test"' % (tempdir, pydir)
+        cmd = ('APIGEN_TARGET="%s" PYTHONPATH="%s" '
+               '"%s/bin/py.test"') % (tempdir, fs_root, pydir)
     try:
         output = py.process.cmdexec('%s --apigen="%s/apigen.py" "%s"' % (
-                                        cmd, pydir.join('apigen'),
+                                        cmd, fs_root, pakdir))
-                                        pakdir))
     except py.error.Error, e:
         print e.out
         raise
@@ -130,6 +139,10 @@ def test_apigen_functional():
     assert namespace_api.check(file=True)
     html = namespace_api.read()
     assert '<a href="main.SomeTestClass.html">SomeTestClass</a>' in html
+    index = apidir.join('index.html')
+    assert index.check(file=True)
+    html = index.read()
+    assert 'pkg docstring' in html
 
     sourcedir = tempdir.join('source')
     assert sourcedir.check(dir=True)

py/apigen/todo-apigen.txt

@@ -10,7 +10,7 @@
   viewed.  method views (when navigating there through
   the class view) should also have the source there 
 
-  DONE I guess (todo: add syntax coloring)
+  DONE I think
 
 * have class-level attributes be displayed