[svn r37602] Fixed support for files ending on a comment rather than a newline, fixed some

unicode() call so it's not done on objects that are already unicode.

--HG--
branch : trunk

py/apigen/source/color.py

@@ -4,7 +4,7 @@ import re
 
 class PythonSchema(object):
     """ contains information for syntax coloring """
-    comment = [('#', '\n')]
+    comment = [('#', '\n'), ('#', '$')]
     multiline_string = ['"""', "'''"]
     string = ['"""', "'''", '"', "'"]
     # XXX not complete
@@ -125,7 +125,6 @@ class Tokenizer(object):
                 break
         return data, token
 
-
     def _check_comments(self, data):
         # fortunately we don't have to deal with multi-line comments
         token = None

py/apigen/source/html.py

@@ -46,7 +46,10 @@ def prepare_line(text, tokenizer, encoding):
         if type(item) in [str, unicode]:
             tokens = tokenizer.tokenize(item)
             for t in tokens:
+                if not isinstance(t.data, unicode):
                     data = unicode(t.data, encoding)
+                else:
+                    data = t.data
                 if t.type in ['keyword', 'alt_keyword', 'number',
                               'string', 'comment']:
                     ret.append(html.span(data, class_=t.type))

py/apigen/source/testing/test_color.py

@@ -45,8 +45,12 @@ class TestTokenizer(object):
         assert self.tokens('foo # bar\n') == [Token('foo', type='word'),
                                               Token(' ', type='whitespace'),
                                               Token('# bar\n', type='comment')]
+        assert self.tokens("# foo 'bar\n") == [Token("# foo 'bar\n",
+                                                     type='comment')]
+        assert self.tokens('# foo') == [Token('# foo', type='comment')]
 
     def test_string_simple(self):
+        assert self.tokens('""') == [Token('""', type='string')]
         assert self.tokens('"foo"') == [Token('"foo"', type='string')]
         assert self.tokens('"foo"\'bar\'') == [Token('"foo"', type='string'),
                                                Token("'bar'", type='string')]