Cleaned up the template debug implementation.

This patch does three major things:

* Merges the django.template.debug implementation into django.template.base.

* Simplifies the debug implementation.

  The old implementation copied debug information to every token and node.
  The django_template_source attribute was set in multiple places, some
  quite hacky, like django.template.defaulttags.ForNode.

  Debug information is now annotated in two high-level places:

  * Template.compile_nodelist for errors during parsing
  * Node.render_annotated for errors during rendering

  These were chosen because they have access to the template and context
  as well as to all exceptions that happen during either the parse or
  render phase.

* Moves the contextual line traceback information creation from
  django.views.debug into django.template.base.Template. The debug views now
  only deal with the presentation of the debug information.

django/template/base.py

@@ -69,7 +69,7 @@ from django.utils.encoding import (
     force_str, force_text, python_2_unicode_compatible,
 )
 from django.utils.formats import localize
-from django.utils.html import conditional_escape
+from django.utils.html import conditional_escape, escape
 from django.utils.itercompat import is_iterable
 from django.utils.module_loading import module_has_submodule
 from django.utils.safestring import (
@@ -187,12 +187,13 @@ class Template(object):
         if engine is None:
             from .engine import Engine
             engine = Engine.get_default()
-        if engine.debug and origin is None:
+        if origin is None:
             origin = StringOrigin(template_string)
-        self.nodelist = engine.compile_string(template_string, origin)
         self.name = name
         self.origin = origin
         self.engine = engine
+        self.source = template_string
+        self.nodelist = self.compile_nodelist()
 
     def __iter__(self):
         for node in self.nodelist:
@@ -214,13 +215,139 @@ class Template(object):
         finally:
             context.render_context.pop()
 
+    def compile_nodelist(self):
+        """
+        Parse and compile the template source into a nodelist. If debug
+        is True and an exception occurs during parsing, the exception is
+        is annotated with contextual line information where it occurred in the
+        template source.
+        """
+        if self.engine.debug:
+            lexer = DebugLexer(self.source)
+        else:
+            lexer = Lexer(self.source)
+
+        tokens = lexer.tokenize()
+        parser = Parser(tokens)
+
+        try:
+            return parser.parse()
+        except Exception as e:
+            if self.engine.debug:
+                e.template_debug = self.get_exception_info(e, e.token)
+            raise
+
+    def get_exception_info(self, exception, token):
+        """
+        Return a dictionary containing contextual line information of where
+        the exception occurred in the template. The following information is
+        provided:
+
+        message
+            The message of the exception raised.
+
+        source_lines
+            The lines before, after, and including the line the exception
+            occurred on.
+
+        line
+            The line number the exception occurred on.
+
+        before, during, after
+            The line the exception occurred on split into three parts:
+            1. The content before the token that raised the error.
+            2. The token that raised the error.
+            3. The content after the token that raised the error.
+
+        total
+            The number of lines in source_lines.
+
+        top
+            The line number where source_lines starts.
+
+        bottom
+            The line number where source_lines ends.
+
+        start
+            The start position of the token in the template source.
+
+        end
+            The end position of the token in the template source.
+        """
+        start, end = token.position
+        context_lines = 10
+        line = 0
+        upto = 0
+        source_lines = []
+        before = during = after = ""
+        for num, next in enumerate(linebreak_iter(self.source)):
+            if start >= upto and end <= next:
+                line = num
+                before = escape(self.source[upto:start])
+                during = escape(self.source[start:end])
+                after = escape(self.source[end:next])
+            source_lines.append((num, escape(self.source[upto:next])))
+            upto = next
+        total = len(source_lines)
+
+        top = max(1, line - context_lines)
+        bottom = min(total, line + 1 + context_lines)
+
+        # In some rare cases exc_value.args can be empty or an invalid
+        # unicode string.
+        try:
+            message = force_text(exception.args[0])
+        except (IndexError, UnicodeDecodeError):
+            message = '(Could not get exception message)'
+
+        return {
+            'message': message,
+            'source_lines': source_lines[top:bottom],
+            'before': before,
+            'during': during,
+            'after': after,
+            'top': top,
+            'bottom': bottom,
+            'total': total,
+            'line': line,
+            'name': self.origin.name,
+            'start': start,
+            'end': end,
+        }
+
+
+def linebreak_iter(template_source):
+    yield 0
+    p = template_source.find('\n')
+    while p >= 0:
+        yield p + 1
+        p = template_source.find('\n', p + 1)
+    yield len(template_source) + 1
+
 
 class Token(object):
-    def __init__(self, token_type, contents):
+    def __init__(self, token_type, contents, position=None, lineno=None):
-        # token_type must be TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK or
+        """
-        # TOKEN_COMMENT.
+        A token representing a string from the template.
+
+        token_type
+            One of TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, or TOKEN_COMMENT.
+
+        contents
+            The token source string.
+
+        position
+            An optional tuple containing the start and end index of the token
+            in the template source. This is used for traceback information
+            when debug is on.
+
+        lineno
+            The line number the token appears on in the template source.
+            This is used for traceback information and gettext files.
+        """
         self.token_type, self.contents = token_type, contents
-        self.lineno = None
+        self.lineno = lineno
+        self.position = position
 
     def __str__(self):
         token_name = TOKEN_MAPPING[self.token_type]
@@ -244,10 +371,8 @@ class Token(object):
 
 
 class Lexer(object):
-    def __init__(self, template_string, origin):
+    def __init__(self, template_string):
         self.template_string = template_string
-        self.origin = origin
-        self.lineno = 1
         self.verbatim = False
 
     def tokenize(self):
@@ -255,14 +380,16 @@ class Lexer(object):
         Return a list of tokens from a given template_string.
         """
         in_tag = False
+        lineno = 1
         result = []
         for bit in tag_re.split(self.template_string):
             if bit:
-                result.append(self.create_token(bit, in_tag))
+                result.append(self.create_token(bit, None, lineno, in_tag))
             in_tag = not in_tag
+            lineno += bit.count('\n')
         return result
 
-    def create_token(self, token_string, in_tag):
+    def create_token(self, token_string, position, lineno, in_tag):
         """
         Convert the given token string into a new Token object and return it.
         If in_tag is True, we are processing something that matched a tag,
@@ -278,35 +405,69 @@ class Lexer(object):
                 self.verbatim = False
         if in_tag and not self.verbatim:
             if token_string.startswith(VARIABLE_TAG_START):
-                token = Token(TOKEN_VAR, token_string[2:-2].strip())
+                token = Token(TOKEN_VAR, token_string[2:-2].strip(), position, lineno)
             elif token_string.startswith(BLOCK_TAG_START):
                 if block_content[:9] in ('verbatim', 'verbatim '):
                     self.verbatim = 'end%s' % block_content
-                token = Token(TOKEN_BLOCK, block_content)
+                token = Token(TOKEN_BLOCK, block_content, position, lineno)
             elif token_string.startswith(COMMENT_TAG_START):
                 content = ''
                 if token_string.find(TRANSLATOR_COMMENT_MARK):
                     content = token_string[2:-2].strip()
-                token = Token(TOKEN_COMMENT, content)
+                token = Token(TOKEN_COMMENT, content, position, lineno)
         else:
-            token = Token(TOKEN_TEXT, token_string)
+            token = Token(TOKEN_TEXT, token_string, position, lineno)
-        token.lineno = self.lineno
-        self.lineno += token_string.count('\n')
         return token
 
 
+class DebugLexer(Lexer):
+    def tokenize(self):
+        """
+        Split a template string into tokens and annotates each token with its
+        start and end position in the source. This is slower than the default
+        lexer so we only use it when debug is True.
+        """
+        lineno = 1
+        result = []
+        upto = 0
+        for match in tag_re.finditer(self.template_string):
+            start, end = match.span()
+            if start > upto:
+                token_string = self.template_string[upto:start]
+                result.append(self.create_token(token_string, (upto, start), lineno, in_tag=False))
+                lineno += token_string.count('\n')
+                upto = start
+            token_string = self.template_string[start:end]
+            result.append(self.create_token(token_string, (start, end), lineno, in_tag=True))
+            lineno += token_string.count('\n')
+            upto = end
+        last_bit = self.template_string[upto:]
+        if last_bit:
+            result.append(self.create_token(last_bit, (upto, upto + len(last_bit)), lineno, in_tag=False))
+        return result
+
+
 class Parser(object):
     def __init__(self, tokens):
         self.tokens = tokens
         self.tags = {}
         self.filters = {}
+        self.command_stack = []
         for lib in builtins:
             self.add_library(lib)
 
     def parse(self, parse_until=None):
+        """
+        Iterate through the parser tokens and compils each one into a node.
+
+        If parse_until is provided, parsing will stop once one of the
+        specified tokens has been reached. This is formatted as a list of
+        tokens, e.g. ['elif', 'else', 'endif']. If no matching token is
+        reached, raise an exception with the unclosed block tag details.
+        """
         if parse_until is None:
             parse_until = []
-        nodelist = self.create_nodelist()
+        nodelist = NodeList()
         while self.tokens:
             token = self.next_token()
             # Use the raw values here for TOKEN_* for a tiny performance boost.
@@ -314,38 +475,43 @@ class Parser(object):
                 self.extend_nodelist(nodelist, TextNode(token.contents), token)
             elif token.token_type == 1:  # TOKEN_VAR
                 if not token.contents:
-                    self.empty_variable(token)
+                    raise self.error(token, 'Empty variable tag')
                 try:
                     filter_expression = self.compile_filter(token.contents)
                 except TemplateSyntaxError as e:
-                    if not self.compile_filter_error(token, e):
+                    raise self.error(token, e)
-                        raise
+                var_node = VariableNode(filter_expression)
-                var_node = self.create_variable_node(filter_expression)
                 self.extend_nodelist(nodelist, var_node, token)
             elif token.token_type == 2:  # TOKEN_BLOCK
                 try:
                     command = token.contents.split()[0]
                 except IndexError:
-                    self.empty_block_tag(token)
+                    raise self.error(token, 'Empty block tag')
                 if command in parse_until:
-                    # put token back on token list so calling
+                    # A matching token has been reached. Return control to
-                    # code knows why it terminated
+                    # the caller. Put the token back on the token list so the
+                    # caller knows where it terminated.
                     self.prepend_token(token)
                     return nodelist
-                # execute callback function for this tag and append
+                # Add the token to the command stack. This is used for error
-                # resulting node
+                # messages if further parsing fails due to an unclosed block
-                self.enter_command(command, token)
+                # tag.
+                self.command_stack.append((command, token))
+                # Get the tag callback function from the ones registered with
+                # the parser.
                 try:
                     compile_func = self.tags[command]
                 except KeyError:
                     self.invalid_block_tag(token, command, parse_until)
+                # Compile the callback into a node object and add it to
+                # the node list.
                 try:
                     compiled_result = compile_func(self, token)
-                except TemplateSyntaxError as e:
+                except Exception as e:
-                    if not self.compile_function_error(token, e):
+                    raise self.error(token, e)
-                        raise
                 self.extend_nodelist(nodelist, compiled_result, token)
-                self.exit_command()
+                # Compile success. Remove the token from the command stack.
+                self.command_stack.pop()
         if parse_until:
             self.unclosed_block_tag(parse_until)
         return nodelist
@@ -357,38 +523,30 @@ class Parser(object):
                 return
         self.unclosed_block_tag([endtag])
 
-    def create_variable_node(self, filter_expression):
-        return VariableNode(filter_expression)
-
-    def create_nodelist(self):
-        return NodeList()
-
     def extend_nodelist(self, nodelist, node, token):
-        if node.must_be_first and nodelist:
+        # Check that non-text nodes don't appear before an extends tag.
-            try:
+        if node.must_be_first and nodelist.contains_nontext:
-                if nodelist.contains_nontext:
+            raise self.error(
-                    raise AttributeError
+                token, '%r must be the first tag in the template.' % node,
-            except AttributeError:
+            )
-                raise TemplateSyntaxError("%r must be the first tag "
-                                          "in the template." % node)
         if isinstance(nodelist, NodeList) and not isinstance(node, TextNode):
             nodelist.contains_nontext = True
+        # Set token here since we can't modify the node __init__ method
+        node.token = token
         nodelist.append(node)
 
-    def enter_command(self, command, token):
+    def error(self, token, e):
-        pass
+        """
-
+        Return an exception annotated with the originating token. Since the
-    def exit_command(self):
+        parser can be called recursively, check if a token is already set. This
-        pass
+        ensures the innermost token is highlighted if an exception occurs,
-
+        e.g. a compile error within the body of an if statement.
-    def error(self, token, msg):
+        """
-        return TemplateSyntaxError(msg)
+        if not isinstance(e, Exception):
-
+            e = TemplateSyntaxError(e)
-    def empty_variable(self, token):
+        if not hasattr(e, 'token'):
-        raise self.error(token, "Empty variable tag")
+            e.token = token
-
+        return e
-    def empty_block_tag(self, token):
-        raise self.error(token, "Empty block tag")
 
     def invalid_block_tag(self, token, command, parse_until=None):
         if parse_until:
@@ -397,13 +555,9 @@ class Parser(object):
         raise self.error(token, "Invalid block tag: '%s'" % command)
 
     def unclosed_block_tag(self, parse_until):
-        raise self.error(None, "Unclosed tags: %s " % ', '.join(parse_until))
+        command, token = self.command_stack.pop()
-
+        msg = "Unclosed tag '%s'. Looking for one of: %s." % (command, ', '.join(parse_until))
-    def compile_filter_error(self, token, e):
+        raise self.error(token, msg)
-        pass
-
-    def compile_function_error(self, token, e):
-        pass
 
     def next_token(self):
         return self.tokens.pop(0)
@@ -752,6 +906,7 @@ class Node(object):
     # they can be preceded by text nodes.
     must_be_first = False
     child_nodelists = ('nodelist',)
+    token = None
 
     def render(self, context):
         """
@@ -759,6 +914,20 @@ class Node(object):
         """
         pass
 
+    def render_annotated(self, context):
+        """
+        Render the node. If debug is True and an exception occurs during
+        rendering, the exception is annotated with contextual line information
+        where it occurred in the template. For internal usage this method is
+        preferred over using the render method directly.
+        """
+        try:
+            return self.render(context)
+        except Exception as e:
+            if context.template.engine.debug and not hasattr(e, 'template_debug'):
+                e.template_debug = context.template.get_exception_info(e, self.token)
+            raise
+
     def __iter__(self):
         yield self
 
@@ -786,7 +955,7 @@ class NodeList(list):
         bits = []
         for node in self:
             if isinstance(node, Node):
-                bit = self.render_node(node, context)
+                bit = node.render_annotated(context)
             else:
                 bit = node
             bits.append(force_text(bit))
@@ -799,9 +968,6 @@ class NodeList(list):
             nodes.extend(node.get_nodes_by_type(nodetype))
         return nodes
 
-    def render_node(self, node, context):
-        return node.render(context)
-
 
 class TextNode(Node):
     def __init__(self, s):

django/template/debug.py

@@ -1,102 +0,0 @@
-from django.template.base import (
-    Lexer, NodeList, Parser, TemplateSyntaxError, VariableNode, tag_re,
-)
-from django.utils.encoding import force_text
-from django.utils.formats import localize
-from django.utils.html import conditional_escape
-from django.utils.safestring import EscapeData, SafeData
-from django.utils.timezone import template_localtime
-
-
-class DebugLexer(Lexer):
-    def tokenize(self):
-        "Return a list of tokens from a given template_string"
-        result, upto = [], 0
-        for match in tag_re.finditer(self.template_string):
-            start, end = match.span()
-            if start > upto:
-                result.append(self.create_token(self.template_string[upto:start], (upto, start), False))
-                upto = start
-            result.append(self.create_token(self.template_string[start:end], (start, end), True))
-            upto = end
-        last_bit = self.template_string[upto:]
-        if last_bit:
-            result.append(self.create_token(last_bit, (upto, upto + len(last_bit)), False))
-        return result
-
-    def create_token(self, token_string, source, in_tag):
-        token = super(DebugLexer, self).create_token(token_string, in_tag)
-        token.source = self.origin, source
-        return token
-
-
-class DebugParser(Parser):
-    def __init__(self, lexer):
-        super(DebugParser, self).__init__(lexer)
-        self.command_stack = []
-
-    def enter_command(self, command, token):
-        self.command_stack.append((command, token.source))
-
-    def exit_command(self):
-        self.command_stack.pop()
-
-    def error(self, token, msg):
-        return self.source_error(token.source, msg)
-
-    def source_error(self, source, msg):
-        e = TemplateSyntaxError(msg)
-        e.django_template_source = source
-        return e
-
-    def create_nodelist(self):
-        return DebugNodeList()
-
-    def create_variable_node(self, contents):
-        return DebugVariableNode(contents)
-
-    def extend_nodelist(self, nodelist, node, token):
-        node.source = token.source
-        super(DebugParser, self).extend_nodelist(nodelist, node, token)
-
-    def unclosed_block_tag(self, parse_until):
-        command, source = self.command_stack.pop()
-        msg = "Unclosed tag '%s'. Looking for one of: %s " % (command, ', '.join(parse_until))
-        raise self.source_error(source, msg)
-
-    def compile_filter_error(self, token, e):
-        if not hasattr(e, 'django_template_source'):
-            e.django_template_source = token.source
-
-    def compile_function_error(self, token, e):
-        if not hasattr(e, 'django_template_source'):
-            e.django_template_source = token.source
-
-
-class DebugNodeList(NodeList):
-    def render_node(self, node, context):
-        try:
-            return node.render(context)
-        except Exception as e:
-            if not hasattr(e, 'django_template_source'):
-                e.django_template_source = node.source
-            raise
-
-
-class DebugVariableNode(VariableNode):
-    def render(self, context):
-        try:
-            output = self.filter_expression.resolve(context)
-            output = template_localtime(output, use_tz=context.use_tz)
-            output = localize(output, use_l10n=context.use_l10n)
-            output = force_text(output)
-        except UnicodeDecodeError:
-            return ''
-        except Exception as e:
-            if not hasattr(e, 'django_template_source'):
-                e.django_template_source = self.source
-            raise
-        if (context.autoescape and not isinstance(output, SafeData)) or isinstance(output, EscapeData):
-            return conditional_escape(output)
-        else:
-            return output

django/template/defaulttags.py

@@ -209,19 +209,10 @@ class ForNode(Node):
                         context.update(unpacked_vars)
                 else:
                     context[self.loopvars[0]] = item
-                # In debug mode provide the source of the node which raised
+
-                # the exception
+                for node in self.nodelist_loop:
-                if context.template.engine.debug:
+                    nodelist.append(node.render_annotated(context))
-                    for node in self.nodelist_loop:
+
-                        try:
-                            nodelist.append(node.render(context))
-                        except Exception as e:
-                            if not hasattr(e, 'django_template_source'):
-                                e.django_template_source = node.source
-                            raise
-                else:
-                    for node in self.nodelist_loop:
-                        nodelist.append(node.render(context))
                 if pop_context:
                     # The loop variables were pushed on to the context so pop them
                     # off again. This is necessary because the tag lets the length

django/template/engine.py

@@ -6,7 +6,7 @@ from django.utils.deprecation import RemovedInDjango20Warning
 from django.utils.functional import cached_property
 from django.utils.module_loading import import_string
 
-from .base import Context, Lexer, Parser, Template, TemplateDoesNotExist
+from .base import Context, Template, TemplateDoesNotExist
 from .context import _builtin_context_processors
 
 _context_instance_undefined = object()
@@ -235,20 +235,6 @@ class Engine(object):
         # If we get here, none of the templates could be loaded
         raise TemplateDoesNotExist(', '.join(not_found))
 
-    def compile_string(self, template_string, origin):
-        """
-        Compiles template_string into a NodeList ready for rendering.
-        """
-        if self.debug:
-            from .debug import DebugLexer, DebugParser
-            lexer_class, parser_class = DebugLexer, DebugParser
-        else:
-            lexer_class, parser_class = Lexer, Parser
-        lexer = lexer_class(template_string, origin)
-        tokens = lexer.tokenize()
-        parser = parser_class(tokens)
-        return parser.parse()
-
     def make_origin(self, display_name, loader, name, dirs):
         if self.debug and display_name:
             # Inner import to avoid circular dependency

django/template/loader.py

@@ -16,9 +16,6 @@ class LoaderOrigin(Origin):
         super(LoaderOrigin, self).__init__(display_name)
         self.loader, self.loadname, self.dirs = loader, name, dirs
 
-    def reload(self):
-        return self.loader(self.loadname, self.dirs)[0]
-
 
 def get_template(template_name, dirs=_dirs_undefined, using=None):
     """

django/utils/translation/trans_real.py

@@ -556,7 +556,7 @@ def templatize(src, origin=None):
             message = trim_whitespace(message)
         return message
 
-    for t in Lexer(src, origin).tokenize():
+    for t in Lexer(src).tokenize():
         if incomment:
             if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment':
                 content = ''.join(comment)

django/views/debug.py

@@ -15,7 +15,6 @@ from django.template.defaultfilters import force_escape, pprint
 from django.utils import lru_cache, six, timezone
 from django.utils.datastructures import MultiValueDict
 from django.utils.encoding import force_bytes, smart_text
-from django.utils.html import escape
 from django.utils.module_loading import import_string
 from django.utils.translation import ugettext as _
 
@@ -265,7 +264,7 @@ class ExceptionReporter(object):
         self.tb = tb
         self.is_email = is_email
 
-        self.template_info = None
+        self.template_info = getattr(self.exc_value, 'template_debug', None)
         self.template_does_not_exist = False
         self.loader_debug_info = None
 
@@ -323,12 +322,6 @@ class ExceptionReporter(object):
                     'templates': template_list,
                 })
 
-        # TODO: add support for multiple template engines (#24119).
-        if (default_template_engine is not None
-                and default_template_engine.debug
-                and hasattr(self.exc_value, 'django_template_source')):
-            self.get_template_exception_info()
-
         frames = self.get_traceback_frames()
         for i, frame in enumerate(frames):
             if 'vars' in frame:
@@ -393,46 +386,6 @@ class ExceptionReporter(object):
         c = Context(self.get_traceback_data(), autoescape=False, use_l10n=False)
         return t.render(c)
 
-    def get_template_exception_info(self):
-        origin, (start, end) = self.exc_value.django_template_source
-        template_source = origin.reload()
-        context_lines = 10
-        line = 0
-        upto = 0
-        source_lines = []
-        before = during = after = ""
-        for num, next in enumerate(linebreak_iter(template_source)):
-            if start >= upto and end <= next:
-                line = num
-                before = escape(template_source[upto:start])
-                during = escape(template_source[start:end])
-                after = escape(template_source[end:next])
-            source_lines.append((num, escape(template_source[upto:next])))
-            upto = next
-        total = len(source_lines)
-
-        top = max(1, line - context_lines)
-        bottom = min(total, line + 1 + context_lines)
-
-        # In some rare cases, exc_value.args might be empty.
-        try:
-            message = self.exc_value.args[0]
-        except IndexError:
-            message = '(Could not get exception message)'
-
-        self.template_info = {
-            'message': message,
-            'source_lines': source_lines[top:bottom],
-            'before': before,
-            'during': during,
-            'after': after,
-            'top': top,
-            'bottom': bottom,
-            'total': total,
-            'line': line,
-            'name': origin.name,
-        }
-
     def _get_lines_from_file(self, filename, lineno, context_lines, loader=None, module_name=None):
         """
         Returns context_lines before and after lineno from file.

tests/template_tests/syntax_tests/test_basic.py

@@ -311,3 +311,15 @@ class BasicSyntaxTests(SimpleTestCase):
         """
         output = self.engine.render_to_string('basic-syntax38', {"var": {"callable": lambda: "foo bar"}})
         self.assertEqual(output, 'foo bar')
+
+    @setup({'template': '{% block content %}'})
+    def test_unclosed_block(self):
+        msg = "Unclosed tag 'block'. Looking for one of: endblock."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% if a %}'})
+    def test_unclosed_block2(self):
+        msg = "Unclosed tag 'if'. Looking for one of: elif, else, endif."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')

tests/template_tests/test_engine.py

@@ -52,23 +52,11 @@ class DeprecatedRenderToStringTest(SimpleTestCase):
 
 class LoaderTests(SimpleTestCase):
 
-    def test_debug_nodelist_name(self):
-        engine = Engine(dirs=[TEMPLATE_DIR], debug=True)
-        template_name = 'index.html'
-        template = engine.get_template(template_name)
-        name = template.nodelist[0].source[0].name
-        self.assertTrue(name.endswith(template_name))
-
     def test_origin(self):
         engine = Engine(dirs=[TEMPLATE_DIR], debug=True)
         template = engine.get_template('index.html')
         self.assertEqual(template.origin.loadname, 'index.html')
 
-    def test_origin_debug_false(self):
-        engine = Engine(dirs=[TEMPLATE_DIR], debug=False)
-        template = engine.get_template('index.html')
-        self.assertEqual(template.origin, None)
-
     def test_loader_priority(self):
         """
         #21460 -- Check that the order of template loader works.

tests/template_tests/test_loaders.py

@@ -64,24 +64,6 @@ class CachedLoaderTests(SimpleTestCase):
             "Cached loader failed to cache the TemplateDoesNotExist exception",
         )
 
-    def test_debug_nodelist_name(self):
-        template_name = 'index.html'
-        engine = Engine(dirs=[TEMPLATE_DIR], debug=True)
-
-        template = engine.get_template(template_name)
-        name = template.nodelist[0].source[0].name
-        self.assertTrue(
-            name.endswith(template_name),
-            'Template loaded through cached loader has incorrect name for debug page: %s' % template_name,
-        )
-
-        template = engine.get_template(template_name)
-        name = template.nodelist[0].source[0].name
-        self.assertTrue(
-            name.endswith(template_name),
-            'Cached template loaded through cached loader has incorrect name for debug page: %s' % template_name,
-        )
-
 
 @unittest.skipUnless(pkg_resources, 'setuptools is not installed')
 class EggLoaderTests(SimpleTestCase):

tests/template_tests/test_nodelist.py

@@ -51,6 +51,5 @@ class ErrorIndexTest(TestCase):
             try:
                 template.render(context)
             except (RuntimeError, TypeError) as e:
-                error_source_index = e.django_template_source[1]
+                debug = e.template_debug
-                self.assertEqual(error_source_index,
+                self.assertEqual((debug['start'], debug['end']), expected_error_source_index)
-                                 expected_error_source_index)

tests/template_tests/test_parser.py

@@ -5,11 +5,10 @@ from __future__ import unicode_literals
 
 from unittest import TestCase
 
-from django.template import Library, Template, TemplateSyntaxError
+from django.template import Library, TemplateSyntaxError
 from django.template.base import (
     TOKEN_BLOCK, FilterExpression, Parser, Token, Variable,
 )
-from django.test import override_settings
 from django.utils import six
 
 
@@ -73,14 +72,6 @@ class ParserTests(TestCase):
         with six.assertRaisesRegex(self, TypeError, "Variable must be a string or number, got <(class|type) 'dict'>"):
             Variable({})
 
-    @override_settings(DEBUG=True)
-    def test_compile_filter_error(self):
-        # regression test for #19819
-        msg = "Could not parse the remainder: '@bar' from 'foo@bar'"
-        with six.assertRaisesRegex(self, TemplateSyntaxError, msg) as cm:
-            Template("{% if 1 %}{{ foo@bar }}{% endif %}")
-        self.assertEqual(cm.exception.django_template_source[1], (10, 23))
-
     def test_filter_args_count(self):
         p = Parser("")
         l = Library()

tests/template_tests/tests.py

@@ -6,7 +6,7 @@ import sys
 from django.contrib.auth.models import Group
 from django.core import urlresolvers
 from django.template import (
-    Context, Template, TemplateSyntaxError, engines, loader,
+    Context, Engine, Template, TemplateSyntaxError, engines, loader,
 )
 from django.test import SimpleTestCase, override_settings
 
@@ -48,31 +48,61 @@ class TemplateTests(SimpleTestCase):
             self.assertGreater(depth, 5,
                 "The traceback context was lost when reraising the traceback. See #19827")
 
-    @override_settings(DEBUG=True)
     def test_no_wrapped_exception(self):
         """
         # 16770 -- The template system doesn't wrap exceptions, but annotates
         them.
         """
+        engine = Engine(debug=True)
         c = Context({"coconuts": lambda: 42 / 0})
-        t = Template("{{ coconuts }}")
+        t = engine.from_string("{{ coconuts }}")
-        with self.assertRaises(ZeroDivisionError) as cm:
+
+        with self.assertRaises(ZeroDivisionError) as e:
             t.render(c)
 
-        self.assertEqual(cm.exception.django_template_source[1], (0, 14))
+        debug = e.exception.template_debug
+        self.assertEqual(debug['start'], 0)
+        self.assertEqual(debug['end'], 14)
 
     def test_invalid_block_suggestion(self):
         """
         #7876 -- Error messages should include the unexpected block name.
         """
+        engine = Engine()
+
         with self.assertRaises(TemplateSyntaxError) as e:
-            Template("{% if 1 %}lala{% endblock %}{% endif %}")
+            engine.from_string("{% if 1 %}lala{% endblock %}{% endif %}")
 
         self.assertEqual(
             e.exception.args[0],
             "Invalid block tag: 'endblock', expected 'elif', 'else' or 'endif'",
         )
 
+    def test_compile_filter_expression_error(self):
+        """
+        19819 -- Make sure the correct token is highlighted for
+        FilterExpression errors.
+        """
+        engine = Engine(debug=True)
+        msg = "Could not parse the remainder: '@bar' from 'foo@bar'"
+
+        with self.assertRaisesMessage(TemplateSyntaxError, msg) as e:
+            engine.from_string("{% if 1 %}{{ foo@bar }}{% endif %}")
+
+        debug = e.exception.template_debug
+        self.assertEqual((debug['start'], debug['end']), (10, 23))
+        self.assertEqual((debug['during']), '{{ foo@bar }}')
+
+    def test_compile_tag_error(self):
+        """
+        Errors raised while compiling nodes should include the token
+        information.
+        """
+        engine = Engine(debug=True)
+        with self.assertRaises(RuntimeError) as e:
+            engine.from_string("{% load bad_tag %}{% badtag %}")
+        self.assertEqual(e.exception.template_debug['during'], '{% badtag %}')
+
     def test_super_errors(self):
         """
         #18169 -- NoReverseMatch should not be silence in block.super.