Added ctx to visitTerminal in parse tree listener. That should be useful information. made commensurate change in the parse tree walker to make sure I have a proper context to send to the visitTerminal method. renamed a few fields of parser/lexer to have _ in front to avoid name collisions with user actions. change the name of the listener method so that they're more explicit using the terms Error and Parser to identify what kind of listener we are adding or removing. Added a default TraceListener to Parser so that we can call setTrace(true) to have it start dumping out information as a first step in the debugging process. there are now multiple parse tree listeners possible because I made a list. we may want to pass in multiple actions to the parser.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9795]

runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java

@@ -91,7 +91,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 //			System.err.print("[SPURIOUS] ");
 			return; // don't count spurious errors
 		}
-		recognizer.syntaxErrors++;
+		recognizer._syntaxErrors++;
 		beginErrorCondition(recognizer);
 		if ( e instanceof NoViableAltException ) {
 			reportNoViableAlternative(recognizer, (NoViableAltException) e);
@@ -105,7 +105,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 		else {
 			System.err.println("unknown recognition error type: "+e.getClass().getName());
 			if ( recognizer!=null ) {
-				recognizer.notifyListeners((Token)e.offendingToken, e.getMessage(), e);
+				recognizer.notifyErrorListeners((Token) e.offendingToken, e.getMessage(), e);
 			}
 		}
 	}
@@ -206,7 +206,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 			input = "<unknown input>";
 		}
 		String msg = "no viable alternative at input "+escapeWSAndQuote(input);
-		recognizer.notifyListeners((Token) e.offendingToken, msg, e);
+		recognizer.notifyErrorListeners((Token) e.offendingToken, msg, e);
 	}
 
 	public void reportInputMismatch(Parser recognizer,
@@ -215,7 +215,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 	{
 		String msg = "mismatched input "+getTokenErrorDisplay((Token)e.offendingToken)+
 		" expecting "+e.getExpectedTokens().toString(recognizer.getTokenNames());
-		recognizer.notifyListeners((Token)e.offendingToken, msg, e);
+		recognizer.notifyErrorListeners((Token) e.offendingToken, msg, e);
 	}
 
 	public void reportFailedPredicate(Parser recognizer,
@@ -224,12 +224,12 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 	{
 		String ruleName = recognizer.getRuleNames()[recognizer._ctx.getRuleIndex()];
 		String msg = "rule "+ruleName+" "+e.msg;
-		recognizer.notifyListeners((Token)e.offendingToken, msg, e);
+		recognizer.notifyErrorListeners((Token) e.offendingToken, msg, e);
 	}
 
 	public void reportUnwantedToken(Parser recognizer) {
 		if (errorRecoveryMode) return;
-		recognizer.syntaxErrors++;
+		recognizer._syntaxErrors++;
 		beginErrorCondition(recognizer);
 
 		Token t = recognizer.getCurrentToken();
@@ -237,12 +237,12 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 		IntervalSet expecting = getExpectedTokens(recognizer);
 		String msg = "extraneous input "+tokenName+" expecting "+
 			expecting.toString(recognizer.getTokenNames());
-		recognizer.notifyListeners(t, msg, null);
+		recognizer.notifyErrorListeners(t, msg, null);
 	}
 
 	public void reportMissingToken(Parser recognizer) {
 		if (errorRecoveryMode) return;
-		recognizer.syntaxErrors++;
+		recognizer._syntaxErrors++;
 		beginErrorCondition(recognizer);
 
 		Token t = recognizer.getCurrentToken();
@@ -250,7 +250,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 		String msg = "missing "+expecting.toString(recognizer.getTokenNames())+
 			" at "+getTokenErrorDisplay(t);
 
-		recognizer.notifyListeners(t, msg, null);
+		recognizer.notifyErrorListeners(t, msg, null);
 	}
 
 	/** Attempt to recover from a single missing or extra token.

runtime/Java/src/org/antlr/v4/runtime/Lexer.java

@@ -322,7 +322,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
 	public void notifyListeners(LexerNoViableAltException e) {
 		String msg = "token recognition error at: '"+
 			_input.substring(tokenStartCharIndex, _input.index())+"'";
-		ANTLRErrorListener<Integer>[] listeners = getListeners();
+		ANTLRErrorListener<Integer>[] listeners = getErrorListeners();
 		if ( listeners.length == 0 ) {
 			System.err.println("line "+tokenStartLine+":"+
 							   tokenStartCharPositionInLine+" "+

runtime/Java/src/org/antlr/v4/runtime/Parser.java

@@ -39,7 +39,22 @@ import java.util.List;
 
 /** This is all the parsing support code essentially; most of it is error recovery stuff. */
 public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Token>> {
-	public static final String NEXT_TOKEN_RULE_NAME = "nextToken";
+	public class TraceListener implements ParseTreeListener<Token> {
+		@Override
+		public void enterEveryRule(ParserRuleContext<Token> ctx) {
+			System.out.println("enter   " + getRuleNames()[ctx.ruleIndex] + ", LT(1)=" + _input.LT(1).getText());
+		}
+
+		@Override
+		public void exitEveryRule(ParserRuleContext<Token> ctx) {
+			System.out.println("exit    "+getRuleNames()[ctx.ruleIndex]+", LT(1)="+_input.LT(1).getText());
+		}
+
+		@Override
+		public void visitTerminal(ParserRuleContext<Token> ctx, Token token) {
+			System.out.println("consume "+token+" rule "+getRuleNames()[ctx.ruleIndex]+" alt="+ctx.altNum);
+		}
+	}
 
 	protected TokenStream _input;
 
@@ -50,8 +65,9 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 	 */
 	protected ParserRuleContext<Token> _ctx;
 
-	protected boolean buildParseTrees;
+	protected boolean _buildParseTrees;
-	protected boolean traceATNStates;
+
+	protected TraceListener _tracer;
 
 	/** If the listener is non-null, trigger enter and exit rule events
      *  *during* the parse. This is typically done only when not building
@@ -59,10 +75,10 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
      *  the parse or during tree walks later. Both could be done.
      *  Not intended for tree parsing but would work.
      */
-    protected ParseTreeListener<Token> _listener;
+    protected List<ParseTreeListener<Token>> _parseListeners;
 
 	/** Did the recognizer encounter a syntax error?  Track how many. */
-	protected int syntaxErrors = 0;
+	protected int _syntaxErrors = 0;
 
 	public Parser(IntStream input) {
 		setInputStream(input);
@@ -73,7 +89,8 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 		if ( getInputStream()!=null ) getInputStream().seek(0);
 		_errHandler.endErrorCondition(this);
 		_ctx = null;
-		syntaxErrors = 0;
+		_syntaxErrors = 0;
+		_tracer = null;
 		ATNSimulator interpreter = getInterpreter();
 		if (interpreter != null) {
 			interpreter.reset();
@@ -85,23 +102,20 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 	 *  that fails, throw MismatchedTokenException.
 	 */
 	public Token match(int ttype) throws RecognitionException {
-//		System.out.println("match "+((TokenStream)input).LT(1)+" vs expected "+ttype);
+		Token t = getCurrentToken();
-		Token currentSymbol = getCurrentToken();
 		if ( getInputStream().LA(1)==ttype ) {
 			_errHandler.endErrorCondition(this);
 			consume();
 		}
 		else {
-			currentSymbol = _errHandler.recoverInline(this);
+			t = _errHandler.recoverInline(this);
-			if ( buildParseTrees && currentSymbol instanceof Token &&
+			if ( _buildParseTrees && t.getTokenIndex()==-1 ) {
-			     ((Token)currentSymbol).getTokenIndex()==-1 )
-			{
 				// we must have conjured up a new token during single token insertion
 				// if it's not the current symbol
-				_ctx.addErrorNode((Token)currentSymbol);
+				_ctx.addErrorNode(t);
 			}
 		}
-		return currentSymbol;
+		return t;
 	}
 
 	/** Track the RuleContext objects during the parse and hook them up
@@ -125,11 +139,11 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 	 *  for garbage collection.
 	 */
 	public void setBuildParseTree(boolean buildParseTrees) {
-		this.buildParseTrees = buildParseTrees;
+		this._buildParseTrees = buildParseTrees;
 	}
 
 	public boolean getBuildParseTree() {
-		return buildParseTrees;
+		return _buildParseTrees;
 	}
 
 //	public void setTraceATNStates(boolean traceATNStates) {
@@ -140,13 +154,24 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 //		return traceATNStates;
 //	}
 
-    public ParseTreeListener<Token> getListener() {
+    public List<ParseTreeListener<Token>> getParseListeners() {
-        return _listener;
+        return _parseListeners;
     }
 
-    public void setListener(ParseTreeListener<Token> listener) {
+    public void addParseListener(ParseTreeListener<Token> listener) {
-        this._listener = listener;
+		if ( listener==null ) return;
+		if ( _parseListeners==null ) {
+			_parseListeners = new ArrayList<ParseTreeListener<Token>>();
 		}
+        this._parseListeners.add(listener);
+    }
+
+	public void removeParseListener(ParseTreeListener<Token> l) {
+		if ( l==null ) return;
+		if ( _parseListeners!=null ) _parseListeners.remove(l);
+	}
+
+	public void removeParseListeners() { if ( _parseListeners!=null ) _parseListeners.clear(); }
 
     /** Get number of recognition errors (lexer, parser, tree parser).  Each
 	 *  recognizer tracks its own number.  So parser and lexer each have
@@ -156,7 +181,7 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 	 *  See also reportError()
 	 */
 	public int getNumberOfSyntaxErrors() {
-		return syntaxErrors;
+		return _syntaxErrors;
 	}
 
 	/** Tell our token source and error strategy about a new way to create tokens */
@@ -204,11 +229,11 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 		return _input.LT(1);
 	}
 
-    public void notifyListeners(String msg)	{
+    public void notifyErrorListeners(String msg)	{
-		notifyListeners(getCurrentToken(), msg, null);
+		notifyErrorListeners(getCurrentToken(), msg, null);
 	}
 
-	public void notifyListeners(Token offendingToken, String msg,
+	public void notifyErrorListeners(Token offendingToken, String msg,
 									 @Nullable RecognitionException e)
 	{
 		int line = -1;
@@ -217,7 +242,7 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 			line = ((Token) offendingToken).getLine();
 			charPositionInLine = ((Token) offendingToken).getCharPositionInLine();
 		}
-		ANTLRErrorListener<Token>[] listeners = getListeners();
+		ANTLRErrorListener<Token>[] listeners = getErrorListeners();
 		if ( listeners.length == 0 ) {
 			System.err.println("line "+line+":"+charPositionInLine+" "+msg);
 			return;
@@ -243,7 +268,7 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 	public Token consume() {
 		Token o = getCurrentToken();
 		getInputStream().consume();
-		if ( buildParseTrees ) {
+		if (_buildParseTrees) {
 			// TODO: tree parsers?
 			if ( _errHandler.inErrorRecoveryMode(this) ) {
 //				System.out.println("consume in error recovery mode for "+o);
@@ -251,7 +276,9 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 			}
 			else _ctx.addChild((Token)o);
 		}
-        if ( _listener != null) _listener.visitTerminal(o);
+		if ( _parseListeners != null) {
+			for (ParseTreeListener<Token> l : _parseListeners) l.visitTerminal(_ctx, o);
+		}
 		return o;
 	}
 
@@ -274,18 +301,22 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 		_ctx = localctx;
 		_ctx.start = _input.LT(1);
 		_ctx.ruleIndex = ruleIndex;
-		if ( buildParseTrees ) addContextToParseTree();
+		if (_buildParseTrees) addContextToParseTree();
-        if ( _listener != null) {
+        if ( _parseListeners != null) {
-            _listener.enterEveryRule(_ctx);
+            for (ParseTreeListener<Token> l : _parseListeners) {
-            _ctx.enterRule(_listener);
+				_ctx.enterRule(l);
+				l.enterEveryRule(_ctx);
+			}
         }
 	}
 
     public void exitRule(int ruleIndex) {
         // trigger event on _ctx, before it reverts to parent
-        if ( _listener != null) {
+        if ( _parseListeners != null) {
-            _ctx.exitRule(_listener);
+			for (ParseTreeListener<Token> l : _parseListeners) {
-            _listener.exitEveryRule(_ctx);
+				_ctx.exitRule(l);
+				l.exitEveryRule(_ctx);
+			}
         }
 		_ctx = (ParserRuleContext<Token>)_ctx.parent;
     }
@@ -293,7 +324,7 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 	public void enterOuterAlt(ParserRuleContext<Token> localctx, int altNum) {
 		// if we have new localctx, make sure we replace existing ctx
 		// that is previous child of parse tree
-		if ( buildParseTrees && _ctx != localctx ) {
+		if ( _buildParseTrees && _ctx != localctx ) {
 			ParserRuleContext parent = (ParserRuleContext)_ctx.parent;
 			parent.removeLastChild();
 			if ( parent!=null )	parent.addChild(localctx);
@@ -471,4 +502,19 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
 		_ctx.s = atnState;
 //		if ( traceATNStates ) _ctx.trace(atnState);
 	}
+
+	/** During a parse is extremely useful to listen in on the rule entry and exit
+	 *  events as well as token matches. This is for quick and dirty debugging.
+	 */
+	public void setTrace(boolean trace) {
+		if ( !trace ) {
+			removeParseListener(_tracer);
+			_tracer = null;
+		}
+		else {
+			if ( _tracer!=null ) removeParseListener(_tracer);
+			else _tracer = new TraceListener();
+			addParseListener( _tracer );
+		}
+	}
 }

runtime/Java/src/org/antlr/v4/runtime/Recognizer.java

@@ -97,7 +97,7 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 		return "'"+s+"'";
 	}
 
-	public void addListener(ANTLRErrorListener<Symbol> pl) {
+	public void addErrorListener(ANTLRErrorListener<Symbol> pl) {
 		if ( _listeners ==null ) {
 			_listeners =
 				Collections.synchronizedList(new ArrayList<ANTLRErrorListener<Symbol>>(2));
@@ -105,11 +105,13 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 		if ( pl!=null ) _listeners.add(pl);
 	}
 
-	public void removeListener(ANTLRErrorListener<Symbol> pl) { _listeners.remove(pl); }
+	public void removeErrorListener(ANTLRErrorListener<Symbol> pl) {
+		if ( _listeners!=null ) _listeners.remove(pl);
+	}
 
-	public void removeListeners() { _listeners.clear(); }
+	public void removeErrorListeners() { if ( _listeners!=null ) _listeners.clear(); }
 
-	public @NotNull ANTLRErrorListener<Symbol>[] getListeners() {
+	public @NotNull ANTLRErrorListener<Symbol>[] getErrorListeners() {
 		if (_listeners == null) {
 			return EMPTY_LISTENERS;
 		}

runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java

@@ -32,7 +32,7 @@ package org.antlr.v4.runtime.tree;
 import org.antlr.v4.runtime.ParserRuleContext;
 
 public interface ParseTreeListener<Symbol> {
-    void visitTerminal(Symbol symbol);
+    void visitTerminal(ParserRuleContext<Symbol> ctx, Symbol symbol);
     void enterEveryRule(ParserRuleContext<Symbol> ctx);
     void exitEveryRule(ParserRuleContext<Symbol> ctx);
 }

runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java

@@ -48,8 +48,15 @@ public class ParseTreeWalker {
 		exitRule(listener, r);
     }
 
-    protected <Symbol> void visitTerminal(ParseTreeListener<Symbol> listener, ParseTree.TerminalNode<Symbol> t) {
+    protected <Symbol> void visitTerminal(ParseTreeListener<Symbol> listener,
-        listener.visitTerminal(t.getSymbol());
+										  ParseTree.TerminalNode<Symbol> t)
+	{
+		ParseTree.RuleNode r = (ParseTree.RuleNode)t.getParent();
+		ParserRuleContext<Symbol> ctx = null;
+		if ( r.getRuleContext() instanceof ParserRuleContext ) {
+			ctx = (ParserRuleContext<Symbol>)r.getRuleContext();
+		}
+        listener.visitTerminal(ctx, t.getSymbol());
     }
 
 	/** The discovery of a rule node, involves sending two events:

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -52,7 +52,7 @@ public class Blank<file.grammarName>Listener implements <file.grammarName>Listen
 
 	@Override public void enterEveryRule(ParserRuleContext\<<InputSymbolType()> > ctx) { }
 	@Override public void exitEveryRule(ParserRuleContext\<<InputSymbolType()> > ctx) { }
-	@Override public void visitTerminal(<InputSymbolType()> symbol) { }
+	@Override public void visitTerminal(ParserRuleContext\<<InputSymbolType()> > ctx, <InputSymbolType()> symbol) { }
 }
 >>
 

tool/test/org/antlr/v4/test/TestPerformance.java

@@ -43,7 +43,7 @@ public class TestPerformance extends BaseTest {
     /**
      *  Use ParseTreeWalker.DEFAULT.walk with the BlankJavaParserListener to show parse tree walking overhead.
      *  If {@link #BUILD_PARSE_TREES} is false, the listener will instead be called during the parsing process via
-     *  {@link org.antlr.v4.runtime.Parser#setListener}.
+     *  {@link org.antlr.v4.runtime.Parser#addParseListener}.
      */
     private static final boolean BLANK_LISTENER = false;
 
@@ -262,7 +262,7 @@ public class TestPerformance extends BaseTest {
                             sharedParser = parserCtor.newInstance(tokens);
                             sharedParser.setBuildParseTree(BUILD_PARSE_TREES);
                             if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
-                                sharedParser.setListener(sharedListener);
+                                sharedParser.addParseListener(sharedListener);
                             }
                             if (BAIL_ON_ERROR) {
                                 sharedParser.setErrorHandler(new BailErrorStrategy());