From 8099cec3bd81c4428fab51d3b79bd373a62cba07 Mon Sep 17 00:00:00 2001 From: parrt Date: Tue, 3 Jan 2012 12:46:18 -0800 Subject: [PATCH] Added ctx to visitTerminal in parse tree listener. That should be useful information. made commensurate change in the parse tree walker to make sure I have a proper context to send to the visitTerminal method. renamed a few fields of parser/lexer to have _ in front to avoid name collisions with user actions. change the name of the listener method so that they're more explicit using the terms Error and Parser to identify what kind of listener we are adding or removing. Added a default TraceListener to Parser so that we can call setTrace(true) to have it start dumping out information as a first step in the debugging process. there are now multiple parse tree listeners possible because I made a list. we may want to pass in multiple actions to the parser. [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9795] --- .../v4/runtime/DefaultErrorStrategy.java | 18 +-- .../Java/src/org/antlr/v4/runtime/Lexer.java | 2 +- .../Java/src/org/antlr/v4/runtime/Parser.java | 120 ++++++++++++------ .../src/org/antlr/v4/runtime/Recognizer.java | 10 +- .../v4/runtime/tree/ParseTreeListener.java | 2 +- .../v4/runtime/tree/ParseTreeWalker.java | 11 +- .../v4/tool/templates/codegen/Java/Java.stg | 2 +- .../org/antlr/v4/test/TestPerformance.java | 4 +- 8 files changed, 112 insertions(+), 57 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java index 8de3ce476..a02d2ffcc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java @@ -91,7 +91,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { // System.err.print("[SPURIOUS] "); return; // don't count spurious errors } - recognizer.syntaxErrors++; + recognizer._syntaxErrors++; beginErrorCondition(recognizer); if ( e instanceof NoViableAltException ) { reportNoViableAlternative(recognizer, (NoViableAltException) e); @@ -105,7 +105,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { else { System.err.println("unknown recognition error type: "+e.getClass().getName()); if ( recognizer!=null ) { - recognizer.notifyListeners((Token)e.offendingToken, e.getMessage(), e); + recognizer.notifyErrorListeners((Token) e.offendingToken, e.getMessage(), e); } } } @@ -206,7 +206,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { input = ""; } String msg = "no viable alternative at input "+escapeWSAndQuote(input); - recognizer.notifyListeners((Token) e.offendingToken, msg, e); + recognizer.notifyErrorListeners((Token) e.offendingToken, msg, e); } public void reportInputMismatch(Parser recognizer, @@ -215,7 +215,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { { String msg = "mismatched input "+getTokenErrorDisplay((Token)e.offendingToken)+ " expecting "+e.getExpectedTokens().toString(recognizer.getTokenNames()); - recognizer.notifyListeners((Token)e.offendingToken, msg, e); + recognizer.notifyErrorListeners((Token) e.offendingToken, msg, e); } public void reportFailedPredicate(Parser recognizer, @@ -224,12 +224,12 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { { String ruleName = recognizer.getRuleNames()[recognizer._ctx.getRuleIndex()]; String msg = "rule "+ruleName+" "+e.msg; - recognizer.notifyListeners((Token)e.offendingToken, msg, e); + recognizer.notifyErrorListeners((Token) e.offendingToken, msg, e); } public void reportUnwantedToken(Parser recognizer) { if (errorRecoveryMode) return; - recognizer.syntaxErrors++; + recognizer._syntaxErrors++; beginErrorCondition(recognizer); Token t = recognizer.getCurrentToken(); @@ -237,12 +237,12 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { IntervalSet expecting = getExpectedTokens(recognizer); String msg = "extraneous input "+tokenName+" expecting "+ expecting.toString(recognizer.getTokenNames()); - recognizer.notifyListeners(t, msg, null); + recognizer.notifyErrorListeners(t, msg, null); } public void reportMissingToken(Parser recognizer) { if (errorRecoveryMode) return; - recognizer.syntaxErrors++; + recognizer._syntaxErrors++; beginErrorCondition(recognizer); Token t = recognizer.getCurrentToken(); @@ -250,7 +250,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { String msg = "missing "+expecting.toString(recognizer.getTokenNames())+ " at "+getTokenErrorDisplay(t); - recognizer.notifyListeners(t, msg, null); + recognizer.notifyErrorListeners(t, msg, null); } /** Attempt to recover from a single missing or extra token. diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index a9b3c531d..09397db5d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -322,7 +322,7 @@ public abstract class Lexer extends Recognizer public void notifyListeners(LexerNoViableAltException e) { String msg = "token recognition error at: '"+ _input.substring(tokenStartCharIndex, _input.index())+"'"; - ANTLRErrorListener[] listeners = getListeners(); + ANTLRErrorListener[] listeners = getErrorListeners(); if ( listeners.length == 0 ) { System.err.println("line "+tokenStartLine+":"+ tokenStartCharPositionInLine+" "+ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index b1aeda4c3..22b581aa0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -39,7 +39,22 @@ import java.util.List; /** This is all the parsing support code essentially; most of it is error recovery stuff. */ public abstract class Parser extends Recognizer> { - public static final String NEXT_TOKEN_RULE_NAME = "nextToken"; + public class TraceListener implements ParseTreeListener { + @Override + public void enterEveryRule(ParserRuleContext ctx) { + System.out.println("enter " + getRuleNames()[ctx.ruleIndex] + ", LT(1)=" + _input.LT(1).getText()); + } + + @Override + public void exitEveryRule(ParserRuleContext ctx) { + System.out.println("exit "+getRuleNames()[ctx.ruleIndex]+", LT(1)="+_input.LT(1).getText()); + } + + @Override + public void visitTerminal(ParserRuleContext ctx, Token token) { + System.out.println("consume "+token+" rule "+getRuleNames()[ctx.ruleIndex]+" alt="+ctx.altNum); + } + } protected TokenStream _input; @@ -50,19 +65,20 @@ public abstract class Parser extends Recognizer _ctx; - protected boolean buildParseTrees; - protected boolean traceATNStates; + protected boolean _buildParseTrees; - /** If the listener is non-null, trigger enter and exit rule events + protected TraceListener _tracer; + + /** If the listener is non-null, trigger enter and exit rule events * *during* the parse. This is typically done only when not building * parse trees for later visiting. We either trigger events during * the parse or during tree walks later. Both could be done. * Not intended for tree parsing but would work. */ - protected ParseTreeListener _listener; + protected List> _parseListeners; /** Did the recognizer encounter a syntax error? Track how many. */ - protected int syntaxErrors = 0; + protected int _syntaxErrors = 0; public Parser(IntStream input) { setInputStream(input); @@ -73,7 +89,8 @@ public abstract class Parser extends Recognizer getListener() { - return _listener; + public List> getParseListeners() { + return _parseListeners; } - public void setListener(ParseTreeListener listener) { - this._listener = listener; + public void addParseListener(ParseTreeListener listener) { + if ( listener==null ) return; + if ( _parseListeners==null ) { + _parseListeners = new ArrayList>(); + } + this._parseListeners.add(listener); } + public void removeParseListener(ParseTreeListener l) { + if ( l==null ) return; + if ( _parseListeners!=null ) _parseListeners.remove(l); + } + + public void removeParseListeners() { if ( _parseListeners!=null ) _parseListeners.clear(); } + /** Get number of recognition errors (lexer, parser, tree parser). Each * recognizer tracks its own number. So parser and lexer each have * separate count. Does not count the spurious errors found between @@ -156,7 +181,7 @@ public abstract class Parser extends Recognizer[] listeners = getListeners(); + ANTLRErrorListener[] listeners = getErrorListeners(); if ( listeners.length == 0 ) { System.err.println("line "+line+":"+charPositionInLine+" "+msg); return; @@ -243,7 +268,7 @@ public abstract class Parser extends Recognizer l : _parseListeners) l.visitTerminal(_ctx, o); + } return o; } @@ -274,18 +301,22 @@ public abstract class Parser extends Recognizer l : _parseListeners) { + _ctx.enterRule(l); + l.enterEveryRule(_ctx); + } } } public void exitRule(int ruleIndex) { // trigger event on _ctx, before it reverts to parent - if ( _listener != null) { - _ctx.exitRule(_listener); - _listener.exitEveryRule(_ctx); + if ( _parseListeners != null) { + for (ParseTreeListener l : _parseListeners) { + _ctx.exitRule(l); + l.exitEveryRule(_ctx); + } } _ctx = (ParserRuleContext)_ctx.parent; } @@ -293,7 +324,7 @@ public abstract class Parser extends Recognizer localctx, int altNum) { // if we have new localctx, make sure we replace existing ctx // that is previous child of parse tree - if ( buildParseTrees && _ctx != localctx ) { + if ( _buildParseTrees && _ctx != localctx ) { ParserRuleContext parent = (ParserRuleContext)_ctx.parent; parent.removeLastChild(); if ( parent!=null ) parent.addChild(localctx); @@ -471,4 +502,19 @@ public abstract class Parser extends Recognizer { return "'"+s+"'"; } - public void addListener(ANTLRErrorListener pl) { + public void addErrorListener(ANTLRErrorListener pl) { if ( _listeners ==null ) { _listeners = Collections.synchronizedList(new ArrayList>(2)); @@ -105,11 +105,13 @@ public abstract class Recognizer { if ( pl!=null ) _listeners.add(pl); } - public void removeListener(ANTLRErrorListener pl) { _listeners.remove(pl); } + public void removeErrorListener(ANTLRErrorListener pl) { + if ( _listeners!=null ) _listeners.remove(pl); + } - public void removeListeners() { _listeners.clear(); } + public void removeErrorListeners() { if ( _listeners!=null ) _listeners.clear(); } - public @NotNull ANTLRErrorListener[] getListeners() { + public @NotNull ANTLRErrorListener[] getErrorListeners() { if (_listeners == null) { return EMPTY_LISTENERS; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java index 355854aa4..fa6872c62 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java @@ -32,7 +32,7 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.ParserRuleContext; public interface ParseTreeListener { - void visitTerminal(Symbol symbol); + void visitTerminal(ParserRuleContext ctx, Symbol symbol); void enterEveryRule(ParserRuleContext ctx); void exitEveryRule(ParserRuleContext ctx); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java index 9328e1b6c..0e4932dbd 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java @@ -48,8 +48,15 @@ public class ParseTreeWalker { exitRule(listener, r); } - protected void visitTerminal(ParseTreeListener listener, ParseTree.TerminalNode t) { - listener.visitTerminal(t.getSymbol()); + protected void visitTerminal(ParseTreeListener listener, + ParseTree.TerminalNode t) + { + ParseTree.RuleNode r = (ParseTree.RuleNode)t.getParent(); + ParserRuleContext ctx = null; + if ( r.getRuleContext() instanceof ParserRuleContext ) { + ctx = (ParserRuleContext)r.getRuleContext(); + } + listener.visitTerminal(ctx, t.getSymbol()); } /** The discovery of a rule node, involves sending two events: diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 4bd04bfa1..0c03768a6 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -52,7 +52,7 @@ public class BlankListener implements Listen @Override public void enterEveryRule(ParserRuleContext\< > ctx) { } @Override public void exitEveryRule(ParserRuleContext\< > ctx) { } - @Override public void visitTerminal( symbol) { } + @Override public void visitTerminal(ParserRuleContext\< > ctx, symbol) { } } >> diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java index 8867bc0ad..da274af84 100644 --- a/tool/test/org/antlr/v4/test/TestPerformance.java +++ b/tool/test/org/antlr/v4/test/TestPerformance.java @@ -43,7 +43,7 @@ public class TestPerformance extends BaseTest { /** * Use ParseTreeWalker.DEFAULT.walk with the BlankJavaParserListener to show parse tree walking overhead. * If {@link #BUILD_PARSE_TREES} is false, the listener will instead be called during the parsing process via - * {@link org.antlr.v4.runtime.Parser#setListener}. + * {@link org.antlr.v4.runtime.Parser#addParseListener}. */ private static final boolean BLANK_LISTENER = false; @@ -262,7 +262,7 @@ public class TestPerformance extends BaseTest { sharedParser = parserCtor.newInstance(tokens); sharedParser.setBuildParseTree(BUILD_PARSE_TREES); if (!BUILD_PARSE_TREES && BLANK_LISTENER) { - sharedParser.setListener(sharedListener); + sharedParser.addParseListener(sharedListener); } if (BAIL_ON_ERROR) { sharedParser.setErrorHandler(new BailErrorStrategy());