added -parse-listener option and generated code if option on. parse listener differs from tree listener

This commit is contained in:
Terence Parr 2012-02-18 16:49:05 -08:00
parent 71b75c88dd
commit 4e8931519c
13 changed files with 226 additions and 153 deletions

View File

@ -0,0 +1,18 @@
package org.antlr.v4.runtime;
/** We must distinguish between listeners triggered during the parse
* from listeners triggered during a subsequent tree walk. During
* the parse, the ctx object arg for enter methods don't have any labels set.
* We can only access the general ParserRuleContext<Symbol> ctx.
* Also, we can only call exit methods for left-recursive rules. Let's
* make the interface clear these semantics up. If you need the ctx,
* use Parser.getRuleContext().
*/
public interface ParseListener<Symbol> {
void visitTerminal(ParserRuleContext<Symbol> ctx, Symbol symbol);
/** Enter all but left-recursive rules */
void enterNonLRRule(ParserRuleContext<Symbol> ctx);
void exitEveryRule(ParserRuleContext<Symbol> ctx);
}

View File

@ -30,18 +30,15 @@ package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.misc.*;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
/** This is all the parsing support code essentially; most of it is error recovery stuff. */
public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>> {
public class TraceListener implements ParseTreeListener<Token> {
public class TraceListener implements ParseListener<Token> {
@Override
public void enterEveryRule(ParserRuleContext<Token> ctx) {
public void enterNonLRRule(ParserRuleContext<Token> ctx) {
System.out.println("enter " + getRuleNames()[ctx.ruleIndex] + ", LT(1)=" + _input.LT(1).getText());
}
@ -75,7 +72,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
* the parse or during tree walks later. Both could be done.
* Not intended for tree parsing but would work.
*/
protected List<ParseTreeListener<Token>> _parseListeners;
protected List<ParseListener<Token>> _parseListeners;
/** Did the recognizer encounter a syntax error? Track how many. */
protected int _syntaxErrors = 0;
@ -154,19 +151,19 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
// return traceATNStates;
// }
public List<ParseTreeListener<Token>> getParseListeners() {
public List<ParseListener<Token>> getParseListeners() {
return _parseListeners;
}
public void addParseListener(ParseTreeListener<Token> listener) {
public void addParseListener(ParseListener<Token> listener) {
if ( listener==null ) return;
if ( _parseListeners==null ) {
_parseListeners = new ArrayList<ParseTreeListener<Token>>();
_parseListeners = new ArrayList<ParseListener<Token>>();
}
this._parseListeners.add(listener);
}
public void removeParseListener(ParseTreeListener<Token> l) {
public void removeParseListener(ParseListener<Token> l) {
if ( l==null ) return;
if ( _parseListeners!=null ) _parseListeners.remove(l);
}
@ -174,8 +171,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
public void removeParseListeners() { if ( _parseListeners!=null ) _parseListeners.clear(); }
public void triggerEnterRuleEvent() {
for (ParseTreeListener<Token> l : _parseListeners) {
l.enterEveryRule(_ctx);
for (ParseListener<Token> l : _parseListeners) {
l.enterNonLRRule(_ctx);
_ctx.enterRule(l);
}
}
@ -183,7 +180,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
public void triggerExitRuleEvent() {
// reverse order walk of listeners
for (int i = _parseListeners.size()-1; i >= 0; i--) {
ParseTreeListener<Token> l = _parseListeners.get(i);
ParseListener<Token> l = _parseListeners.get(i);
_ctx.exitRule(l);
l.exitEveryRule(_ctx);
}
@ -293,7 +290,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
else _ctx.addChild((Token)o);
}
if ( _parseListeners != null) {
for (ParseTreeListener<Token> l : _parseListeners) l.visitTerminal(_ctx, o);
for (ParseListener<Token> l : _parseListeners) l.visitTerminal(_ctx, o);
}
return o;
}
@ -455,6 +452,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
// return atn.nextTokens(s, ctx);
// }
public ParserRuleContext<Token> getRuleContext() { return _ctx; }
/** Return List<String> of the rule names in your parser instance
* leading up to a call to the current rule. You could override if
* you want more details such as the file/line info of where

View File

@ -128,10 +128,18 @@ public class ParserRuleContext<Symbol> extends RuleContext {
// Double dispatch methods for listeners and visitors
// parse listener
public void enterRule(ParseListener<Symbol> listener) { }
public void exitRule(ParseListener<Symbol> listener) { }
// parse tree listener
public void enterRule(ParseTreeListener<Symbol> listener) { }
public void exitRule(ParseTreeListener<Symbol> listener) { }
// visitor
public <T> T accept(ParseTreeVisitor<? extends T> visitor) { visitor.visitChildren(this); return null; }
/** Does not set parent link; other add methods do */
public void addChild(TerminalNode<Symbol> t) {
if ( children==null ) children = new ArrayList<ParseTree>();

View File

@ -145,6 +145,16 @@ public class TestA {
printMethodName(ctx);
}
@Override
public void enterPrimary(AParser.PrimaryContext ctx) {
printMethodName(ctx);
}
@Override
public void exitPrimary(AParser.PrimaryContext ctx) {
printMethodName(ctx);
}
public void printMethodName(ParserRuleContext ctx) {
Throwable t = new Throwable();
StackTraceElement[] stack = t.getStackTrace();
@ -161,7 +171,7 @@ public class TestA {
CommonTokenStream tokens = new CommonTokenStream(lexer);
AParser p = new AParser(tokens);
p.setBuildParseTree(true);
p.addParseListener(new Tracer());
// p.addParseListener(new Tracer());
ParserRuleContext<Token> t = p.s();
System.out.println("tree = "+t.toStringTree(p));

View File

@ -43,25 +43,25 @@ public class TestA2 {
A2Parser p;
public Do(A2Parser p) { this.p = p; }
@Override
public void exit(A2Parser.AddContext ctx) {
public void exitAdd(A2Parser.AddContext ctx) {
ctx.v = ctx.e(0).v + ctx.e(1).v;
System.out.println("Add: " + ctx.v);
}
@Override
public void exit(A2Parser.IntContext ctx) {
public void exitInt(A2Parser.IntContext ctx) {
ctx.v = Integer.valueOf(ctx.INT().getText());
System.out.println("Int: "+ctx.v);
}
@Override
public void exit(A2Parser.MultContext ctx) {
public void exitMult(A2Parser.MultContext ctx) {
ctx.v = ctx.e(0).v * ctx.e(1).v;
System.out.println("Mult: " + ctx.v);
}
@Override
public void exit(A2Parser.ParensContext ctx) {
public void exitParens(A2Parser.ParensContext ctx) {
ctx.v = ctx.e().v;
System.out.println("Parens: "+ctx.v);
}
@ -71,14 +71,13 @@ public class TestA2 {
CommonTokenStream tokens = new CommonTokenStream(lexer);
A2Parser p = new A2Parser(tokens);
p.setBuildParseTree(true);
p.addParseListener(new Do(p));
ParserRuleContext<Token> t = p.s();
System.out.println("tree = "+t.toStringTree(p));
ParseTreeWalker walker = new ParseTreeWalker();
Do doer = new Do(p);
walker.walk(doer, t);
A2Parser.eContext ectx = (A2Parser.eContext)t.getChild(0);
A2Parser.EContext ectx = (A2Parser.EContext)t.getChild(0);
System.out.println("result from tree walk = "+ ectx.v);
}
}

View File

@ -47,12 +47,42 @@ import org.antlr.v4.runtime.Token;
public class <file.grammarName>BaseListener implements <file.grammarName>Listener {
<file.listenerNames:{lname |
@Override public void enter<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { \}
@Override public void exit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { \}}; separator="\n">
public void enter<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { \}
public void exit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { \}}; separator="\n">
@Override public void enterEveryRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
@Override public void exitEveryRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
@Override public void visitTerminal(ParserRuleContext\<<InputSymbolType()>\> ctx, <InputSymbolType()> symbol) { }
public void enterEveryRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
public void exitEveryRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
public void visitTerminal(ParserRuleContext\<<InputSymbolType()>\> ctx, <InputSymbolType()> symbol) { }
}
>>
ParseListenerFile(file, header) ::= <<
<header>
import org.antlr.v4.runtime.tree.*;
import org.antlr.v4.runtime.*;
public interface <file.grammarName>ParseListener extends ParseListener\<<InputSymbolType()>\> {
<file.listenerEnterNames:{lname |
void enter<lname; format="cap">(ParseListener\<<InputSymbolType()>\> ctx);}; separator="\n">
<file.listenerExitNames:{lname |
void exit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx);}; separator="\n">
}
>>
BaseParseListenerFile(file, header) ::= <<
<header>
import org.antlr.v4.runtime.*;
public class <file.grammarName>BaseParseListener implements <file.grammarName>ParseListener {
<file.listenerEnterNames:{lname |
public void enter<lname; format="cap">(ParseListener\<<InputSymbolType()>\> ctx) { \}}; separator="\n">
<file.listenerExitNames:{lname |
public void exit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { \}}; separator="\n">
public void enterNonLRRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
public void exitEveryRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
public void visitTerminal(ParserRuleContext\<<InputSymbolType()>\> ctx, <InputSymbolType()> symbol) { }
}
>>
@ -587,14 +617,12 @@ public static class <struct.name> extends <currentRule.name; format="cap">Contex
>>
ListenerDispatchMethod(method) ::= <<
@Override
public void <if(method.isEnter)>enter<else>exit<endif>Rule(ParseTreeListener\<<InputSymbolType()>\> listener) {
if ( listener instanceof <parser.grammarName>Listener ) ((<parser.grammarName>Listener)listener).<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(this);
}
>>
VisitorDispatchMethod(method) ::= <<
@Override
public \<T> T accept(ParseTreeVisitor\<? extends T> visitor) {
if ( visitor instanceof <parser.grammarName>Visitor ) return ((<parser.grammarName>Visitor\<T>)visitor).visit<struct.derivedFromName; format="cap">(this);
else return null;

View File

@ -31,29 +31,18 @@ package org.antlr.v4;
import org.antlr.runtime.*;
import org.antlr.v4.analysis.AnalysisPipeline;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.automata.*;
import org.antlr.v4.codegen.CodeGenPipeline;
import org.antlr.v4.parse.ANTLRLexer;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.ToolANTLRParser;
import org.antlr.v4.runtime.misc.LogManager;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.parse.*;
import org.antlr.v4.runtime.misc.*;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTErrorNode;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.antlr.v4.tool.ast.*;
import org.stringtemplate.v4.STGroup;
import java.io.*;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.*;
public class Tool {
public String VERSION = "4.0-"+new Date();
@ -96,6 +85,7 @@ public class Tool {
public boolean verbose_dfa = false;
public boolean no_auto_element_labels = false;
public boolean gen_listener = true;
public boolean gen_parse_listener = false;
public boolean gen_visitor = false;
public static Option[] optionDefs = {
@ -110,6 +100,8 @@ public class Tool {
new Option("msgFormat", "-message-format", OptionArgType.STRING, "specify output style for messages"),
new Option("gen_listener", "-listener", "generate parse tree listener (default)"),
new Option("gen_listener", "-no-listener", "don't generate parse tree listener"),
new Option("gen_parse_listener", "-parse-listener", "generate parse listener"),
new Option("gen_parse_listener", "-no-parse-listener", "don't generate parse listener (default)"),
new Option("gen_visitor", "-visitor", "generate parse tree visitor"),
new Option("gen_visitor", "-no-visitor", "don't generate parse tree visitor (default)"),

View File

@ -30,6 +30,7 @@
package org.antlr.v4.codegen;
import org.antlr.v4.tool.Grammar;
import org.stringtemplate.v4.ST;
public class CodeGenPipeline {
Grammar g;
@ -42,14 +43,22 @@ public class CodeGenPipeline {
CodeGenerator gen = new CodeGenerator(g);
if ( g.isLexer() ) {
gen.writeRecognizer(gen.generateLexer());
ST lexer = gen.generateLexer();
if ( g.tool.launch_ST_inspector ) lexer.inspect();
gen.writeRecognizer(lexer);
}
else {
gen.writeRecognizer(gen.generateParser());
ST parser = gen.generateParser();
if ( g.tool.launch_ST_inspector ) parser.inspect();
gen.writeRecognizer(parser);
if ( g.tool.gen_listener ) {
gen.writeListener(gen.generateListener());
gen.writeBaseListener(gen.generateBaseListener());
}
if ( g.tool.gen_parse_listener ) {
gen.writeParseListener(gen.generateParseListener());
gen.writeBaseParseListener(gen.generateBaseParseListener());
}
if ( g.tool.gen_visitor ) {
gen.writeVisitor(gen.generateVisitor());
gen.writeBaseVisitor(gen.generateBaseVisitor());

View File

@ -114,100 +114,35 @@ public class CodeGenerator {
}
// CREATE TEMPLATES BY WALKING MODEL
public ST generateLexer() {
OutputModelFactory factory = new LexerFactory(this);
// CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES
OutputModelController controller = new OutputModelController(factory);
factory.setController(controller);
OutputModelObject outputModel = controller.buildLexerOutputModel();
OutputModelWalker walker = new OutputModelWalker(tool, templates);
ST st = walker.walk(outputModel);
if ( tool.launch_ST_inspector ) {
st.inspect();
//if ( templates.isDefined("headerFile") ) headerFileST.inspect();
}
// String x = ATNSerializer.getDecoded(g, g.atn);
// System.out.println(x);
return st;
}
public ST generateParser() {
public ST generateModelST(String factoryMethod) {
OutputModelFactory factory = new ParserFactory(this);
// CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES
OutputModelController controller = new OutputModelController(factory);
factory.setController(controller);
OutputModelObject outputModel = controller.buildParserOutputModel();
OutputModelObject outputModel = null;
try {
Method m = OutputModelController.class.getDeclaredMethod(factoryMethod);
outputModel = (OutputModelObject)m.invoke(controller);
}
catch (Exception e) {
tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, "can't exec factory method", e);
}
OutputModelWalker walker = new OutputModelWalker(tool, templates);
ST st = walker.walk(outputModel);
if ( tool.launch_ST_inspector ) {
st.inspect();
//if ( templates.isDefined("headerFile") ) headerFileST.inspect();
return walker.walk(outputModel);
}
return st;
}
public ST generateListener() {
OutputModelFactory factory = new ParserFactory(this);
OutputModelController controller = new OutputModelController(factory);
factory.setController(controller);
OutputModelObject listenerModel = controller.buildListenerOutputModel();
OutputModelWalker walker = new OutputModelWalker(tool, templates);
ST st = walker.walk(listenerModel);
return st;
}
public ST generateVisitor() {
OutputModelFactory factory = new ParserFactory(this);
OutputModelController controller = new OutputModelController(factory);
factory.setController(controller);
OutputModelObject visitorModel = controller.buildVisitorOutputModel();
OutputModelWalker walker = new OutputModelWalker(tool, templates);
ST st = walker.walk(visitorModel);
return st;
}
public ST generateBaseListener() {
OutputModelFactory factory = new ParserFactory(this);
OutputModelController controller = new OutputModelController(factory);
factory.setController(controller);
OutputModelObject baseModel = controller.buildBaseListenerOutputModel();
OutputModelWalker walker = new OutputModelWalker(tool, templates);
ST st = walker.walk(baseModel);
return st;
}
public ST generateBaseVisitor() {
OutputModelFactory factory = new ParserFactory(this);
OutputModelController controller = new OutputModelController(factory);
factory.setController(controller);
OutputModelObject baseModel = controller.buildBaseVisitorOutputModel();
OutputModelWalker walker = new OutputModelWalker(tool, templates);
ST st = walker.walk(baseModel);
return st;
}
public ST generateLexer() { return generateModelST("buildLexerOutputModel"); }
public ST generateParser() { return generateModelST("buildParserOutputModel"); }
public ST generateListener() { return generateModelST("buildListenerOutputModel"); }
public ST generateBaseListener() { return generateModelST("buildBaseListenerOutputModel"); }
public ST generateParseListener() { return generateModelST("buildParseListenerOutputModel"); }
public ST generateBaseParseListener() { return generateModelST("buildBaseParseListenerOutputModel"); }
public ST generateVisitor() { return generateModelST("buildVisitorOutputModel"); }
public ST generateBaseVisitor() { return generateModelST("buildBaseVisitorOutputModel"); }
/** Generate a token vocab file with all the token names/types. For example:
* ID=7
@ -253,6 +188,14 @@ public class CodeGenerator {
target.genFile(g,outputFileST, getBaseListenerFileName());
}
public void writeParseListener(ST outputFileST) {
target.genFile(g,outputFileST, getParseListenerFileName());
}
public void writeBaseParseListener(ST outputFileST) {
target.genFile(g,outputFileST, getBaseParseListenerFileName());
}
public void writeVisitor(ST outputFileST) {
target.genFile(g,outputFileST, getVisitorFileName());
}
@ -360,6 +303,20 @@ public class CodeGenerator {
return listenerName+extST.render();
}
public String getParseListenerFileName() {
assert g.name != null;
ST extST = templates.getInstanceOf("codeFileExtension");
String listenerName = g.name + "ParseListener";
return listenerName+extST.render();
}
public String getBaseParseListenerFileName() {
assert g.name != null;
ST extST = templates.getInstanceOf("codeFileExtension");
String listenerName = g.name + "BaseParseListener";
return listenerName+extST.render();
}
/** A given grammar T, return a blank listener implementation
* such as TBaseListener.java, if we're using the Java target.
*/

View File

@ -114,6 +114,16 @@ public class OutputModelController {
return new BaseListenerFile(delegate, gen.getBaseListenerFileName());
}
public OutputModelObject buildParseListenerOutputModel() {
CodeGenerator gen = delegate.getGenerator();
return new ParseListenerFile(delegate, gen.getParseListenerFileName());
}
public OutputModelObject buildBaseParseListenerOutputModel() {
CodeGenerator gen = delegate.getGenerator();
return new BaseParseListenerFile(delegate, gen.getBaseParseListenerFileName());
}
public OutputModelObject buildVisitorOutputModel() {
CodeGenerator gen = delegate.getGenerator();
return new VisitorFile(delegate, gen.getVisitorFileName());

View File

@ -0,0 +1,9 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
public class BaseParseListenerFile extends ParseListenerFile {
public BaseParseListenerFile(OutputModelFactory factory, String fileName) {
super(factory, fileName);
}
}

View File

@ -0,0 +1,46 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.misc.Triple;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import java.util.*;
public class ParseListenerFile extends OutputFile {
public String grammarName;
public String parserName;
public Set<String> listenerEnterNames = new HashSet<String>();
public Set<String> listenerExitNames = new HashSet<String>();
@ModelElement public Action header;
public ParseListenerFile(OutputModelFactory factory, String fileName) {
super(factory, fileName);
Grammar g = factory.getGrammar();
parserName = g.getRecognizerName();
grammarName = g.name;
for (Rule r : g.rules.values()) {
List<Triple<Integer,AltAST,String>> labels = r.getAltLabels();
// EXIT RULES
if ( labels!=null ) {
// add exit rules for alt labels
for (Triple<Integer,AltAST,String> pair : labels) {
listenerExitNames.add(pair.c);
if ( !(r instanceof LeftRecursiveRule) ) {
listenerEnterNames.add(pair.c);
}
}
}
else {
// add exit rule if no labels
listenerExitNames.add(r.name);
if ( !(r instanceof LeftRecursiveRule) ) {
listenerEnterNames.add(r.name);
}
}
}
ActionAST ast = g.namedActions.get("header");
if ( ast!=null ) header = new Action(factory, ast);
}
}

View File

@ -29,30 +29,17 @@
package org.antlr.v4.test;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.*;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.junit.Assert;
import org.junit.Test;
import org.antlr.v4.runtime.tree.*;
import org.junit.*;
import java.io.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
import java.lang.reflect.*;
import java.net.*;
import java.util.*;
import java.util.logging.*;
public class TestPerformance extends BaseTest {
/** Parse all java files under this package within the JDK_SOURCE_ROOT. */
@ -461,7 +448,8 @@ public class TestPerformance extends BaseTest {
sharedParser = parserCtor.newInstance(tokens);
sharedParser.setBuildParseTree(BUILD_PARSE_TREES);
if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
sharedParser.addParseListener(sharedListener);
// TJP commented out for now; changed interface
// sharedParser.addParseListener(sharedListener);
}
if (BAIL_ON_ERROR) {
sharedParser.setErrorHandler(new BailErrorStrategy());