forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

added listener unit tests. fixed bug that didn't create ctx getters properly for recursive rules. added Symbol extends Token to parse tree stuff. added visitTerminal to Visitor. recursive alts now track their original, unedited AltAST subtree so we can properly count rule refs etc... later. dup of RuleRefAST was making wrong node. don't gen dispatch methods if no listener.

This commit is contained in:
Terence Parr 2012-02-22 12:44:33 -08:00
parent 58ef729be5
commit 8a34176d82
21 changed files with 430 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runtime/Java/src/org/antlr/v4/runtime/ParseListener.java
View File

@ -8,7 +8,7 @@ package org.antlr.v4.runtime;
* make the interface clear these semantics up. If you need the ctx,
* use Parser.getRuleContext().
*/
public interface ParseListener<Symbol> {
public interface ParseListener<Symbol extends Token> {
void visitTerminal(ParserRuleContext<Symbol> ctx, Symbol symbol);
/** Enter all but left-recursive rules */

45
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
View File

@ -28,11 +28,17 @@
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.*;
import org.antlr.v4.runtime.tree.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeVisitor;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/** A rule invocation record for parsing and tree parsing.
*
@ -57,8 +63,8 @@ import java.util.*;
* group values such as this aggregate. The getters/setters are there to
* satisfy the superclass interface.
*/
public class ParserRuleContext<Symbol> extends RuleContext {
public static final ParserRuleContext<?> EMPTY = new ParserRuleContext<Object>();
public class ParserRuleContext<Symbol extends Token> extends RuleContext {
public static final ParserRuleContext<Token> EMPTY = new ParserRuleContext<Token>();
/** If we are debugging or building a parse tree for a visitor,
* we need to track all of the tokens and rule invocations associated
@ -137,7 +143,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
public void exitRule(ParseTreeListener<Symbol> listener) { }
// visitor
public <T> T accept(ParseTreeVisitor<? extends T> visitor) { visitor.visitChildren(this); return null; }
public <T> T accept(ParseTreeVisitor<? extends T> visitor) { return visitor.visitChildren(this); }
/** Does not set parent link; other add methods do */
@ -209,13 +215,11 @@ public class ParserRuleContext<Symbol> extends RuleContext {
for (ParseTree o : children) {
if ( o instanceof TerminalNode<?> ) {
TerminalNode<?> tnode = (TerminalNode<?>)o;
if ( tnode.getSymbol() instanceof Token ) {
Token symbol = (Token)tnode.getSymbol();
if ( symbol.getType()==ttype ) {
j++;
if ( j == i ) {
return symbol;
}
Token symbol = tnode.getSymbol();
if ( symbol.getType()==ttype ) {
j++;
if ( j == i ) {
return symbol;
}
}
}
@ -233,13 +237,11 @@ public class ParserRuleContext<Symbol> extends RuleContext {
for (ParseTree o : children) {
if ( o instanceof TerminalNode<?> ) {
TerminalNode<?> tnode = (TerminalNode<?>)o;
if ( tnode.getSymbol() instanceof Token ) {
Token symbol = (Token)tnode.getSymbol();
if ( tokens==null ) {
tokens = new ArrayList<Token>();
}
tokens.add(symbol);
Token symbol = tnode.getSymbol();
if ( tokens==null ) {
tokens = new ArrayList<Token>();
}
tokens.add(symbol);
}
}
@ -298,9 +300,6 @@ public class ParserRuleContext<Symbol> extends RuleContext {
String ruleName = recog.getRuleNames()[s.ruleIndex];
buf.append(ruleName);
if ( p.parent != null ) buf.append(" ");
// ATNState invoker = atn.states.get(ctx.invokingState);
// RuleTransition rt = (RuleTransition)invoker.transition(0);
// buf.append(recog.getRuleNames()[rt.target.ruleIndex]);
p = (ParserRuleContext<?>)p.parent;
}
buf.append("]");

25
runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java
View File

@ -45,11 +45,11 @@ public interface ParseTree extends SyntaxTree {
RuleContext getRuleContext();
}
public interface TerminalNode<Symbol> extends ParseTree {
public interface TerminalNode<Symbol extends Token> extends ParseTree {
Symbol getSymbol();
}
public static class TerminalNodeImpl<Symbol> implements TerminalNode<Symbol> {
public static class TerminalNodeImpl<Symbol extends Token> implements TerminalNode<Symbol> {
public Symbol symbol;
public ParseTree parent;
/** Which ATN node matched this token? */
@ -72,13 +72,7 @@ public interface ParseTree extends SyntaxTree {
public Interval getSourceInterval() {
if ( symbol ==null ) return Interval.INVALID;
if (symbol instanceof Token) {
return new Interval(((Token)symbol).getStartIndex(), ((Token)symbol).getStopIndex());
} else if (symbol instanceof SyntaxTree) {
return ((SyntaxTree)symbol).getSourceInterval();
} else {
throw new UnsupportedOperationException("This symbol type is not supported by the default implementation.");
}
return new Interval(symbol.getStartIndex(), symbol.getStopIndex());
}
@Override
@ -86,13 +80,8 @@ public interface ParseTree extends SyntaxTree {
@Override
public String toString() {
if (symbol instanceof Token) {
if ( ((Token)symbol).getType() == Token.EOF ) return "<EOF>";
return ((Token)symbol).getText();
}
else {
throw new UnsupportedOperationException("This symbol type is not supported by the default implementation.");
}
if ( symbol.getType() == Token.EOF ) return "<EOF>";
return symbol.getText();
}
@Override
@ -107,12 +96,10 @@ public interface ParseTree extends SyntaxTree {
* and deletion as well as during "consume until error recovery set"
* upon no viable alternative exceptions.
*/
public static class ErrorNodeImpl<Symbol> extends TerminalNodeImpl<Symbol> {
public static class ErrorNodeImpl<Symbol extends Token> extends TerminalNodeImpl<Symbol> {
public ErrorNodeImpl(Symbol token) {
super(token);
}
// @Override
// public String toString() { return "<ERROR:"+super.toString()+">"; }
}
// the following methods narrow the return type; they are not additional methods

3
runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java
View File

@ -30,8 +30,9 @@
package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
public interface ParseTreeListener<Symbol> {
public interface ParseTreeListener<Symbol extends Token> {
void visitTerminal(ParserRuleContext<Symbol> ctx, Symbol symbol);
void enterEveryRule(ParserRuleContext<Symbol> ctx);
void exitEveryRule(ParserRuleContext<Symbol> ctx);

24
runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java
View File

@ -1,6 +1,7 @@
package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
/** T is return type of visit methods. Use T=Void for no return type. */
public class ParseTreeVisitor<T> {
@ -8,19 +9,30 @@ public class ParseTreeVisitor<T> {
return ctx.accept(this);
}
/** Visit all rule, nonleaf children. Not useful if you are using T as
* non-Void. This returns nothing, losing all computations from below.
* But handy if you are just walking the tree with a visitor and only
/** Visit all rule, nonleaf children. Not that useful if you are using T as
* non-Void. This returns value returned from last child visited,
* losing all computations from first n-1 children. Works fine for
* ctxs with one child then.
* Handy if you are just walking the tree with a visitor and only
* care about some nodes. The ParserRuleContext.accept() method
* walks all children by default; i.e., calls this method.
*/
public <Symbol> void visitChildren(ParserRuleContext<Symbol> ctx) {
public T visitChildren(ParserRuleContext<? extends Token> ctx) {
T result = null;
for (ParseTree c : ctx.children) {
if ( c instanceof ParseTree.RuleNode) {
ParseTree.RuleNode r = (ParseTree.RuleNode)c;
ParserRuleContext<Symbol> rctx = (ParserRuleContext<Symbol>)r.getRuleContext();
visit(rctx);
ParserRuleContext<?> rctx = (ParserRuleContext<? extends Token>)r.getRuleContext();
result = visit(rctx);
}
else {
result = visitTerminal(ctx, ((ParseTree.TerminalNode<? extends Token>)c).getSymbol());
}
}
return result;
}
public T visitTerminal(ParserRuleContext<? extends Token> ctx, Token symbol) {
return null;
}
}

9
runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java
View File

@ -30,12 +30,13 @@
package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
public class ParseTreeWalker {
public static final ParseTreeWalker DEFAULT = new ParseTreeWalker();
@SuppressWarnings("unchecked")
public <Symbol> void walk(ParseTreeListener<Symbol> listener, ParseTree t) {
public <Symbol extends Token> void walk(ParseTreeListener<Symbol> listener, ParseTree t) {
if ( t instanceof ParseTree.TerminalNode) {
visitTerminal(listener, (ParseTree.TerminalNode<Symbol>) t);
return;
@ -50,7 +51,7 @@ public class ParseTreeWalker {
}
@SuppressWarnings("unchecked")
protected <Symbol> void visitTerminal(ParseTreeListener<Symbol> listener,
protected <Symbol extends Token> void visitTerminal(ParseTreeListener<Symbol> listener,
ParseTree.TerminalNode<Symbol> t)
{
ParseTree.RuleNode r = (ParseTree.RuleNode)t.getParent();
@ -66,14 +67,14 @@ public class ParseTreeWalker {
* First we trigger the generic and then the rule specific.
* We to them in reverse order upon finishing the node.
*/
protected <Symbol> void enterRule(ParseTreeListener<Symbol> listener, ParseTree.RuleNode r) {
protected <Symbol extends Token> void enterRule(ParseTreeListener<Symbol> listener, ParseTree.RuleNode r) {
@SuppressWarnings("unchecked")
ParserRuleContext<Symbol> ctx = (ParserRuleContext<Symbol>)r.getRuleContext();
listener.enterEveryRule(ctx);
ctx.enterRule(listener);
}
protected <Symbol> void exitRule(ParseTreeListener<Symbol> listener, ParseTree.RuleNode r) {
protected <Symbol extends Token> void exitRule(ParseTreeListener<Symbol> listener, ParseTree.RuleNode r) {
@SuppressWarnings("unchecked")
ParserRuleContext<Symbol> ctx = (ParserRuleContext<Symbol>)r.getRuleContext();
ctx.exitRule(listener);

56
tool/playground/Errors.g
View File

@ -1,56 +0,0 @@
grammar Errors;
stat: 'return' INT
| ID '=' expr ';'
| ID '(' expr (',' expr)* ')' ';'
/ ID .* '(' expr (',' expr)* ')' ';'
/ ID '=' .* ';' // bad assignment
/ .* ';' // bad stat
/ .* // match anything else? when to stop?
/ // match anything else?
;
catch[Exception e] { }
finally { }
// error to match might be diff than how to resynch? maybe just
// include resynch pattern on end of error alt.
/*
Traps any recog exception in anything called from rule or matched in that rule.
a : expr ';'
/ '--' ID ';' // catches any problem in expr or matching ';'
;
If no err alt matches, defaults to normal error mechanism at rule level.
report. resync.
*/
atom: '(' expr ')'
| INT
/ '(' expr // missing RP; how to resync?
/ '(' ')'
;
// do error alts affect FOLLOW sync sets? nope.
// foo -> bar says how to make resulting tree for bad alts
expr: atom ('*' atom)* ;
atom: INT ;
ID : 'a'..'z'+ ;
WS : (' '|'\n')* ;
/*
Stop .* when it sees any viable following token, even if it uses FOLLOW. So,
err alt
/ .*
would match until it sees something in FOLLOW (but not context-sensitive follow).
actually maybe it would be sensitive; just use real outer context when matching
error alts. who cares about speed.
*/

40
tool/playground/T.g
View File

@ -1,19 +1,27 @@
grammar T;
/* This is ambig too.
s_ : s EOF ;
s : a s
|
;
@members {
public static class LeafListener extends TBaseListener {
public void exitA(TParser.EContext ctx) {
/*
if (ctx.getChildCount()==3) {
System.out.printf("%s %s %s",ctx.e(0).start.getText(),
ctx.e(1).start.getText(),ctx.e().get(0).start.getText());
}
else System.out.println(ctx.INT(0).start.getText());
*/
s : (a)* EOF ; // ambig; can match A B in alt 3 or alt 2 then alt 1
a : e '!'
| e
}
}}
s
@init {setBuildParseTree(true);}
@after { System.out.println($r.ctx.toStringTree(this)); ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(new LeafListener(), $r.ctx);}
: r=e ;
e : e op='*' e
| e op='+' e
| e '++'
| INT
;
e : B
| A // both alts 2,3 can reach end of s upon abEOF
| A B
;
A : 'a' ;
B : 'b' ;
WS : (' '|'\n')+ {skip();} ;
MULT: '*' ;
ADD : '+' ;
INT : [0-9]+ ;
WS : [ \t\n]+ -> skip ;

4
tool/playground/TestT.java
View File

@ -40,8 +40,8 @@ public class TestT {
}
TParser p = new TParser(tokens);
p.setBuildParseTree(true);
final TParser.sContext tree = p.s();
System.out.println(tree.toStringTree(p));
// final TParser.sContext tree = p.s();
// System.out.println(tree.toStringTree(p));
// TreeViewer v = new TreeViewer(p, tree);
// v.setHighlightedBoxColor(TreeViewer.LIGHT_RED);
// v.addHighlightedNodes(new ArrayList<Tree>() {{

4
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -101,10 +101,12 @@ BaseVisitorFile(file, header) ::= <<
<header>
import org.antlr.v4.runtime.tree.*;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.ParserRuleContext;
public class <file.grammarName>BaseVisitor\<T> extends ParseTreeVisitor\<T> implements <file.grammarName>Visitor\<T> {
<file.visitorNames:{lname |
@Override public T visit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { visitChildren(ctx); return null; \}}; separator="\n">
@Override public T visit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { return visitChildren(ctx); \}}; separator="\n">
@Override public T visitTerminal(ParserRuleContext\<? extends Token> ctx, Token symbol) { return null; }
}
>>

47
tool/src/org/antlr/v4/Tool.java
View File

@ -29,20 +29,50 @@
package org.antlr.v4;
import org.antlr.runtime.*;
import org.antlr.runtime.ANTLRFileStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.ParserRuleReturnScope;
import org.antlr.runtime.RecognitionException;
import org.antlr.v4.analysis.AnalysisPipeline;
import org.antlr.v4.automata.*;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenPipeline;
import org.antlr.v4.parse.*;
import org.antlr.v4.runtime.misc.*;
import org.antlr.v4.parse.ANTLRLexer;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.ToolANTLRParser;
import org.antlr.v4.runtime.misc.LogManager;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.ANTLRToolListener;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.DefaultToolListener;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarTransformPipeline;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTErrorNode;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.stringtemplate.v4.STGroup;
import java.io.*;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.Field;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
public class Tool {
public String VERSION = "4.0-"+new Date();
@ -83,7 +113,6 @@ public class Tool {
public boolean force_atn = false;
public boolean log = false;
public boolean verbose_dfa = false;
public boolean no_auto_element_labels = false;
public boolean gen_listener = true;
public boolean gen_parse_listener = false;
public boolean gen_visitor = false;

12
tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java
View File

@ -36,17 +36,23 @@ public class LeftRecursiveRuleAltInfo {
public String leftRecursiveRuleRefLabel;
public String altLabel;
public String altText;
public AltAST altAST;
public AltAST altAST; // transformed ALT
public AltAST originalAltAST;
public int nextPrec;
public LeftRecursiveRuleAltInfo(int altNum, String altText) {
this(altNum, altText, null, null);
this(altNum, altText, null, null, null);
}
public LeftRecursiveRuleAltInfo(int altNum, String altText, String leftRecursiveRuleRefLabel, String altLabel) {
public LeftRecursiveRuleAltInfo(int altNum, String altText,
String leftRecursiveRuleRefLabel,
String altLabel,
AltAST originalAltAST)
{
this.altNum = altNum;
this.altText = altText;
this.leftRecursiveRuleRefLabel = leftRecursiveRuleRefLabel;
this.altLabel = altLabel;
this.originalAltAST = originalAltAST;
}
}

58
tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java
View File

@ -29,17 +29,28 @@
package org.antlr.v4.analysis;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.*;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.TokenStream;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.Tool;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.misc.Pair;
import org.antlr.v4.parse.*;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.LeftRecursiveRuleWalker;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.ast.*;
import org.stringtemplate.v4.*;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTWithOptions;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupFile;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/** Using a tree walker on the rules, determine if a rule is directly left-recursive and if it follows
* our pattern.
@ -124,8 +135,8 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
}
@Override
public void binaryAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
public void binaryAlt(AltAST originalAltTree, int alt) {
AltAST altTree = (AltAST)originalAltTree.dupTree();
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
GrammarAST lrlabel = stripLeftRecursion(altTree);
@ -143,7 +154,8 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
stripAltLabel(altTree);
String altText = text(altTree);
altText = altText.trim();
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel);
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, originalAltTree);
a.nextPrec = nextPrec;
binaryAlts.put(alt, a);
//System.out.println("binaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
@ -151,8 +163,8 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
/** Convert e ? e : e -> ? e : e_[nextPrec] */
@Override
public void ternaryAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
public void ternaryAlt(AltAST originalAltTree, int alt) {
AltAST altTree = (AltAST)originalAltTree.dupTree();
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
GrammarAST lrlabel = stripLeftRecursion(altTree);
@ -168,15 +180,16 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
String altText = text(altTree);
altText = altText.trim();
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel);
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, originalAltTree);
a.nextPrec = nextPrec;
ternaryAlts.put(alt, a);
//System.out.println("ternaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void prefixAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
public void prefixAlt(AltAST originalAltTree, int alt) {
AltAST altTree = (AltAST)originalAltTree.dupTree();
stripAltLabel(altTree);
int nextPrec = precedence(alt);
@ -185,15 +198,16 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
String altText = text(altTree);
altText = altText.trim();
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel);
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, originalAltTree);
a.nextPrec = nextPrec;
prefixAlts.add(a);
//System.out.println("prefixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void suffixAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
public void suffixAlt(AltAST originalAltTree, int alt) {
AltAST altTree = (AltAST)originalAltTree.dupTree();
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
GrammarAST lrlabel = stripLeftRecursion(altTree);
@ -204,18 +218,20 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
stripAltLabel(altTree);
String altText = text(altTree);
altText = altText.trim();
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel);
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, originalAltTree);
suffixAlts.put(alt, a);
// System.out.println("suffixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void otherAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
public void otherAlt(AltAST originalAltTree, int alt) {
AltAST altTree = (AltAST)originalAltTree.dupTree();
stripAltLabel(altTree);
String altText = text(altTree);
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel);
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, originalAltTree);
otherAlts.add(a);
// System.out.println("otherAlt " + alt + ": " + altText);
}

39
tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java
View File

@ -29,14 +29,37 @@
package org.antlr.v4.analysis;
import org.antlr.runtime.*;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.ParserRuleReturnScope;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.TokenStream;
import org.antlr.v4.Tool;
import org.antlr.v4.misc.*;
import org.antlr.v4.parse.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.misc.OrderedHashMap;
import org.antlr.v4.misc.Pair;
import org.antlr.v4.parse.ANTLRLexer;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.ScopeParser;
import org.antlr.v4.parse.ToolANTLRParser;
import org.antlr.v4.tool.AttributeDict;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarTransformPipeline;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.antlr.v4.tool.ast.RuleAST;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/** Remove left-recursive rule refs, add precedence args to recursive rule refs.
* Rewrite rule so we can create ATN.
@ -201,12 +224,16 @@ public class LeftRecursiveRuleTransformer {
LeftRecursiveRuleAltInfo altInfo = r.recPrimaryAlts.get(i);
altInfo.altAST = (AltAST)primaryBlk.getChild(i);
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.parent = altInfo.altAST.parent;
// System.out.println(altInfo.altAST.toStringTree());
}
for (int i = 0; i < r.recOpAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.getElement(i);
altInfo.altAST = (AltAST)opsBlk.getChild(i);
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.parent = altInfo.altAST.parent;
// System.out.println(altInfo.altAST.toStringTree());
}
}

9
tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java
View File

@ -29,11 +29,14 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.*;
import org.antlr.v4.codegen.model.decl.*;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.decl.RuleContextDecl;
import org.antlr.v4.codegen.model.decl.StructDecl;
import org.antlr.v4.misc.Pair;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.GrammarAST;
public class LeftRecursiveRuleFunction extends RuleFunction {

66
tool/src/org/antlr/v4/codegen/model/RuleFunction.java
View File

@ -30,7 +30,15 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.decl.*;
import org.antlr.v4.codegen.model.decl.AltLabelStructDecl;
import org.antlr.v4.codegen.model.decl.ContextRuleGetterDecl;
import org.antlr.v4.codegen.model.decl.ContextRuleListGetterDecl;
import org.antlr.v4.codegen.model.decl.ContextRuleListIndexedGetterDecl;
import org.antlr.v4.codegen.model.decl.ContextTokenGetterDecl;
import org.antlr.v4.codegen.model.decl.ContextTokenListGetterDecl;
import org.antlr.v4.codegen.model.decl.ContextTokenListIndexedGetterDecl;
import org.antlr.v4.codegen.model.decl.Decl;
import org.antlr.v4.codegen.model.decl.StructDecl;
import org.antlr.v4.misc.FrequencySet;
import org.antlr.v4.misc.Triple;
import org.antlr.v4.misc.Utils;
@ -42,9 +50,18 @@ import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.antlr.v4.parse.ANTLRParser.*;
import static org.antlr.v4.parse.ANTLRParser.CLOSURE;
import static org.antlr.v4.parse.ANTLRParser.POSITIVE_CLOSURE;
import static org.antlr.v4.parse.ANTLRParser.RULE_REF;
import static org.antlr.v4.parse.ANTLRParser.TOKEN_REF;
/** */
public class RuleFunction extends OutputModelObject {
@ -84,13 +101,11 @@ public class RuleFunction extends OutputModelObject {
altToContext = new AltLabelStructDecl[r.getOriginalNumberOfAlts()+1];
// Add ctx labels for elements in alts with no -> label
if ( !factory.getGrammar().tool.no_auto_element_labels ) {
List<AltAST> altsNoLabels = r.getUnlabeledAltASTs();
if ( altsNoLabels!=null ) {
Set<Decl> decls = getDeclsForAllElements(altsNoLabels);
// we know to put in rule ctx, so do it directly
for (Decl d : decls) ruleCtx.addDecl(d);
}
List<AltAST> altsNoLabels = r.getUnlabeledAltASTs();
if ( altsNoLabels!=null ) {
Set<Decl> decls = getDeclsForAllElements(altsNoLabels);
// we know to put in rule ctx, so do it directly
for (Decl d : decls) ruleCtx.addDecl(d);
}
// make structs for -> labeled alts, define ctx labels for elements
@ -103,11 +118,9 @@ public class RuleFunction extends OutputModelObject {
String label = pair.c;
altToContext[altNum] = new AltLabelStructDecl(factory, r, altNum, label);
altLabelCtxs.put(label, altToContext[altNum]);
if ( !factory.getGrammar().tool.no_auto_element_labels ) {
Set<Decl> decls = getDeclsForAltElements(altAST);
// we know which ctx to put in, so do it directly
for (Decl d : decls) altToContext[altNum].addDecl(d);
}
Set<Decl> decls = getDeclsForAltElements(altAST);
// we know which ctx to put in, so do it directly
for (Decl d : decls) altToContext[altNum].addDecl(d);
}
}
@ -139,23 +152,18 @@ public class RuleFunction extends OutputModelObject {
}
}
/** for all alts, find which ref X or r in way which needs List
/** for all alts, find which ref X or r needs List
Must see across alts. If any alt needs X or r as list, then
define as list.
*/
public Set<Decl> getDeclsForAllElements(List<AltAST> altASTs) {
Set<String> needsList = new HashSet<String>();
List<GrammarAST> allRefs = new ArrayList<GrammarAST>();
// for (Alternative a :alts) {
for (AltAST ast : altASTs) {
IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF);
List<GrammarAST> refs = ast.getNodesWithType(reftypes);
FrequencySet<String> altFreq = new FrequencySet<String>();
for (GrammarAST t : refs) {
String refLabelName = t.getText();
altFreq.add(refLabelName);
allRefs.add(t);
}
allRefs.addAll(refs);
FrequencySet<String> altFreq = getElementFrequenciesForAlt(ast);
for (GrammarAST t : refs) {
String refLabelName = t.getText();
if ( altFreq.count(t.getText())>1 ) needsList.add(refLabelName);
@ -172,6 +180,18 @@ public class RuleFunction extends OutputModelObject {
return decls;
}
/** Given list of X and r refs in alt, compute how many of each there are */
protected FrequencySet<String> getElementFrequenciesForAlt(AltAST ast) {
IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF);
List<GrammarAST> refs = ast.getNodesWithType(reftypes);
FrequencySet<String> altFreq = new FrequencySet<String>();
for (GrammarAST t : refs) {
String refLabelName = t.getText();
altFreq.add(refLabelName);
}
return altFreq;
}
/** Get list of decls for token/rule refs.
* Single ref X becomes X() getter
* Multiple refs to X becomes List X() method, X(int i) method.

21
tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java
View File

@ -30,11 +30,20 @@
package org.antlr.v4.codegen.model.decl;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.codegen.model.DispatchMethod;
import org.antlr.v4.codegen.model.ListenerDispatchMethod;
import org.antlr.v4.codegen.model.ModelElement;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.codegen.model.ParseListenerDispatchMethod;
import org.antlr.v4.codegen.model.VisitorDispatchMethod;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Attribute;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/** This object models the structure holding all of the parameters,
* return values, local variables, and labels associated with a rule.
@ -60,8 +69,10 @@ public class StructDecl extends Decl {
dispatchMethods = new ArrayList<DispatchMethod>();
if ( !r.hasAltSpecificContexts() ) {
// no enter/exit for this ruleContext if rule has labels
dispatchMethods.add(new ListenerDispatchMethod(factory, true));
dispatchMethods.add(new ListenerDispatchMethod(factory, false));
if ( factory.getGrammar().tool.gen_listener ) {
dispatchMethods.add(new ListenerDispatchMethod(factory, true));
dispatchMethods.add(new ListenerDispatchMethod(factory, false));
}
if ( factory.getGrammar().tool.gen_visitor ) {
dispatchMethods.add(new VisitorDispatchMethod(factory));
}

12
tool/src/org/antlr/v4/tool/LeftRecursiveRule.java
View File

@ -72,11 +72,11 @@ public class LeftRecursiveRule extends Rule {
List<AltAST> alts = new ArrayList<AltAST>();
for (int i = 0; i < recPrimaryAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = recPrimaryAlts.get(i);
if ( altInfo.altLabel==null ) alts.add(altInfo.altAST);
if ( altInfo.altLabel==null ) alts.add(altInfo.originalAltAST);
}
for (int i = 0; i < recOpAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = recOpAlts.getElement(i);
if ( altInfo.altLabel==null ) alts.add(altInfo.altAST);
if ( altInfo.altLabel==null ) alts.add(altInfo.originalAltAST);
}
if ( alts.size()==0 ) return null;
return alts;
@ -91,13 +91,17 @@ public class LeftRecursiveRule extends Rule {
for (int i = 0; i < recPrimaryAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = recPrimaryAlts.get(i);
if ( altInfo.altLabel!=null ) {
labels.add(new Triple<Integer,AltAST,String>(altInfo.altNum,altInfo.altAST,altInfo.altLabel));
labels.add(new Triple<Integer,AltAST,String>(altInfo.altNum,
altInfo.originalAltAST,
altInfo.altLabel));
}
}
for (int i = 0; i < recOpAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = recOpAlts.getElement(i);
if ( altInfo.altLabel!=null ) {
labels.add(new Triple<Integer,AltAST,String>(altInfo.altNum,altInfo.altAST,altInfo.altLabel));
labels.add(new Triple<Integer,AltAST,String>(altInfo.altNum,
altInfo.originalAltAST,
altInfo.altLabel));
}
}
if ( labels.size()==0 ) return null;

1
tool/src/org/antlr/v4/tool/ast/AltAST.java
View File

@ -50,6 +50,7 @@ public class AltAST extends GrammarAST {
super(node);
this.alt = ((AltAST)node).alt;
this.altLabel = ((AltAST)node).altLabel;
this.leftRecursiveAltInfo = ((AltAST)node).leftRecursiveAltInfo;
}
public AltAST(Token t) { super(t); }

8
tool/src/org/antlr/v4/tool/ast/RuleRefAST.java
View File

@ -29,6 +29,7 @@
package org.antlr.v4.tool.ast;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.Token;
import org.antlr.runtime.tree.Tree;
@ -41,8 +42,13 @@ public class RuleRefAST extends GrammarASTWithOptions implements RuleElementAST
public RuleRefAST(int type) { super(type); }
public RuleRefAST(int type, Token t) { super(type, t); }
/** Dup token too since we overwrite during LR rule transform */
@Override
public Tree dupNode() { return new TerminalAST(this); }
public Tree dupNode() {
RuleRefAST r = new RuleRefAST(this);
r.token = new CommonToken(r.token);
return r;
}
@Override
public Object visit(GrammarASTVisitor v) { return v.visit(this); }

149
tool/test/org/antlr/v4/test/TestListeners.java Normal file
View File

@ -0,0 +1,149 @@
package org.antlr.v4.test;
import org.junit.Test;
public class TestListeners extends BaseTest {
@Test public void testBasic() throws Exception {
String grammar =
"grammar T;\n" +
"@members {\n" +
"public static class LeafListener extends TBaseListener {\n" +
" public void visitTerminal(ParserRuleContext<Token> ctx, Token symbol) {\n" +
" System.out.println(symbol.getText());\n" +
" }\n" +
" }}\n" +
"s\n" +
"@init {setBuildParseTree(true);}\n" +
"@after {" +
" System.out.println($r.ctx.toStringTree(this));" +
" ParseTreeWalker walker = new ParseTreeWalker();\n" +
" walker.walk(new LeafListener(), $r.ctx);" +
"}\n" +
" : r=a ;\n" +
"a : INT INT" +
" | ID" +
" ;\n" +
"MULT: '*' ;\n" +
"ADD : '+' ;\n" +
"INT : [0-9]+ ;\n" +
"ID : [a-z]+ ;\n" +
"WS : [ \\t\\n]+ -> skip ;\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1 2", false);
String expecting = "(a 1 2)\n" +
"1\n" +
"2\n";
assertEquals(expecting, result);
}
@Test public void testTokenGetters() throws Exception {
String grammar =
"grammar T;\n" +
"@members {\n" +
"public static class LeafListener extends TBaseListener {\n" +
" public void exitA(TParser.AContext ctx) {\n" +
" if (ctx.getChildCount()==2) System.out.printf(\"%s %s %s\",ctx.INT(0).getText(),ctx.INT(1).getText(),ctx.INT());\n" +
" else System.out.println(ctx.ID());\n" +
" }\n" +
" }}\n" +
"s\n" +
"@init {setBuildParseTree(true);}\n" +
"@after {" +
" System.out.println($r.ctx.toStringTree(this));" +
" ParseTreeWalker walker = new ParseTreeWalker();\n" +
" walker.walk(new LeafListener(), $r.ctx);" +
"}\n" +
" : r=a ;\n" +
"a : INT INT" +
" | ID" +
" ;\n" +
"MULT: '*' ;\n" +
"ADD : '+' ;\n" +
"INT : [0-9]+ ;\n" +
"ID : [a-z]+ ;\n" +
"WS : [ \\t\\n]+ -> skip ;\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1 2", false);
String expecting = "(a 1 2)\n" +
"1 2 [[@0,0:0='1',<5>,1:0], [@1,2:2='2',<5>,1:2]]\n";
assertEquals(expecting, result);
result = execParser("T.g", grammar, "TParser", "TLexer", "s", "abc", false);
expecting = "(a abc)\n" +
"[@0,0:2='abc',<6>,1:0]\n";
assertEquals(expecting, result);
}
@Test public void testRuleGetters() throws Exception {
String grammar =
"grammar T;\n" +
"@members {\n" +
"public static class LeafListener extends TBaseListener {\n" +
" public void exitA(TParser.AContext ctx) {\n" +
" if (ctx.getChildCount()==2) {\n" +
" System.out.printf(\"%s %s %s\",ctx.b(0).start.getText(),\n" +
" ctx.b(1).start.getText(),ctx.b().get(0).start.getText());\n" +
" }\n" +
" else System.out.println(ctx.b(0).start.getText());\n" +
" }\n" +
" }}\n" +
"s\n" +
"@init {setBuildParseTree(true);}\n" +
"@after {" +
" System.out.println($r.ctx.toStringTree(this));" +
" ParseTreeWalker walker = new ParseTreeWalker();\n" +
" walker.walk(new LeafListener(), $r.ctx);" +
"}\n" +
" : r=a ;\n" +
"a : b b" + // forces list
" | b" + // a list still
" ;\n" +
"b : ID | INT ;\n" +
"MULT: '*' ;\n" +
"ADD : '+' ;\n" +
"INT : [0-9]+ ;\n" +
"ID : [a-z]+ ;\n" +
"WS : [ \\t\\n]+ -> skip ;\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1 2", false);
String expecting = "(a (b 1) (b 2))\n" +
"1 2 1\n";
assertEquals(expecting, result);
result = execParser("T.g", grammar, "TParser", "TLexer", "s", "abc", false);
expecting = "(a (b abc))\n" +
"abc\n";
assertEquals(expecting, result);
}
@Test public void testLR() throws Exception {
String grammar =
"grammar T;\n" +
"@members {\n" +
"public static class LeafListener extends TBaseListener {\n" +
" public void exitA(TParser.EContext ctx) {\n" +
" if (ctx.getChildCount()==3) {\n" +
" System.out.printf(\"%s %s %s\",ctx.e(0).start.getText(),\n" +
" ctx.e(1).start.getText(),ctx.e().get(0).start.getText());\n" +
" }\n" +
" else System.out.println(ctx.INT().getText());\n" +
" }\n" +
" }}\n" +
"s\n" +
"@init {setBuildParseTree(true);}\n" +
"@after {" +
" System.out.println($r.ctx.toStringTree(this));" +
" ParseTreeWalker walker = new ParseTreeWalker();\n" +
" walker.walk(new LeafListener(), $r.ctx);" +
"}\n" +
" : r=e ;\n" +
"e : e op='*' e\n" +
" | e op='+' e\n" +
" | INT\n" +
" ;\n" +
"MULT: '*' ;\n" +
"ADD : '+' ;\n" +
"INT : [0-9]+ ;\n" +
"WS : [ \\t\\n]+ -> skip ;\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1+2*3", false);
String expecting = "(e (e 1) + (e (e 2) * (e 3)))\n";
assertEquals(expecting, result);
}
}