forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

This change is a major restructuring of how left recursive rules are transformed. Previously I simply rewrote the grammar and ANTLR was none the wiser. However, it quickly became apparent that ANTLR needed to do many different things for recursive rules so I had to insert transformation later in the pipeline. Specifically, I needed it after Rule object creation. I made a special LeftRecursiveRule object that track information collected during transformation so that I could use it later. I made major changes to the left recursion templates as well as the little code snippets in Java.stg. I created a new template called LeftRecursiveRuleFunction an accompanying model object that handles the special case, even though there is some duplication. the biggest difference in the grammar is the introduction of => ID notation on any outermost alternative. This information is not added to the tree, instead the ALT node is annotated with the ID. Rule.getAltLabels() now looks also in the LeftRecursiveRuleAltInfo objects. I have moved to the left recursion transformation to its own object and have moved some objects into the analysis package. Further, I have split out the Rule object creation into its own RuleCollector. I renamed discoverAlt in the grammar tree visitor to be discoverOuterAlt an added discoverAlt so we can get information about individual alts even inside subrules. Listeners always get an event for the generic rule context, which is used if there is no specific label for an alternative. Added a list of iteration operations for LL(*) subrules. Split buildRuleFunction into buildLeftRecursiveRuleFunction and one for normal rule function creation. I have to insert lots of extra code to manage the contexts, but of course it's all done using the templates. As long as those templates are correct, this code generation mechanism will work. I removed the st field from the parser rule context. I injected the left recursion transformation inside the SemanticPipeline. Visitor dispatch methods are always added to the generated context structures. Fixed some unit tests. About 24 fail. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9854]
This commit is contained in:
parrt 2012-01-11 11:00:05 -08:00
parent 742a08862f
commit 848c6e4313
35 changed files with 901 additions and 426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -463,9 +463,12 @@ public abstract class Parser extends Recognizer<Token, v2ParserATNSimulator<Toke
* This is very useful for error messages.
*/
public List<String> getRuleInvocationStack() {
return getRuleInvocationStack(_ctx);
}
public List<String> getRuleInvocationStack(RuleContext p) {
String[] ruleNames = getRuleNames();
List<String> stack = new ArrayList<String>();
RuleContext p = _ctx;
while ( p!=null ) {
// compute what follows who invoked us
stack.add(ruleNames[p.getRuleIndex()]);

34
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
View File

@ -34,7 +34,6 @@ import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.stringtemplate.v4.ST;
import java.util.ArrayList;
import java.util.Collections;
@ -102,7 +101,6 @@ public class ParserRuleContext<Symbol> extends RuleContext {
public int s = -1;
public Symbol start, stop;
public ST st;
/** Set during parsing to identify which rule parser is in. */
public int ruleIndex;
@ -112,7 +110,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
public ParserRuleContext() { }
/** COPY a ctx (it deliberately not using copy constructor) */
/** COPY a ctx (I'm deliberately not using copy constructor) */
public void copyFrom(ParserRuleContext<Symbol> ctx) {
// from RuleContext
this.parent = ctx.parent;
@ -121,16 +119,15 @@ public class ParserRuleContext<Symbol> extends RuleContext {
this.start = ctx.start;
this.stop = ctx.stop;
this.st = ctx.st;
this.ruleIndex = ctx.ruleIndex;
}
public ParserRuleContext(@Nullable ParserRuleContext parent, int invokingStateNumber, int stateNumber) {
public ParserRuleContext(@Nullable ParserRuleContext<Symbol> parent, int invokingStateNumber, int stateNumber) {
super(parent, invokingStateNumber);
this.s = stateNumber;
}
public ParserRuleContext(@Nullable ParserRuleContext parent, int stateNumber) {
public ParserRuleContext(@Nullable ParserRuleContext<Symbol> parent, int stateNumber) {
this(parent, parent!=null ? parent.s : -1 /* invoking state */, stateNumber);
}
@ -150,7 +147,8 @@ public class ParserRuleContext<Symbol> extends RuleContext {
public void enterRule(ParseTreeListener<Symbol> listener) { }
public void exitRule(ParseTreeListener<Symbol> listener) { }
public void addChild(TerminalNode<?> t) {
/** Does not set parent link; other add methods do */
public void addChild(TerminalNode<Symbol> t) {
if ( children==null ) children = new ArrayList<ParseTree>();
children.add(t);
}
@ -175,16 +173,16 @@ public class ParserRuleContext<Symbol> extends RuleContext {
// states.add(s);
// }
public void addChild(Token matchedToken) {
TerminalNodeImpl<?> t = new TerminalNodeImpl<Token>(matchedToken);
t.parent = this;
public void addChild(Symbol matchedToken) {
TerminalNodeImpl<Symbol> t = new TerminalNodeImpl<Symbol>(matchedToken);
addChild(t);
t.parent = this;
}
public void addErrorNode(Token badToken) {
TerminalNodeImpl<?> t = new ErrorNodeImpl<Token>(badToken);
t.parent = this;
public void addErrorNode(Symbol badToken) {
TerminalNodeImpl<Symbol> t = new ErrorNodeImpl<Symbol>(badToken);
addChild(t);
t.parent = this;
}
@Override
@ -198,7 +196,6 @@ public class ParserRuleContext<Symbol> extends RuleContext {
@Override
public int getRuleIndex() { return ruleIndex; }
public ST getTemplate() { return st; }
public Symbol getStart() { return start; }
public Symbol getStop() { return stop; }
@ -206,7 +203,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
public String toString(@NotNull Recognizer<?,?> recog, RuleContext stop) {
if ( recog==null ) return super.toString(recog, stop);
StringBuilder buf = new StringBuilder();
ParserRuleContext<?> p = this;
ParserRuleContext p = this;
buf.append("[");
while ( p != null && p != stop ) {
ATN atn = recog.getATN();
@ -217,21 +214,20 @@ public class ParserRuleContext<Symbol> extends RuleContext {
// ATNState invoker = atn.states.get(ctx.invokingState);
// RuleTransition rt = (RuleTransition)invoker.transition(0);
// buf.append(recog.getRuleNames()[rt.target.ruleIndex]);
p = (ParserRuleContext<?>)p.parent;
p = (ParserRuleContext)p.parent;
}
buf.append("]");
return buf.toString();
}
/** Used for rule context info debugging during runtime, not so much for ATN debugging */
/** Used for rule context info debugging during parse-time, not so much for ATN debugging */
public String toInfoString(Parser recognizer) {
List<String> rules = recognizer.getRuleInvocationStack();
List<String> rules = recognizer.getRuleInvocationStack(this);
Collections.reverse(rules);
return "ParserRuleContext"+rules+"{" +
"altNum=" + altNum +
", start=" + start +
", stop=" + stop +
", st=" + st +
'}';
}
}

19
tool/resources/org/antlr/v4/tool/templates/LeftRecursiveRules.stg
View File

@ -37,24 +37,13 @@ recRule(ruleName, precArgDef, argName, primaryAlts, opAlts, setResultAction,
userRetvals, leftRecursiveRuleRefLabels) ::=
<<
<ruleName>[<precArgDef>]<if(userRetvals)> returns [<userRetvals>]<endif>
options {simrecursion_=true;}
: ( <primaryAlts:{alt | <alt.altText> /* <alt.altLabel> */}; separator="\n | ">
: ( <primaryAlts:{alt | <alt.altText> }; separator="\n | ">
)
( options {simrecursion_=true;}
:
<opAlts; separator=" |\n\n">
<if(leftRecursiveRuleRefLabels)>
| {/* (safely) force ANTLR to know about left-recursive rule labels we removed */}
BOGUS_ <leftRecursiveRuleRefLabels:{lab | <lab>=<ruleName> BOGUS_}; separator=" ">
<endif>
( <opAlts; separator="\n | ">
)*
;
>>
recRuleAlt(alt, startAction, pred) ::= <<
{<pred>}?
{
<startAction>
}
<alt>
recRuleAlt(alt, pred) ::= <<
{<pred>}? <alt.altText>
>>

39
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -181,7 +181,9 @@ RuleFunction(currentRule,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAc
}
>>
LRecursiveRuleFunction(currentRule,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAction,postamble) ::= <<
LeftRecursiveRuleFunction(currentRule,code,locals,ruleCtx,altLabelCtxs,
namedActions,finallyAction,postamble) ::=
<<
<ruleCtx>
<altLabelCtxs; separator="\n">
@ -195,7 +197,7 @@ LRecursiveRuleFunction(currentRule,code,locals,ruleCtx,altLabelCtxs,namedActions
<namedActions.init>
<locals; separator="\n">
try {
<code>
<code>
_localctx.stop = _input.LT(-1);
<postamble; separator="\n">
<namedActions.after>
@ -348,16 +350,13 @@ case <i>:
// TODO: we we need uniqueID? a single _alt might work
StarBlock(choice, alts, sync) ::= <<
StarBlock(choice, alts, sync, iteration) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
int _alt<choice.uniqueID> = getInterpreter().adaptivePredict(_input,<choice.decision>,_ctx);
while ( _alt<choice.uniqueID>!=<choice.exitAlt> && _alt<choice.uniqueID>!=-1 ) {
if ( _alt<choice.uniqueID>==1 ) {
<if(choice.isForRecursiveRule)>
if ( _parseListeners!=null ) triggerExitRuleEvent();
_prevctx = _localctx;
<endif>
<iteration>
<alts> <! should only be one !>
}
setState(<choice.loopBackStateNumber>);
@ -501,7 +500,8 @@ ListLabelName(label) ::= "<label>_list"
CaptureNextToken(d) ::= "<d.varName> = _input.LT(1);"
CaptureNextTokenType(d) ::= "<d.varName> = _input.LA(1);"
StructDecl(s,attrs,visitorDispatchMethods,interfaces,extensionMembers,superClass={ParserRuleContext\<<InputSymbolType()>>}) ::= <<
StructDecl(s,attrs,visitorDispatchMethods,interfaces,extensionMembers,
superClass={ParserRuleContext\<<InputSymbolType()>>}) ::= <<
public static class <s.name> extends <superClass><if(interfaces)> implements <interfaces; separator=", "><endif> {
<attrs:{a | public <a>;}; separator="\n">
<if(s.ctorAttrs)>public <s.name>(ParserRuleContext parent, int state) { super(parent, state); }<endif>
@ -509,7 +509,7 @@ public static class <s.name> extends <superClass><if(interfaces)> implements <in
super(parent, state);
<s.ctorAttrs:{a | this.<a.name> = <a.name>;}; separator="\n">
}
<if(!visitorDispatchMethods)> <! don't need copy if no subclasses !>
<if(s.provideCopyFrom)> <! don't need copy unless we have subclasses !>
public <s.name>() { }
public void copyFrom(<s.name> ctx) {
super.copyFrom(ctx);
@ -546,6 +546,8 @@ recRuleArg() ::= "$_p"
recRuleAltPredicate(ruleName,opPrec) ::= "<opPrec> >= <recRuleArg()>"
recRuleSetResultAction() ::= "$tree=$<ruleName>.tree;"
recRuleSetReturnAction(src,name) ::= "$<name>=$<src>.<name>;"
recRuleSetStopToken() ::= "_ctx.stop = _input.LT(-1);"
recRuleAltStartAction(ruleName, ctxName, label) ::= <<
_localctx = new <ctxName>Context(_parentctx, _startState, _p);
_localctx.addChild(_prevctx);
@ -553,6 +555,25 @@ _localctx.addChild(_prevctx);
pushNewRecursionContext(_localctx, RULE_<ruleName>);
>>
recRuleLabeledAltStartAction(ruleName, ctxName, label) ::= <<
_localctx = new <ctxName>Context(new <ruleName>Context(_parentctx, _startState, _p));
_localctx.addChild(_prevctx);
<if(label)>_localctx.<label> = _prevctx;<endif>
pushNewRecursionContext(_localctx, RULE_<ruleName>);
>>
recRuleReplaceContext(ctxName) ::= <<
_localctx = new <ctxName>Context(_localctx);
_ctx = _localctx;
_prevctx = _localctx;
>>
recRuleSetPrevCtx() ::= <<
if ( _parseListeners!=null ) triggerExitRuleEvent();
_prevctx = _localctx;
>>
LexerFile(lexerFile, lexer, namedActions) ::= <<
// $ANTLR ANTLRVersion> <lexerFile.fileName> generatedTimestamp>
<namedActions.header>

25
tool/src/org/antlr/v4/codegen/model/LRecursiveRuleFunction.java → tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java
View File

@ -27,13 +27,26 @@
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.codegen.model;
package org.antlr.v4.analysis;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.AltAST;
public class LRecursiveRuleFunction extends RuleFunction {
public LRecursiveRuleFunction(OutputModelFactory factory, Rule r) {
super(factory, r);
public class LeftRecursiveRuleAltInfo {
public int alt;
public String leftRecursiveRuleRefLabel;
public String altLabel;
public String altText;
public AltAST altAST;
public int nextPrec;
public LeftRecursiveRuleAltInfo(int alt, String altText) {
this(alt, altText, null, null);
}
public LeftRecursiveRuleAltInfo(int alt, String altText, String leftRecursiveRuleRefLabel, String altLabel) {
this.alt = alt;
this.altText = altText;
this.leftRecursiveRuleRefLabel = leftRecursiveRuleRefLabel;
this.altLabel = altLabel;
}
}

164
tool/src/org/antlr/v4/parse/LeftRecursiveRuleAnalyzer.java → tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java
View File

@ -27,7 +27,7 @@
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.parse;
package org.antlr.v4.analysis;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.TokenStream;
@ -35,7 +35,10 @@ import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.Tool;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.LeftRecursiveRuleWalker;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTWithOptions;
import org.stringtemplate.v4.ST;
@ -44,31 +47,22 @@ import org.stringtemplate.v4.STGroupFile;
import java.util.*;
/** */
/** Using a tree walker on the rules, determine if a rule is directly left-recursive and if it follows
* our pattern.
*/
public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
public static enum ASSOC { left, right }
public static class Alt {
public Alt(String altText) {
this(altText, null);
}
public Alt(String altText, String leftRecursiveRuleRefLabel) {
this.altText = altText;
this.leftRecursiveRuleRefLabel = leftRecursiveRuleRefLabel;
}
public String leftRecursiveRuleRefLabel;
public String altLabel;
public String altText;
}
public Tool tool;
public String ruleName;
public LinkedHashMap<Integer, Alt> binaryAlts = new LinkedHashMap<Integer, Alt>();
public LinkedHashMap<Integer, Alt> ternaryAlts = new LinkedHashMap<Integer, Alt>();
public LinkedHashMap<Integer, Alt> suffixAlts = new LinkedHashMap<Integer, Alt>();
public List<Alt> prefixAlts = new ArrayList<Alt>();
public List<Alt> otherAlts = new ArrayList<Alt>();
public List<String> leftRecursiveRuleRefLabels = new ArrayList<String>();
public LinkedHashMap<Integer, LeftRecursiveRuleAltInfo> binaryAlts = new LinkedHashMap<Integer, LeftRecursiveRuleAltInfo>();
public LinkedHashMap<Integer, LeftRecursiveRuleAltInfo> ternaryAlts = new LinkedHashMap<Integer, LeftRecursiveRuleAltInfo>();
public LinkedHashMap<Integer, LeftRecursiveRuleAltInfo> suffixAlts = new LinkedHashMap<Integer, LeftRecursiveRuleAltInfo>();
public List<LeftRecursiveRuleAltInfo> prefixAlts = new ArrayList<LeftRecursiveRuleAltInfo>();
public List<LeftRecursiveRuleAltInfo> otherAlts = new ArrayList<LeftRecursiveRuleAltInfo>();
/** Pointer to ID node of ^(= ID element) */
public List<GrammarAST> leftRecursiveRuleRefLabels = new ArrayList<GrammarAST>();
public GrammarAST retvals;
@ -135,72 +129,99 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
}
@Override
public void binaryAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
public void binaryAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
GrammarAST lrlabel = stripLeftRecursion(altTree);
String label = lrlabel != null ? lrlabel.getText() : null;
if ( lrlabel!=null ) leftRecursiveRuleRefLabels.add(lrlabel);
stripAssocOptions(altTree);
stripAltLabel(altTree);
// rewrite e to be e_[rec_arg]
int nextPrec = nextPrecedence(alt);
altTree = addPrecedenceArgToRules(altTree, nextPrec);
stripAltLabel(altTree);
String altText = text(altTree);
altText = altText.trim();
binaryAlts.put(alt, new Alt(altText, label));
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel);
a.nextPrec = nextPrec;
binaryAlts.put(alt, a);
//System.out.println("binaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
/** Convert e ? e : e -> ? e : e_[nextPrec] */
@Override
public void ternaryAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
public void ternaryAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
GrammarAST lrlabel = stripLeftRecursion(altTree);
String label = lrlabel != null ? lrlabel.getText() : null;
if ( lrlabel!=null ) leftRecursiveRuleRefLabels.add(lrlabel);
stripAssocOptions(altTree);
stripAltLabel(altTree);
int nextPrec = nextPrecedence(alt);
altTree = addPrecedenceArgToLastRule(altTree, nextPrec);
String altText = text(altTree);
altText = altText.trim();
ternaryAlts.put(alt, new Alt(altText, label));
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel);
a.nextPrec = nextPrec;
ternaryAlts.put(alt, a);
//System.out.println("ternaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void prefixAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
public void prefixAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
stripAltLabel(altTree);
int nextPrec = precedence(alt);
// rewrite e to be e_[prec]
altTree = addPrecedenceArgToRules(altTree, nextPrec);
String altText = text(altTree);
altText = altText.trim();
prefixAlts.add(new Alt(altText));
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel);
a.nextPrec = nextPrec;
prefixAlts.add(a);
//System.out.println("prefixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void suffixAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
public void suffixAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
GrammarAST lrlabel = stripLeftRecursion(altTree);
String label = lrlabel != null ? lrlabel.getText() : null;
stripAltLabel(altTree);
if ( lrlabel!=null ) leftRecursiveRuleRefLabels.add(lrlabel);
String altText = text(altTree);
altText = altText.trim();
suffixAlts.put(alt, new Alt(altText, label));
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel);
suffixAlts.put(alt, a);
// System.out.println("suffixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void otherAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
public void otherAlt(AltAST altTree, int alt) {
altTree = (AltAST)altTree.dupTree();
GrammarAST lrlabel = stripLeftRecursion(altTree);
String label = lrlabel != null ? lrlabel.getText() : null;
stripAltLabel(altTree);
if ( lrlabel!=null ) leftRecursiveRuleRefLabels.add(lrlabel);
String altText = text(altTree);
otherAlts.add(new Alt(altText, label));
String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null;
LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel);
// if ( altLabel!=null ) {
// a.startAction = codegenTemplates.getInstanceOf("recRuleReplaceContext");
// a.startAction.add("ctxName", altLabel);
// }
otherAlts.add(a);
// System.out.println("otherAlt " + alt + ": " + altText);
}
@ -217,23 +238,18 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
ruleST.add("setResultAction", setResultST);
ruleST.add("userRetvals", retvals);
LinkedHashMap<Integer, Alt> opPrecRuleAlts = new LinkedHashMap<Integer, Alt>();
LinkedHashMap<Integer, LeftRecursiveRuleAltInfo> opPrecRuleAlts = new LinkedHashMap<Integer, LeftRecursiveRuleAltInfo>();
opPrecRuleAlts.putAll(binaryAlts);
opPrecRuleAlts.putAll(ternaryAlts);
opPrecRuleAlts.putAll(suffixAlts);
for (int alt : opPrecRuleAlts.keySet()) {
Alt altInfo = opPrecRuleAlts.get(alt);
LeftRecursiveRuleAltInfo altInfo = opPrecRuleAlts.get(alt);
ST altST = recRuleTemplates.getInstanceOf("recRuleAlt");
ST predST = codegenTemplates.getInstanceOf("recRuleAltPredicate");
ST altActionST = codegenTemplates.getInstanceOf("recRuleAltStartAction");
altActionST.add("ctxName", ruleName); // todo: handle alt labels
altActionST.add("ruleName", ruleName);
altActionST.add("label", altInfo.leftRecursiveRuleRefLabel);
predST.add("opPrec", precedence(alt));
predST.add("ruleName", ruleName);
altST.add("pred", predST);
altST.add("alt", altInfo.altText);
altST.add("startAction", altActionST);
altST.add("alt", altInfo);
ruleST.add("opAlts", altST);
}
@ -247,7 +263,7 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
return ruleST.render();
}
public GrammarAST addPrecedenceArgToRules(GrammarAST t, int prec) {
public AltAST addPrecedenceArgToRules(AltAST t, int prec) {
if ( t==null ) return null;
for (GrammarAST rref : t.getNodesWithType(RULE_REF)) {
if ( rref.getText().equals(ruleName) ) {
@ -257,6 +273,16 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
return t;
}
public AltAST addPrecedenceArgToLastRule(AltAST t, int prec) {
if ( t==null ) return null;
GrammarAST last = null;
for (GrammarAST rref : t.getNodesWithType(RULE_REF)) { last = rref; }
if ( last !=null && last.getText().equals(ruleName) ) {
last.setText(ruleName+"["+prec+"]");
}
return t;
}
public void stripAssocOptions(GrammarAST t) {
if ( t==null ) return;
for (GrammarAST options : t.getNodesWithType(ELEMENT_OPTIONS)) {
@ -297,34 +323,38 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
return false;
}
public GrammarAST addPrecedenceArgToLastRule(GrammarAST t, int prec) {
if ( t==null ) return null;
GrammarAST last = null;
for (GrammarAST rref : t.getNodesWithType(RULE_REF)) { last = rref; }
if ( last !=null && last.getText().equals(ruleName) ) {
last.setText(ruleName+"["+prec+"]");
}
return t;
}
// TODO: this strips the tree properly, but since text()
// uses the start of stop token index and gets text from that
// ineffectively ignores this routine.
public String stripLeftRecursion(GrammarAST altAST) {
String label=null;
public GrammarAST stripLeftRecursion(GrammarAST altAST) {
GrammarAST lrlabel=null;
GrammarAST first = (GrammarAST)altAST.getChild(0);
Tree rref = first.getChild(1); // if label=rule
if ( (first.getType()==RULE_REF && first.getText().equals(ruleName)) ||
(rref!=null && rref.getType()==RULE_REF && rref.getText().equals(ruleName)) )
{
if ( first.getType()==ASSIGN ) label = first.getChild(0).getText();
if ( first.getType()==ASSIGN ) lrlabel = (GrammarAST)first.getChild(0);
// remove rule ref (first child)
altAST.deleteChild(0);
// reset index so it prints properly
GrammarAST newFirstChild = (GrammarAST)altAST.getChild(0);
altAST.setTokenStartIndex(newFirstChild.getTokenStartIndex());
}
return label;
return lrlabel;
}
/** Strip last 2 tokens if -> label; alter indexes in altAST */
public void stripAltLabel(GrammarAST altAST) {
int start = altAST.getTokenStartIndex();
int stop = altAST.getTokenStopIndex();
TokenStream tokens = input.getTokenStream();
// find =>
for (int i=stop; i>=start; i--) {
if ( tokens.get(i).getType()==RARROW ) {
altAST.setTokenStopIndex(i-1);
return;
}
}
}
public String text(GrammarAST t) {

207
tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java Normal file
View File

@ -0,0 +1,207 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.analysis;
import org.antlr.runtime.*;
import org.antlr.v4.Tool;
import org.antlr.v4.misc.OrderedHashMap;
import org.antlr.v4.parse.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import java.util.ArrayList;
import java.util.List;
/** Remove left-recursive rule refs, add precedence args to recursive rule refs.
* Rewrite rule so we can create ATN.
*
* MODIFIES grammar AST in place.
*/
public class LeftRecursiveRuleTransformer {
public GrammarRootAST ast;
public List<Rule> rules;
public Tool tool;
public LeftRecursiveRuleTransformer(GrammarRootAST ast, List<Rule> rules, Tool tool) {
this.ast = ast;
this.rules = rules;
this.tool = tool;
}
public void translateLeftRecursiveRules() {
// TODO: what about -language foo cmd line?
String language = Grammar.getLanguageOption(ast);
// translate all recursive rules
List<String> leftRecursiveRuleNames = new ArrayList<String>();
for (Rule r : rules) {
if ( !Character.isUpperCase(r.name.charAt(0)) ) {
if ( LeftRecursiveRuleAnalyzer.hasImmediateRecursiveRuleRefs(r.ast, r.name) ) {
boolean fitsPattern = translateLeftRecursiveRule(ast, (LeftRecursiveRule)r, language);
if ( fitsPattern ) leftRecursiveRuleNames.add(r.name);
}
}
}
// update all refs to recursive rules to have [0] argument
for (GrammarAST r : ast.getNodesWithType(ANTLRParser.RULE_REF)) {
if ( r.getParent().getType()==ANTLRParser.RULE ) continue; // must be rule def
if ( r.getChildCount()>0 ) continue; // already has arg; must be in rewritten rule
if ( leftRecursiveRuleNames.contains(r.getText()) ) {
// found ref to recursive rule not already rewritten with arg
ActionAST arg = new ActionAST(new CommonToken(ANTLRParser.ARG_ACTION, "0"));
r.addChild(arg);
}
}
}
/** Return true if successful */
public boolean translateLeftRecursiveRule(GrammarRootAST ast,
LeftRecursiveRule r,
String language)
{
//tool.log("grammar", ruleAST.toStringTree());
GrammarAST prevRuleAST = r.ast;
TokenStream tokens = ast.tokens;
Grammar g = ast.g;
String ruleName = prevRuleAST.getChild(0).getText();
LeftRecursiveRuleAnalyzer leftRecursiveRuleWalker =
new LeftRecursiveRuleAnalyzer(tokens, prevRuleAST, tool, ruleName, language);
boolean isLeftRec = false;
try {
// System.out.println("TESTING ---------------\n"+
// leftRecursiveRuleWalker.text(ruleAST));
isLeftRec = leftRecursiveRuleWalker.rec_rule();
}
catch (RecognitionException re) {
isLeftRec = false; // didn't match; oh well
}
if ( !isLeftRec ) return false;
// replace old rule's AST
GrammarAST RULES = (GrammarAST)ast.getFirstChildWithType(ANTLRParser.RULES);
String newRuleText = leftRecursiveRuleWalker.getArtificialOpPrecRule();
// System.out.println("created: "+newRuleText);
RuleAST t = parseArtificialRule(g, newRuleText);
// update grammar AST and set rule's AST.
RULES.setChild(prevRuleAST.getChildIndex(), t);
r.ast = t;
// Reduce sets in newly created rule tree
GrammarTransformPipeline transform = new GrammarTransformPipeline(g, g.tool);
transform.reduceBlocksToSets(r.ast);
transform.expandParameterizedLoops(r.ast);
// track recursive alt info for codegen
r.recPrimaryAlts = new ArrayList<LeftRecursiveRuleAltInfo>();
r.recPrimaryAlts.addAll(leftRecursiveRuleWalker.prefixAlts);
r.recPrimaryAlts.addAll(leftRecursiveRuleWalker.otherAlts);
r.recOpAlts = new OrderedHashMap<Integer, LeftRecursiveRuleAltInfo>();
r.recOpAlts.putAll(leftRecursiveRuleWalker.binaryAlts);
r.recOpAlts.putAll(leftRecursiveRuleWalker.ternaryAlts);
r.recOpAlts.putAll(leftRecursiveRuleWalker.suffixAlts);
// walk alt info records and set their altAST to point to appropriate ALT subtree
// from freshly created AST
setAltASTPointers(r, t);
// update Rule to just one alt and add prec alt
ActionAST arg = (ActionAST)r.ast.getFirstChildWithType(ANTLRParser.ARG_ACTION);
if ( arg!=null ) {
r.args = ScopeParser.parseTypedArgList(arg.getText(), g.tool.errMgr);
r.args.type = AttributeDict.DictType.ARG;
r.args.ast = arg;
arg.resolver = r.alt[1]; // todo: isn't this Rule or something?
}
// define labels on recursive rule refs we delete; they don't point to nodes of course
for (GrammarAST labelNode : leftRecursiveRuleWalker.leftRecursiveRuleRefLabels) {
GrammarAST labelOpNode = (GrammarAST)labelNode.getParent();
GrammarAST elementNode = (GrammarAST)labelOpNode.getChild(1);
LabelElementPair lp = new LabelElementPair(g, labelNode, elementNode, labelOpNode.getType());
r.alt[1].labelDefs.map(labelNode.getText(), lp);
}
tool.log("grammar", "added: "+t.toStringTree());
return true;
}
public RuleAST parseArtificialRule(final Grammar g, String ruleText) {
ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
GrammarASTAdaptor adaptor = new GrammarASTAdaptor();
CommonTokenStream tokens = new CommonTokenStream(lexer);
ToolANTLRParser p = new ToolANTLRParser(tokens, tool);
p.setTreeAdaptor(adaptor);
try {
ParserRuleReturnScope r = p.rule();
RuleAST tree = (RuleAST)r.getTree();
GrammarTransformPipeline.setGrammarPtr(g, tree);
return tree;
}
catch (Exception e) {
tool.errMgr.toolError(ErrorType.INTERNAL_ERROR,
"error parsing rule created during left-recursion detection: "+ruleText,
e);
}
return null;
}
/**
(RULE e int _p (returns int v)
(BLOCK
(ALT
(BLOCK
(ALT INT {$v = $INT.int;})
(ALT '(' (= x e) ')' {$v = $x.v;})
(ALT ID))
(* (BLOCK
(ALT {7 >= $_p}? '*' (= b e) {$v = $a.v * $b.v;})
(ALT {6 >= $_p}? '+' (= b e) {$v = $a.v + $b.v;})
(ALT {3 >= $_p}? '++') (ALT {2 >= $_p}? '--'))))))
*/
public void setAltASTPointers(LeftRecursiveRule r, RuleAST t) {
BlockAST ruleBlk = (BlockAST)t.getFirstChildWithType(ANTLRParser.BLOCK);
AltAST mainAlt = (AltAST)ruleBlk.getChild(0);
BlockAST primaryBlk = (BlockAST)mainAlt.getChild(0);
BlockAST opsBlk = (BlockAST)mainAlt.getChild(1).getChild(0); // (* BLOCK ...)
for (int i = 0; i < r.recPrimaryAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = r.recPrimaryAlts.get(i);
altInfo.altAST = (AltAST)primaryBlk.getChild(i);
// System.out.println(altInfo.altAST.toStringTree());
}
for (int i = 0; i < r.recOpAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.getElement(i);
altInfo.altAST = (AltAST)opsBlk.getChild(i);
// System.out.println(altInfo.altAST.toStringTree());
}
}
}

129
tool/src/org/antlr/v4/codegen/OutputModelController.java
View File

@ -30,17 +30,21 @@
package org.antlr.v4.codegen;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.v4.analysis.LeftRecursiveRuleAltInfo;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.codegen.model.decl.CodeBlock;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.tool.Alternative;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.PredAST;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import java.util.ArrayList;
import java.util.List;
@ -144,12 +148,110 @@ public class OutputModelController {
* output object with stuff found in r.
*/
public void buildRuleFunction(Parser parser, Rule r) {
CodeGenerator gen = delegate.getGenerator();
RuleFunction function = rule(r);
parser.funcs.add(function);
// TRIGGER factory functions for rule alts, elements
pushCurrentRule(function);
if ( r instanceof LeftRecursiveRule ) {
buildLeftRecursiveRuleFunction((LeftRecursiveRule)r,
(LeftRecursiveRuleFunction)function);
}
else {
buildNormalRuleFunction(r, function);
}
Grammar g = getGrammar();
for (ActionAST a : r.actions) {
if ( a instanceof PredAST ) {
PredAST p = (PredAST)a;
RuleSempredFunction rsf = parser.sempredFuncs.get(r);
if ( rsf==null ) {
rsf = new RuleSempredFunction(delegate, r, function.ctxType);
parser.sempredFuncs.put(r, rsf);
}
rsf.actions.put(g.sempreds.get(p), new Action(delegate, p));
}
}
popCurrentRule();
}
public void buildLeftRecursiveRuleFunction(LeftRecursiveRule r, LeftRecursiveRuleFunction function) {
buildNormalRuleFunction(r, function);
// now inject code to start alts
CodeGenerator gen = delegate.getGenerator();
STGroup codegenTemplates = gen.templates;
// pick out alt(s) for primaries
CodeBlockForOuterMostAlt outerAlt = (CodeBlockForOuterMostAlt)function.code.get(0);
List<CodeBlockForAlt> primaryAltsCode = new ArrayList<CodeBlockForAlt>();
SrcOp primaryStuff = outerAlt.ops.get(0);
if ( primaryStuff instanceof Choice ) {
Choice primaryAltBlock = (Choice) primaryStuff;
primaryAltsCode.addAll(primaryAltBlock.alts);
}
else { // just a single alt I guess; no block
primaryAltsCode.add((CodeBlockForAlt)primaryStuff);
}
// pick out alt(s) for op alts
StarBlock opAltStarBlock = (StarBlock)outerAlt.ops.get(1);
CodeBlockForAlt altForOpAltBlock = opAltStarBlock.alts.get(0);
List<CodeBlockForAlt> opAltsCode = new ArrayList<CodeBlockForAlt>();
SrcOp opStuff = altForOpAltBlock.ops.get(0);
if ( opStuff instanceof AltBlock ) {
AltBlock opAltBlock = (AltBlock)opStuff;
opAltsCode.addAll(opAltBlock.alts);
}
else { // just a single alt I guess; no block
opAltsCode.add((CodeBlockForAlt)opStuff);
}
// Insert code in front of each primary alt to create specialized ctx if there was a label
for (int i = 0; i < primaryAltsCode.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = r.recPrimaryAlts.get(i);
if ( altInfo.altLabel==null ) continue;
ST altActionST = codegenTemplates.getInstanceOf("recRuleReplaceContext");
altActionST.add("ctxName", altInfo.altLabel);
Action altAction = new Action(delegate, altActionST.render());
CodeBlockForAlt alt = primaryAltsCode.get(i);
alt.insertOp(0, altAction);
}
// Insert code to set ctx.stop after primary block and before op * loop
ST setStopTokenAST = codegenTemplates.getInstanceOf("recRuleSetStopToken");
Action setStopTokenAction = new Action(delegate, setStopTokenAST.render());
outerAlt.insertOp(1, setStopTokenAction);
// Insert code to set _prevctx at start of * loop
ST setPrevCtx = codegenTemplates.getInstanceOf("recRuleSetPrevCtx");
Action setPrevCtxAction = new Action(delegate, setPrevCtx.render());
opAltStarBlock.addIterationOp(setPrevCtxAction);
// Insert code in front of each op alt to create specialized ctx if there was a label
for (int i = 0; i < opAltsCode.size(); i++) {
ST altActionST;
LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.getElement(i);
if ( altInfo.altLabel!=null ) {
altActionST = codegenTemplates.getInstanceOf("recRuleLabeledAltStartAction");
altActionST.add("ctxName", altInfo.altLabel);
}
else {
altActionST = codegenTemplates.getInstanceOf("recRuleAltStartAction");
altActionST.add("ctxName", r.name);
}
altActionST.add("ruleName", r.name);
altActionST.add("label", altInfo.leftRecursiveRuleRefLabel);
Action altAction = new Action(delegate, altActionST.render());
CodeBlockForAlt alt = opAltsCode.get(i);
alt.insertOp(0, altAction);
}
}
public void buildNormalRuleFunction(Rule r, RuleFunction function) {
CodeGenerator gen = delegate.getGenerator();
// TRIGGER factory functions for rule alts, elements
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(r.ast.token.getInputStream());
GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK);
CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk);
@ -165,27 +267,6 @@ public class OutputModelController {
function.ctxType = gen.target.getRuleFunctionContextStructName(function);
function.postamble = rulePostamble(function, r);
Grammar g = getGrammar();
for (ActionAST a : r.actions) {
if ( a instanceof PredAST ) {
PredAST p = (PredAST)a;
RuleSempredFunction rsf = parser.sempredFuncs.get(r);
if ( rsf==null ) {
rsf = new RuleSempredFunction(delegate, r, function.ctxType);
parser.sempredFuncs.put(r, rsf);
}
rsf.actions.put(g.sempreds.get(p), new Action(delegate, p));
}
}
// we will usually have to build one of these because of labels
// and it will come in very handy for visitors on the parse tree.
// I think I will just always generate this structure now.
// It makes it easier for code generation as well because every
// rule context has a real name.
//if ( function.ruleCtx.isEmpty() ) function.ruleCtx = null;
popCurrentRule();
}
public void buildLexerRuleActions(Lexer lexer, Rule r) {

6
tool/src/org/antlr/v4/codegen/ParserFactory.java
View File

@ -41,6 +41,7 @@ import org.antlr.v4.runtime.atn.PlusBlockStartState;
import org.antlr.v4.runtime.atn.StarLoopEntryState;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.Alternative;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
@ -61,9 +62,8 @@ public class ParserFactory extends DefaultOutputModelFactory {
}
public RuleFunction rule(Rule r) {
GrammarAST optionNode = r.ast.getOption("simrecursion_");
if ( optionNode!=null && optionNode.getText().equals("true") ) {
return new LRecursiveRuleFunction(this, r);
if ( r instanceof LeftRecursiveRule ) {
return new LeftRecursiveRuleFunction(this, (LeftRecursiveRule)r);
}
else {
return new RuleFunction(this, r);

10
tool/src/org/antlr/v4/codegen/model/Action.java
View File

@ -29,9 +29,11 @@
package org.antlr.v4.codegen.model;
import org.antlr.runtime.CommonToken;
import org.antlr.v4.codegen.ActionTranslator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.actions.ActionChunk;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.GrammarAST;
@ -52,4 +54,12 @@ public class Action extends RuleElement {
}
//System.out.println("actions="+chunks);
}
public Action(OutputModelFactory factory, String action) {
super(factory,null);
RuleFunction rf = factory.getCurrentRuleFunction();
ActionAST ast = new ActionAST(new CommonToken(ANTLRParser.ACTION, action));
ast.resolver = rf.rule;
chunks = ActionTranslator.translateActionChunk(factory, rf, action, ast);
}
}

60
tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java Normal file
View File

@ -0,0 +1,60 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.decl.RuleContextDecl;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.stringtemplate.v4.misc.MultiMap;
import java.util.Iterator;
import java.util.Set;
public class LeftRecursiveRuleFunction extends RuleFunction {
public LeftRecursiveRuleFunction(OutputModelFactory factory, LeftRecursiveRule r) {
super(factory, r);
// Since we delete x=lr, we have to manually add decls for all labels on left-recur refs
CodeGenerator gen = factory.getGenerator();
MultiMap<String,LabelElementPair> labelDefs = r.alt[1].labelDefs;
Set<String> labels = labelDefs.keySet();
for (Iterator<String> iterator = labels.iterator(); iterator.hasNext(); ) {
String label = iterator.next();
LabelElementPair l = r.getAnyLabelDef(label);
Rule targetRule = factory.getGrammar().getRule(l.element.getText());
String ctxName = gen.target.getRuleFunctionContextStructName(targetRule);
RuleContextDecl d = new RuleContextDecl(factory,label,ctxName);
addContextDecl(d);
}
}
}

6
tool/src/org/antlr/v4/codegen/model/ListenerFile.java
View File

@ -26,10 +26,8 @@ public class ListenerFile extends OutputFile {
grammarName = g.name;
for (Rule r : g.rules.values()) {
List<String> labels = r.getAltLabels();
if ( labels==null ) {
listenerNames.add(r.name);
}
else { // alt(s) with label(s)
listenerNames.add(r.name);
if ( labels!=null ) {
for (String label : labels) {
listenerNames.add(label);
}

9
tool/src/org/antlr/v4/codegen/model/Loop.java
View File

@ -32,16 +32,25 @@ package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.tool.ast.GrammarAST;
import java.util.ArrayList;
import java.util.List;
public class Loop extends Choice {
public int blockStartStateNumber;
public int loopBackStateNumber;
public int exitAlt;
@ModelElement public List<SrcOp> iteration;
public Loop(OutputModelFactory factory,
GrammarAST blkOrEbnfRootAST,
List<CodeBlockForAlt> alts)
{
super(factory, blkOrEbnfRootAST, alts);
}
public void addIterationOp(SrcOp op) {
if ( iteration==null ) iteration = new ArrayList<SrcOp>();
iteration.add(op);
}
}

5
tool/src/org/antlr/v4/codegen/model/StarBlock.java
View File

@ -31,23 +31,18 @@ package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.StarLoopEntryState;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import java.util.List;
public class StarBlock extends Loop {
public String loopLabel;
public boolean isForRecursiveRule;
public StarBlock(OutputModelFactory factory,
GrammarAST blkOrEbnfRootAST,
List<CodeBlockForAlt> alts)
{
super(factory, blkOrEbnfRootAST, alts);
BlockAST blk = (BlockAST)ast.getChild(0);
GrammarAST optionNode = blk.getOption("simrecursion_");
isForRecursiveRule = optionNode!=null && optionNode.getText().equals("true");
loopLabel = factory.getGenerator().target.getLoopLabel(blkOrEbnfRootAST);
StarLoopEntryState star = (StarLoopEntryState)blkOrEbnfRootAST.atnState;
loopBackStateNumber = star.loopBackState.stateNumber;

11
tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java
View File

@ -30,22 +30,13 @@
package org.antlr.v4.codegen.model.decl;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.VisitorDispatchMethod;
import org.antlr.v4.tool.Rule;
import java.util.ArrayList;
/** A StructDecl to handle a -> label on alt */
public class AltLabelStructDecl extends StructDecl {
public String label;
public AltLabelStructDecl(OutputModelFactory factory, Rule r, String label) {
super(factory, r);
this.label = label;
}
@Override
public void addVisitorDispatchMethods(Rule r) {
visitorDispatchMethods = new ArrayList<VisitorDispatchMethod>();
visitorDispatchMethods.add(new VisitorDispatchMethod(factory, r, true));
visitorDispatchMethods.add(new VisitorDispatchMethod(factory, r, false));
}
}

11
tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java
View File

@ -30,10 +30,12 @@
package org.antlr.v4.codegen.model.decl;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.codegen.model.ModelElement;
import org.antlr.v4.codegen.model.SrcOp;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
public class CodeBlock extends SrcOp {
public int codeBlockLevel;
@ -70,6 +72,11 @@ public class CodeBlock extends SrcOp {
ops.add(op);
}
public void insertOp(int i, SrcOp op) {
if ( ops==null ) ops = new ArrayList<SrcOp>();
ops.add(i, op);
}
public void addOps(List<SrcOp> ops) {
if ( this.ops==null ) this.ops = new ArrayList<SrcOp>();
this.ops.addAll(ops);

11
tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java
View File

@ -46,6 +46,7 @@ import java.util.List;
*/
public class StructDecl extends Decl {
public String superClass;
public boolean provideCopyFrom;
@ModelElement public OrderedHashSet<Decl> attrs = new OrderedHashSet<Decl>();
@ModelElement public Collection<Attribute> ctorAttrs;
@ModelElement public List<VisitorDispatchMethod> visitorDispatchMethods;
@ -55,15 +56,13 @@ public class StructDecl extends Decl {
public StructDecl(OutputModelFactory factory, Rule r) {
super(factory, factory.getGenerator().target.getRuleFunctionContextStructName(r));
addVisitorDispatchMethods(r);
provideCopyFrom = r.hasAltSpecificContexts();
}
public void addVisitorDispatchMethods(Rule r) {
List<String> labels = r.getAltLabels();
if ( labels==null ) {
visitorDispatchMethods = new ArrayList<VisitorDispatchMethod>();
visitorDispatchMethods.add(new VisitorDispatchMethod(factory, r, true));
visitorDispatchMethods.add(new VisitorDispatchMethod(factory, r, false));
}
visitorDispatchMethods = new ArrayList<VisitorDispatchMethod>();
visitorDispatchMethods.add(new VisitorDispatchMethod(factory, r, true));
visitorDispatchMethods.add(new VisitorDispatchMethod(factory, r, false));
}
public void addDecl(Decl d) { attrs.add(d); }

6
tool/src/org/antlr/v4/misc/OrderedHashMap.java
View File

@ -29,7 +29,9 @@
package org.antlr.v4.misc;
import java.util.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
/** I need the get-element-i functionality so I'm subclassing
* LinkedHashMap.
@ -40,6 +42,8 @@ public class OrderedHashMap<K,V> extends LinkedHashMap<K,V> {
public K getKey(int i) { return elements.get(i); }
public V getElement(int i) { return get(elements.get(i)); }
@Override
public V put(K key, V value) {
elements.add(key);

4
tool/src/org/antlr/v4/parse/ANTLRParser.g
View File

@ -535,7 +535,9 @@ ruleAltList
;
labeledAlt
: alternative (POUND id {((AltAST)$alternative.tree).altLabel=$id.tree;})?
: alternative
( RARROW! id! {((AltAST)$alternative.tree).altLabel=$id.tree;}
)?
;

2
tool/src/org/antlr/v4/parse/BlockSetTransformer.g
View File

@ -48,7 +48,7 @@ ebnfBlockSet
@after {
GrammarTransformPipeline.setGrammarPtr(g, $tree);
}
: ^(ebnfSuffix blockSet) -> ^(ebnfSuffix ^(BLOCK<BlockAST> ^(ALT blockSet)))
: ^(ebnfSuffix blockSet) -> ^(ebnfSuffix ^(BLOCK<BlockAST> ^(ALT<AltAST> blockSet)))
;
ebnfSuffix

51
tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g
View File

@ -140,9 +140,9 @@ public void discoverLexerRule(RuleAST rule, GrammarAST ID, List<GrammarAST> modi
public void finishLexerRule(RuleAST rule, GrammarAST ID, GrammarAST block) { }
public void ruleCatch(GrammarAST arg, ActionAST action) { }
public void finallyAction(ActionAST action) { }
/** outermost alt */
public void discoverOuterAlt(AltAST alt) { }
public void finishOuterAlt(AltAST alt) { }
public void discoverAlt(AltAST alt) { }
/** outermost alt */
public void finishAlt(AltAST alt) { }
public void ruleRef(GrammarAST ref, ActionAST arg) { }
@ -246,9 +246,9 @@ currentOuterAltNumber=0;
{currentRuleName=$TOKEN_REF.text; currentRuleAST=$RULE;}
DOC_COMMENT? (^(RULEMODIFIERS m=FRAGMENT {mods.add($m);}))?
{discoverLexerRule((RuleAST)$RULE, $TOKEN_REF, mods, (GrammarAST)input.LT(1));}
ruleBlock
lexerRuleBlock
{
finishLexerRule((RuleAST)$RULE, $TOKEN_REF, $ruleBlock.start);
finishLexerRule((RuleAST)$RULE, $TOKEN_REF, $lexerRuleBlock.start);
currentRuleName=null; currentRuleAST=null;
}
)
@ -313,6 +313,17 @@ ruleModifier
| FRAGMENT
;
lexerRuleBlock
: ^( BLOCK
( {
currentOuterAltRoot = (GrammarAST)input.LT(1);
currentOuterAltNumber++;
}
lexerOuterAlternative
)+
)
;
ruleBlock
: ^( BLOCK
( {
@ -324,19 +335,39 @@ ruleBlock
)
;
lexerOuterAlternative
@init {
discoverOuterAlt((AltAST)$start);
}
@after {
finishOuterAlt((AltAST)$start);
}
: lexerAlternative
;
outerAlternative
@init {
discoverOuterAlt((AltAST)$start);
}
@after {
finishOuterAlt((AltAST)$start);
}
: alternative
;
lexerAlternative
: alternative
;
alternative
@init {
discoverAlt((AltAST)$start);
}
@after {
finishAlt((AltAST)$start);
}
: alternative
;
alternative
: ^(LEXER_ALT_ACTION alternative lexerAction*)
| ^(ALT element+)
: ^(ALT element+)
| ^(ALT EPSILON)
;

29
tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g
View File

@ -52,11 +52,11 @@ private int currentOuterAltNumber; // which outer alt of rule?
public int numAlts; // how many alts for this rule total?
public void setTokenPrec(GrammarAST t, int alt) {}
public void binaryAlt(GrammarAST altTree, int alt) {}
public void ternaryAlt(GrammarAST altTree, int alt) {}
public void prefixAlt(GrammarAST altTree, int alt) {}
public void suffixAlt(GrammarAST altTree, int alt) {}
public void otherAlt(GrammarAST altTree, int alt) {}
public void binaryAlt(AltAST altTree, int alt) {}
public void ternaryAlt(AltAST altTree, int alt) {}
public void prefixAlt(AltAST altTree, int alt) {}
public void suffixAlt(AltAST altTree, int alt) {}
public void otherAlt(AltAST altTree, int alt) {}
public void setReturnValues(GrammarAST t) {}
}
@ -115,17 +115,17 @@ ruleBlock returns [boolean isLeftRec]
/** An alt is either prefix, suffix, binary, or ternary operation or "other" */
outerAlternative returns [boolean isLeftRec]
: (binaryMultipleOp)=> binaryMultipleOp
{binaryAlt($start, currentOuterAltNumber); $isLeftRec=true;}
{binaryAlt((AltAST)$start, currentOuterAltNumber); $isLeftRec=true;}
| (binary)=> binary
{binaryAlt($start, currentOuterAltNumber); $isLeftRec=true;}
{binaryAlt((AltAST)$start, currentOuterAltNumber); $isLeftRec=true;}
| (ternary)=> ternary
{ternaryAlt($start, currentOuterAltNumber); $isLeftRec=true;}
{ternaryAlt((AltAST)$start, currentOuterAltNumber); $isLeftRec=true;}
| (prefix)=> prefix
{prefixAlt($start, currentOuterAltNumber);}
{prefixAlt((AltAST)$start, currentOuterAltNumber);}
| (suffix)=> suffix
{suffixAlt($start, currentOuterAltNumber); $isLeftRec=true;}
{suffixAlt((AltAST)$start, currentOuterAltNumber); $isLeftRec=true;}
| ^(ALT element+) // "other" case
{otherAlt($start, currentOuterAltNumber);}
{otherAlt((AltAST)$start, currentOuterAltNumber);}
;
binary
@ -133,7 +133,12 @@ binary
;
binaryMultipleOp
: ^( ALT recurse ^( BLOCK ( ^( ALT (op=token)+ {setTokenPrec($op.t, currentOuterAltNumber);} ) )+ ) recurse ACTION? )
: ^( ALT recurse
( ^( BLOCK ( ^( ALT (op=token)+ {setTokenPrec($op.t, currentOuterAltNumber);} ) )+ )
| ^(SET (op=token)+ {setTokenPrec($op.t, currentOuterAltNumber);})
)
recurse ACTION?
)
;
ternary

112
tool/src/org/antlr/v4/semantics/RuleCollector.java Normal file
View File

@ -0,0 +1,112 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.semantics;
import org.antlr.v4.analysis.LeftRecursiveRuleAnalyzer;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.parse.ScopeParser;
import org.antlr.v4.tool.AttributeDict;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.RuleAST;
import java.util.ArrayList;
import java.util.List;
public class RuleCollector extends GrammarTreeVisitor {
/** which grammar are we checking */
public Grammar g;
// stuff to collect. this is the output
public List<Rule> rules = new ArrayList<Rule>();
public RuleCollector(Grammar g) { this.g = g; }
public void process(GrammarAST ast) { visitGrammar(ast); }
@Override
public void discoverRule(RuleAST rule, GrammarAST ID,
List<GrammarAST> modifiers, ActionAST arg,
ActionAST returns, GrammarAST thrws,
GrammarAST options, GrammarAST locals,
List<GrammarAST> actions,
GrammarAST block)
{
int numAlts = block.getChildCount();
Rule r = null;
if ( LeftRecursiveRuleAnalyzer.hasImmediateRecursiveRuleRefs(rule, ID.getText()) ) {
r = new LeftRecursiveRule(g, ID.getText(), rule);
}
else {
r = new Rule(g, ID.getText(), rule, numAlts);
}
rules.add(r);
if ( arg!=null ) {
r.args = ScopeParser.parseTypedArgList(arg.getText(), g.tool.errMgr);
r.args.type = AttributeDict.DictType.ARG;
r.args.ast = arg;
arg.resolver = r.alt[currentOuterAltNumber];
}
if ( returns!=null ) {
r.retvals = ScopeParser.parseTypedArgList(returns.getText(), g.tool.errMgr);
r.retvals.type = AttributeDict.DictType.RET;
r.retvals.ast = returns;
}
if ( locals!=null ) {
r.locals = ScopeParser.parseTypedArgList(locals.getText(), g.tool.errMgr);
r.locals.type = AttributeDict.DictType.LOCAL;
r.locals.ast = returns;
}
for (GrammarAST a : actions) {
// a = ^(AT ID ACTION)
ActionAST action = (ActionAST) a.getChild(1);
r.namedActions.put(a.getChild(0).getText(), action);
action.resolver = r;
}
}
@Override
public void discoverLexerRule(RuleAST rule, GrammarAST ID, List<GrammarAST> modifiers,
GrammarAST block)
{
int numAlts = block.getChildCount();
Rule r = new Rule(g, ID.getText(), rule, numAlts);
r.mode = currentModeName;
if ( modifiers.size()>0 ) r.modifiers = modifiers;
rules.add(r);
}
}

23
tool/src/org/antlr/v4/semantics/SemanticPipeline.java
View File

@ -29,6 +29,7 @@
package org.antlr.v4.semantics;
import org.antlr.v4.analysis.LeftRecursiveRuleTransformer;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
@ -78,6 +79,20 @@ public class SemanticPipeline {
// don't continue if we get errors in this basic check
if ( false ) return;
// COLLECT RULE OBJECTS
RuleCollector ruleCollector = new RuleCollector(g);
ruleCollector.process(g.ast);
// TRANSFORM LEFT-RECURSIVE RULES
LeftRecursiveRuleTransformer lrtrans =
new LeftRecursiveRuleTransformer(g.ast, ruleCollector.rules, g.tool);
lrtrans.translateLeftRecursiveRules();
// STORE RULES IN GRAMMAR
for (Rule r : ruleCollector.rules) {
g.defineRule(r);
}
// COLLECT SYMBOLS: RULES, ACTIONS, TERMINALS, ...
SymbolCollector collector = new SymbolCollector(g);
collector.process(g.ast);
@ -86,14 +101,6 @@ public class SemanticPipeline {
SymbolChecks symcheck = new SymbolChecks(g, collector);
symcheck.process(); // side-effect: strip away redef'd rules.
// don't continue if we get symbol errors
//if ( ErrorManager.getNumErrors()>0 ) return;
// hmm...we don't get missing arg errors and such if we bail out here
// STORE RULES/ACTIONS/SCOPES IN GRAMMAR
for (Rule r : collector.rules) {
g.defineRule(r);
}
for (GrammarAST a : collector.namedActions) {
g.defineAction(a);
}

8
tool/src/org/antlr/v4/semantics/SymbolChecks.java
View File

@ -77,15 +77,15 @@ public class SymbolChecks {
// So, call order sensitive
//checkForImportedRuleIssues(collector.qualifiedRulerefs);
// done in sem pipe for now
checkForRuleConflicts(collector.rules); // sets nameToRuleMap
checkForRuleConflicts(g.rules.values()); // sets nameToRuleMap
checkActionRedefinitions(collector.namedActions);
checkTokenAliasRedefinitions(collector.tokensDefs);
//checkRuleArgs(collector.rulerefs);
checkForTokenConflicts(collector.tokenIDRefs); // sets tokenIDs
checkForLabelConflicts(collector.rules);
checkForLabelConflicts(g.rules.values());
}
public void checkForRuleConflicts(List<Rule> rules) {
public void checkForRuleConflicts(Collection<Rule> rules) {
if ( rules==null ) return;
for (Rule r : rules) {
Rule prevRule = nameToRuleMap.get(r.name);
@ -194,7 +194,7 @@ public class SymbolChecks {
* defined in surrounding rule. Also they must have same type
* for repeated defs.
*/
public void checkForLabelConflicts(List<Rule> rules) {
public void checkForLabelConflicts(Collection<Rule> rules) {
for (Rule r : rules) {
checkForRuleArgumentAndReturnValueConflicts(r);
Map<String, LabelElementPair> labelNameSpace =

47
tool/src/org/antlr/v4/semantics/SymbolCollector.java
View File

@ -30,8 +30,6 @@
package org.antlr.v4.semantics;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.parse.ScopeParser;
import org.antlr.v4.tool.AttributeDict;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.Rule;
@ -53,11 +51,9 @@ public class SymbolCollector extends GrammarTreeVisitor {
public Grammar g;
// stuff to collect
public List<Rule> rules = new ArrayList<Rule>();
public List<GrammarAST> rulerefs = new ArrayList<GrammarAST>();
public List<GrammarAST> qualifiedRulerefs = new ArrayList<GrammarAST>();
public List<GrammarAST> terminals = new ArrayList<GrammarAST>();
public List<GrammarAST> labels = new ArrayList<GrammarAST>();
public List<GrammarAST> tokenIDRefs = new ArrayList<GrammarAST>();
public Set<String> strings = new HashSet<String>();
public List<GrammarAST> tokensDefs = new ArrayList<GrammarAST>();
@ -101,54 +97,18 @@ public class SymbolCollector extends GrammarTreeVisitor {
List<GrammarAST> actions,
GrammarAST block)
{
int numAlts = block.getChildCount();
Rule r = new Rule(g, ID.getText(), rule, numAlts);
// if ( g.isLexer() ) r.mode = currentModeName;
// if ( modifiers.size()>0 ) r.modifiers = modifiers;
rules.add(r);
currentRule = r;
if ( arg!=null ) {
r.args = ScopeParser.parseTypedArgList(arg.getText(), g.tool.errMgr);
r.args.type = AttributeDict.DictType.ARG;
r.args.ast = arg;
arg.resolver = r.alt[currentOuterAltNumber];
}
if ( returns!=null ) {
r.retvals = ScopeParser.parseTypedArgList(returns.getText(), g.tool.errMgr);
r.retvals.type = AttributeDict.DictType.RET;
r.retvals.ast = returns;
}
if ( locals!=null ) {
r.locals = ScopeParser.parseTypedArgList(locals.getText(), g.tool.errMgr);
r.locals.type = AttributeDict.DictType.LOCAL;
r.locals.ast = returns;
}
for (GrammarAST a : actions) {
// a = ^(AT ID ACTION)
ActionAST action = (ActionAST) a.getChild(1);
currentRule.namedActions.put(a.getChild(0).getText(), action);
action.resolver = currentRule;
}
currentRule = g.getRule(ID.getText());
}
@Override
public void discoverLexerRule(RuleAST rule, GrammarAST ID, List<GrammarAST> modifiers,
GrammarAST block)
{
int numAlts = block.getChildCount();
Rule r = new Rule(g, ID.getText(), rule, numAlts);
r.mode = currentModeName;
if ( modifiers.size()>0 ) r.modifiers = modifiers;
rules.add(r);
currentRule = r;
currentRule = g.getRule(ID.getText());
}
@Override
public void discoverAlt(AltAST alt) {
public void discoverOuterAlt(AltAST alt) {
currentRule.alt[currentOuterAltNumber].ast = alt;
}
@ -180,7 +140,6 @@ public class SymbolCollector extends GrammarTreeVisitor {
public void label(GrammarAST op, GrammarAST ID, GrammarAST element) {
LabelElementPair lp = new LabelElementPair(g, ID, element, op.getType());
currentRule.alt[currentOuterAltNumber].labelDefs.map(ID.getText(), lp);
labels.add(ID);
}
@Override

91
tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java
View File

@ -29,13 +29,15 @@
package org.antlr.v4.tool;
import org.antlr.runtime.*;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.Tree;
import org.antlr.runtime.tree.TreeVisitor;
import org.antlr.runtime.tree.TreeVisitorAction;
import org.antlr.v4.Tool;
import org.antlr.v4.misc.DoubleKeyMap;
import org.antlr.v4.parse.*;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.BlockSetTransformer;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.tool.ast.*;
import java.util.*;
@ -56,16 +58,13 @@ public class GrammarTransformPipeline {
tool.log("grammar", "before: "+root.toStringTree());
integrateImportedGrammars(g);
if ( root.grammarType==ANTLRParser.PARSER || root.grammarType==ANTLRParser.COMBINED ) {
translateLeftRecursiveRules(root);
}
reduceBlocksToSets(root);
expandParameterizedLoops(root);
tool.log("grammar", "after: "+root.toStringTree());
}
public void reduceBlocksToSets(GrammarRootAST root) {
public void reduceBlocksToSets(GrammarAST root) {
org.antlr.runtime.tree.CommonTreeNodeStream nodes =
new org.antlr.runtime.tree.CommonTreeNodeStream(root);
GrammarASTAdaptor adaptor = new GrammarASTAdaptor();
@ -82,7 +81,7 @@ public class GrammarTransformPipeline {
* Parameter must be a token.
* todo: do we want?
*/
public void expandParameterizedLoops(GrammarRootAST root) {
public void expandParameterizedLoops(GrammarAST root) {
TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor());
v.visit(root, new TreeVisitorAction() {
public Object pre(Object t) {
@ -100,84 +99,6 @@ public class GrammarTransformPipeline {
return t;
}
public void translateLeftRecursiveRules(GrammarRootAST ast) {
// TODO: what about -language foo cmd line?
String language = Grammar.getLanguageOption(ast);
// translate all recursive rules
List<String> leftRecursiveRuleNames = new ArrayList<String>();
for (GrammarAST r : ast.getNodesWithType(ANTLRParser.RULE)) {
String ruleName = r.getChild(0).getText();
if ( !Character.isUpperCase(ruleName.charAt(0)) ) {
if ( LeftRecursiveRuleAnalyzer.hasImmediateRecursiveRuleRefs(r, ruleName) ) {
boolean fitsPattern = translateLeftRecursiveRule(ast, r, language);
if ( fitsPattern ) leftRecursiveRuleNames.add(ruleName);
}
}
}
// update all refs to recursive rules to have [0] argument
for (GrammarAST r : ast.getNodesWithType(ANTLRParser.RULE_REF)) {
if ( r.getParent().getType()==ANTLRParser.RULE ) continue; // must be rule def
if ( r.getChildCount()>0 ) continue; // already has arg; must be in rewritten rule
if ( leftRecursiveRuleNames.contains(r.getText()) ) {
// found ref to recursive rule not already rewritten with arg
ActionAST arg = new ActionAST(new CommonToken(ANTLRParser.ARG_ACTION, "0"));
r.addChild(arg);
}
}
}
/** Return true if successful */
public boolean translateLeftRecursiveRule(GrammarRootAST ast,
GrammarAST ruleAST,
String language)
{
//tool.log("grammar", ruleAST.toStringTree());
TokenStream tokens = ast.tokens;
Grammar g = ast.g;
String ruleName = ruleAST.getChild(0).getText();
LeftRecursiveRuleAnalyzer leftRecursiveRuleWalker =
new LeftRecursiveRuleAnalyzer(tokens, ruleAST, tool, ruleName, language);
boolean isLeftRec = false;
try {
// System.out.println("TESTING ---------------\n"+
// leftRecursiveRuleWalker.text(ruleAST));
isLeftRec = leftRecursiveRuleWalker.rec_rule();
}
catch (RecognitionException re) {
isLeftRec = false; // didn't match; oh well
}
if ( !isLeftRec ) return false;
// replace old rule
GrammarAST RULES = (GrammarAST)ast.getFirstChildWithType(ANTLRParser.RULES);
String newRuleText = leftRecursiveRuleWalker.getArtificialOpPrecRule();
// System.out.println("created: "+newRuleText);
GrammarAST t = parseArtificialRule(g, newRuleText);
RULES.setChild(ruleAST.getChildIndex(), t);
tool.log("grammar", "added: "+t.toStringTree());
return true;
}
public GrammarAST parseArtificialRule(final Grammar g, String ruleText) {
ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
GrammarASTAdaptor adaptor = new GrammarASTAdaptor();
CommonTokenStream tokens = new CommonTokenStream(lexer);
ToolANTLRParser p = new ToolANTLRParser(tokens, tool);
p.setTreeAdaptor(adaptor);
try {
ParserRuleReturnScope r = p.rule();
GrammarAST tree = (GrammarAST) r.getTree();
setGrammarPtr(g, tree);
return tree;
}
catch (Exception e) {
tool.errMgr.toolError(ErrorType.INTERNAL_ERROR,
"error parsing rule created during left-recursion detection: "+ruleText,
e);
}
return null;
}
/** Utility visitor that sets grammar ptr in each node */
public static void setGrammarPtr(final Grammar g, GrammarAST tree) {
// ensure each node has pointer to surrounding grammar

71
tool/src/org/antlr/v4/tool/LeftRecursiveRule.java Normal file
View File

@ -0,0 +1,71 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.tool;
import org.antlr.v4.analysis.LeftRecursiveRuleAltInfo;
import org.antlr.v4.misc.OrderedHashMap;
import org.antlr.v4.tool.ast.RuleAST;
import java.util.ArrayList;
import java.util.List;
public class LeftRecursiveRule extends Rule {
public List<LeftRecursiveRuleAltInfo> recPrimaryAlts;
public OrderedHashMap<Integer, LeftRecursiveRuleAltInfo> recOpAlts;
public LeftRecursiveRule(Grammar g, String name, RuleAST ast) {
super(g, name, ast, 1);
alt = new Alternative[numberOfAlts+1]; // always just one
for (int i=1; i<=numberOfAlts; i++) alt[i] = new Alternative(this, i);
}
@Override
public boolean hasAltSpecificContexts() {
return super.hasAltSpecificContexts() || getAltLabels()!=null;
}
/** Get -> labels and also those we deleted for left-recursive rules. */
@Override
public List<String> getAltLabels() {
List<String> labels = new ArrayList<String>();
List<String> normalAltLabels = super.getAltLabels();
if ( normalAltLabels!=null ) labels.addAll(normalAltLabels);
for (int i = 0; i < recPrimaryAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = recPrimaryAlts.get(i);
if ( altInfo.altLabel!=null ) labels.add(altInfo.altLabel);
}
for (int i = 0; i < recOpAlts.size(); i++) {
LeftRecursiveRuleAltInfo altInfo = recOpAlts.getElement(i);
if ( altInfo.altLabel!=null ) labels.add(altInfo.altLabel);
}
if ( labels.size()==0 ) return null;
return labels;
}
}

5
tool/src/org/antlr/v4/tool/Rule.java
View File

@ -192,6 +192,11 @@ public class Rule implements AttributeResolver {
return defs;
}
public boolean hasAltSpecificContexts() {
return getAltLabels()!=null;
}
/** Get -> labels. */
public List<String> getAltLabels() {
List<String> labels = new ArrayList<String>();
for (int i=1; i<=numberOfAlts; i++) {

1
tool/src/org/antlr/v4/tool/ast/AltAST.java
View File

@ -45,6 +45,7 @@ public class AltAST extends GrammarAST {
public AltAST(GrammarAST node) {
super(node);
this.alt = ((AltAST)node).alt;
this.altLabel = ((AltAST)node).altLabel;
}
public AltAST(Token t) { super(t); }

2
tool/test/org/antlr/v4/test/TestASTStructure.gunit
View File

@ -142,7 +142,6 @@ element:
"x=ID*" -> (* (BLOCK (ALT (= x ID))))
"x=b" -> (= x b)
"x=(A|B)" -> (= x (BLOCK (ALT A) (ALT B)))
"x=(A|B)^" -> (= x (^ (BLOCK (ALT A) (ALT B))))
"x=~(A|B)" -> (= x (~ (SET A B)))
"x+=~(A|B)" -> (+= x (~ (SET A B)))
"x+=~(A|B)+"-> (+ (BLOCK (ALT (+= x (~ (SET A B))))))
@ -150,5 +149,4 @@ element:
"x+=ID*" -> (* (BLOCK (ALT (+= x ID))))
"x+='int'*" -> (* (BLOCK (ALT (+= x 'int'))))
"x+=b+" -> (+ (BLOCK (ALT (+= x b))))
"('*'^)*" -> (* (BLOCK (ALT (^ '*'))))
"({blort} 'x')*" -> (* (BLOCK (ALT {blort} 'x')))

60
tool/test/org/antlr/v4/test/TestASTStructure.java
View File

@ -270,79 +270,63 @@ public class TestASTStructure extends org.antlr.v4.gunit.gUnitBase {
@Test public void test_element17() throws Exception {
// gunit test on line 145
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=(A|B)^", 145);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(= x (^ (BLOCK (ALT A) (ALT B))))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element18() throws Exception {
// gunit test on line 146
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=~(A|B)", 146);
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=~(A|B)", 145);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(= x (~ (SET A B)))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element19() throws Exception {
// gunit test on line 147
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)", 147);
@Test public void test_element18() throws Exception {
// gunit test on line 146
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)", 146);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(+= x (~ (SET A B)))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element20() throws Exception {
// gunit test on line 148
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)+", 148);
@Test public void test_element19() throws Exception {
// gunit test on line 147
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)+", 147);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(+ (BLOCK (ALT (+= x (~ (SET A B))))))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element21() throws Exception {
// gunit test on line 149
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b+", 149);
@Test public void test_element20() throws Exception {
// gunit test on line 148
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b+", 148);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(+ (BLOCK (ALT (= x b))))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element22() throws Exception {
// gunit test on line 150
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=ID*", 150);
@Test public void test_element21() throws Exception {
// gunit test on line 149
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=ID*", 149);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(* (BLOCK (ALT (+= x ID))))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element23() throws Exception {
// gunit test on line 151
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+='int'*", 151);
@Test public void test_element22() throws Exception {
// gunit test on line 150
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+='int'*", 150);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(* (BLOCK (ALT (+= x 'int'))))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element24() throws Exception {
// gunit test on line 152
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=b+", 152);
@Test public void test_element23() throws Exception {
// gunit test on line 151
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=b+", 151);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(+ (BLOCK (ALT (+= x b))))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element25() throws Exception {
// gunit test on line 153
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "('*'^)*", 153);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(* (BLOCK (ALT (^ '*'))))";
assertEquals("testing rule element", expecting, actual);
}
@Test public void test_element26() throws Exception {
// gunit test on line 154
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "({blort} 'x')*", 154);
@Test public void test_element24() throws Exception {
// gunit test on line 152
RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "({blort} 'x')*", 152);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(* (BLOCK (ALT {blort} 'x')))";
assertEquals("testing rule element", expecting, actual);

4
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -229,7 +229,7 @@ public class TestATNSerialization extends BaseTest {
"4:PLUS_BLOCK_START 0\n" +
"5:BLOCK_END 0\n" +
"6:PLUS_LOOP_BACK 0\n" +
"7:LOOP_END 0\n" +
"7:LOOP_END 0 6\n" +
"8:BASIC 0\n" +
"9:BASIC 0\n" +
"10:BASIC 0\n" +
@ -406,7 +406,7 @@ public class TestATNSerialization extends BaseTest {
"5:PLUS_BLOCK_START 0\n" +
"6:BLOCK_END 0\n" +
"7:PLUS_LOOP_BACK 0\n" +
"8:LOOP_END 0\n" +
"8:LOOP_END 0 6\n" +
"rule 0:1 3,-1\n" +
"mode 0:0\n" +
"0->1 EPSILON 0,0,0\n" +

46
tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
View File

@ -32,35 +32,10 @@ package org.antlr.v4.test;
import org.junit.Test;
public class TestBasicSemanticErrors extends BaseTest {
static String[] A = {
// INPUT
"grammar A;\n" +
"\n" +
"\n" +
"a : ID<Foo> -> ID ;\n" +
"\n" +
"b : A^ | ((B!|C)) -> C;",
// YIELDS
"error(68): A.g:7:7: alts with rewrites can't use heterogeneous types left of ->\n" +
"error(78): A.g:9:4: AST operator with non-AST output option: ^\n" +
"error(78): A.g:9:11: AST operator with non-AST output option: !\n" +
"error(79): A.g:9:11: rule b alt 2 uses rewrite syntax and also an AST operator\n",
// INPUT
"tree grammar B;\n" +
"\n" +
"a : A;\n" +
"\n" +
"b : ^(. A) ;",
// YIELDS
"error(80): B.g:9:6: Wildcard invalid as root; wildcard can itself be a tree\n" +
"error(81): B.g:1:5: option output=template conflicts with tree grammar filter mode\n"
};
static String[] U = {
// INPUT
"parser grammar U;\n" +
"options { foo=bar; k=\"*\";}\n" +
"options { foo=bar; k=\"3\";}\n" +
"tokens {\n" +
" f='fkj';\n" +
" S = 'a';\n" +
@ -87,26 +62,7 @@ public class TestBasicSemanticErrors extends BaseTest {
"warning(47): U.g:11:21: illegal option greedy\n" +
"warning(47): U.g:14:16: illegal option ick\n" +
"warning(47): U.g:15:16: illegal option x\n",
// INPUT
"tree grammar V;\n" +
"a : A\n" +
" | A B -> template() \"kjsfdkdsj\" \n" +
" ;",
// YIELDS
"warning(47): V.g:3:8: illegal option rewrite\n",
// INPUT
"tree grammar V;\n" +
"options { rewrite=true; }\n" +
"a : A\n" +
" | A B -> template() \"kjsfdkdsj\" \n" +
" ;",
// YIELDS
"warning(47): V.g:3:8: illegal option rewrite\n"
};
@Test public void testA() { super.testErrors(A, false); }
@Test public void testU() { super.testErrors(U, false); }
}

14
tool/test/org/antlr/v4/test/TestLeftRecursion.java
View File

@ -16,8 +16,18 @@ public class TestLeftRecursion extends BaseTest {
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer",
"s", "x y z", debug);
String expecting = "(s (a (a (a x) y) z))\n";
"s", "x", debug);
String expecting = "(s (a x))\n";
assertEquals(expecting, found);
found = execParser("T.g", grammar, "TParser", "TLexer",
"s", "x y", debug);
expecting = "(s (a (a x) y))\n";
assertEquals(expecting, found);
found = execParser("T.g", grammar, "TParser", "TLexer",
"s", "x y z", debug);
expecting = "(s (a (a (a x) y) z))\n";
assertEquals(expecting, found);
}