labels allowed in left recursive rule refs.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9834]
This commit is contained in:
parrt 2012-01-05 18:35:20 -08:00
parent 4ae23f4a64
commit e8ae8ee005
6 changed files with 132 additions and 154 deletions

View File

@ -3,6 +3,14 @@ ANTLR v4 Honey Badger early access
Jan 5, 2012
* Deleted code to call specific listeners by mistake. added back.
* Labels allowed in left-recursive rules:
e returns [int v]
: a=e '*' b=e {$v = $a.v * $b.v;}
| a=e '+' b=e {$v = $a.v + $b.v;}
| INT {$v = $INT.int;}
| '(' x=e ')' {$v = $x.v;}
;
Jan 4, 2012

View File

@ -34,24 +34,27 @@
group LeftRecursiveRules;
recRule(ruleName, precArgDef, argName, primaryAlts, opAlts, setResultAction,
userRetvals, userRetvalAssignments) ::=
userRetvals, leftRecursiveRuleRefLabels) ::=
<<
<ruleName>[<precArgDef>]<if(userRetvals)> returns [<userRetvals>]<endif>
options {simrecursion_=true;}
: ( <primaryAlts; separator="\n | ">
: ( <primaryAlts:{alt | <alt.altText> /* <alt.altLabel> */}; separator="\n | ">
)
<if(userRetvals)>
{
<userRetvalAssignments; separator="\n">
}
<endif>
( options {simrecursion_=true;}
: <opAlts; separator="\n | ">
:
<opAlts; separator=" |\n\n">
<if(leftRecursiveRuleRefLabels)>
| {/* (safely) force ANTLR to know about left-recursive rule labels we removed */}
BOGUS_ <leftRecursiveRuleRefLabels:{lab | <lab>=<ruleName> BOGUS_}; separator=" ">
<endif>
)*
;
>>
recRuleAlt(alt, pred) ::= "{<pred>}? <alt>"
recRuleRef(ruleName, arg) ::= "<ruleName>[<arg>]"
recRuleAlt(alt, startAction, pred) ::= <<
{<pred>}?
{
<startAction>
}
<alt>
>>

View File

@ -189,6 +189,8 @@ LRecursiveRuleFunction(currentRule,code,locals,ruleCtx,altLabelCtxs,namedActions
<if(currentRule.modifiers)><currentRule.modifiers:{f | <f> }><else>public final <endif><currentRule.ctxType> <currentRule.name>(<currentRule.args; separator=",">) throws RecognitionException {
ParserRuleContext\<Token> _parentctx = _ctx;
<currentRule.ctxType> _localctx = new <currentRule.ctxType>(_ctx, <currentRule.startState><currentRule.args:{a | , <a.name>}>);
<currentRule.ctxType> _prevctx = _localctx;
int _startState = <currentRule.startState>;
pushNewRecursionContext(_localctx, RULE_<currentRule.name>);
<namedActions.init>
<locals; separator="\n">
@ -354,9 +356,7 @@ while ( _alt<choice.uniqueID>!=<choice.exitAlt> && _alt<choice.uniqueID>!=-1 ) {
if ( _alt<choice.uniqueID>==1 ) {
<if(choice.isForRecursiveRule)>
if ( _parseListeners!=null ) triggerExitRuleEvent();
_localctx = new <currentRule.name>Context(_parentctx, <currentRule.startState>, _p);
_localctx.addChild(_ctx);
pushNewRecursionContext(_localctx, RULE_<currentRule.name>);
_prevctx = _localctx;
<endif>
<alts> <! should only be one !>
}
@ -546,6 +546,12 @@ recRuleArg() ::= "$_p"
recRuleAltPredicate(ruleName,opPrec) ::= "<opPrec> >= <recRuleArg()>"
recRuleSetResultAction() ::= "$tree=$<ruleName>.tree;"
recRuleSetReturnAction(src,name) ::= "$<name>=$<src>.<name>;"
recRuleAltStartAction(ruleName, ctxName, label) ::= <<
_localctx = new <ctxName>Context(_parentctx, _startState, _p);
_localctx.addChild(_prevctx);
<if(label)>_localctx.<label> = _prevctx;<endif>
pushNewRecursionContext(_localctx, RULE_<ruleName>);
>>
LexerFile(lexerFile, lexer, namedActions) ::= <<
// $ANTLR ANTLRVersion> <lexerFile.fileName> generatedTimestamp>

View File

@ -35,7 +35,6 @@ import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.Tool;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.tool.AttributeDict;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTWithOptions;
@ -49,13 +48,27 @@ import java.util.*;
public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
public static enum ASSOC { left, right }
public static class Alt {
public Alt(String altText) {
this(altText, null);
}
public Alt(String altText, String leftRecursiveRuleRefLabel) {
this.altText = altText;
this.leftRecursiveRuleRefLabel = leftRecursiveRuleRefLabel;
}
public String leftRecursiveRuleRefLabel;
public String altLabel;
public String altText;
}
public Tool tool;
public String ruleName;
public LinkedHashMap<Integer, String> binaryAlts = new LinkedHashMap<Integer, String>();
public LinkedHashMap<Integer, String> ternaryAlts = new LinkedHashMap<Integer, String>();
public LinkedHashMap<Integer, String> suffixAlts = new LinkedHashMap<Integer, String>();
public List<String> prefixAlts = new ArrayList<String>();
public List<String> otherAlts = new ArrayList<String>();
public LinkedHashMap<Integer, Alt> binaryAlts = new LinkedHashMap<Integer, Alt>();
public LinkedHashMap<Integer, Alt> ternaryAlts = new LinkedHashMap<Integer, Alt>();
public LinkedHashMap<Integer, Alt> suffixAlts = new LinkedHashMap<Integer, Alt>();
public List<Alt> prefixAlts = new ArrayList<Alt>();
public List<Alt> otherAlts = new ArrayList<Alt>();
public List<String> leftRecursiveRuleRefLabels = new ArrayList<String>();
public GrammarAST retvals;
@ -122,103 +135,73 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
}
@Override
public void binaryAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {
public void binaryAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
stripLeftRecursion(altTree);
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
stripAssocOptions(altTree);
// rewrite e to be e_[rec_arg]
int nextPrec = nextPrecedence(alt);
ST refST = recRuleTemplates.getInstanceOf("recRuleRef");
refST.add("ruleName", ruleName);
refST.add("arg", nextPrec);
altTree = replaceRuleRefs(altTree, refST.render());
altTree = addPrecedenceArgToRules(altTree, nextPrec);
String altText = text(altTree);
altText = altText.trim();
if ( rewriteTree!=null ) {
rewriteTree = rewriteTree.dupTree();
rewriteTree = replaceRuleRefs(rewriteTree, "$"+ruleName);
}
String rewriteText = text(rewriteTree);
binaryAlts.put(alt, altText + " " + rewriteText);
binaryAlts.put(alt, new Alt(altText, label));
//System.out.println("binaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
/** Convert e ? e : e -> ? e : e_[nextPrec] */
@Override
public void ternaryAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {
public void ternaryAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
stripLeftRecursion(altTree);
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
stripAssocOptions(altTree);
int nextPrec = nextPrecedence(alt);
ST refST = recRuleTemplates.getInstanceOf("recRuleRef");
refST.add("ruleName", ruleName);
refST.add("arg", nextPrec);
altTree = replaceLastRuleRef(altTree, refST.render());
altTree = addPrecedenceArgToLastRule(altTree, nextPrec);
String altText = text(altTree);
altText = altText.trim();
if ( rewriteTree!=null ) {
rewriteTree = rewriteTree.dupTree();
rewriteTree = replaceRuleRefs(rewriteTree, "$" + ruleName);
}
String rewriteText = text(rewriteTree);
ternaryAlts.put(alt, altText + " " + rewriteText);
ternaryAlts.put(alt, new Alt(altText, label));
//System.out.println("ternaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void prefixAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {
public void prefixAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
int nextPrec = precedence(alt);
// rewrite e to be e_[rec_arg]
ST refST = recRuleTemplates.getInstanceOf("recRuleRef");
refST.add("ruleName", ruleName);
refST.add("arg", nextPrec);
altTree = replaceRuleRefs(altTree, refST.render());
// rewrite e to be e_[prec]
altTree = addPrecedenceArgToRules(altTree, nextPrec);
String altText = text(altTree);
altText = altText.trim();
if ( rewriteTree!=null ) {
rewriteTree = rewriteTree.dupTree();
rewriteTree = replaceRuleRefs(rewriteTree, ruleName);
}
String rewriteText = text(rewriteTree);
prefixAlts.add(altText + " " + rewriteText);
prefixAlts.add(new Alt(altText));
//System.out.println("prefixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void suffixAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {
public void suffixAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
stripLeftRecursion(altTree);
if ( rewriteTree!=null ) {
rewriteTree = rewriteTree.dupTree();
rewriteTree = replaceRuleRefs(rewriteTree, "$" + ruleName);
}
String rewriteText = text(rewriteTree);
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
String altText = text(altTree);
altText = altText.trim();
suffixAlts.put(alt, altText + " " + rewriteText);
suffixAlts.put(alt, new Alt(altText, label));
// System.out.println("suffixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
@Override
public void otherAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {
public void otherAlt(GrammarAST altTree, int alt) {
altTree = altTree.dupTree();
if ( rewriteTree!=null ) rewriteTree = rewriteTree.dupTree();
stripLeftRecursion(altTree);
String label = stripLeftRecursion(altTree);
leftRecursiveRuleRefLabels.add(label);
String altText = text(altTree);
String rewriteText = text(rewriteTree);
otherAlts.add(altText + " " + rewriteText);
//System.out.println("otherAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
otherAlts.add(new Alt(altText, label));
// System.out.println("otherAlt " + alt + ": " + altText);
}
// --------- get transformed rules ----------------
@ -233,45 +216,43 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
ST setResultST = codegenTemplates.getInstanceOf("recRuleSetResultAction");
ruleST.add("setResultAction", setResultST);
ruleST.add("userRetvals", retvals);
fillRetValAssignments(ruleST);
LinkedHashMap<Integer, String> opPrecRuleAlts = new LinkedHashMap<Integer, String>();
LinkedHashMap<Integer, Alt> opPrecRuleAlts = new LinkedHashMap<Integer, Alt>();
opPrecRuleAlts.putAll(binaryAlts);
opPrecRuleAlts.putAll(ternaryAlts);
opPrecRuleAlts.putAll(suffixAlts);
for (int alt : opPrecRuleAlts.keySet()) {
String altText = opPrecRuleAlts.get(alt);
Alt altInfo = opPrecRuleAlts.get(alt);
ST altST = recRuleTemplates.getInstanceOf("recRuleAlt");
ST predST = codegenTemplates.getInstanceOf("recRuleAltPredicate");
ST altActionST = codegenTemplates.getInstanceOf("recRuleAltStartAction");
altActionST.add("ctxName", ruleName); // todo: handle alt labels
altActionST.add("ruleName", ruleName);
altActionST.add("label", altInfo.leftRecursiveRuleRefLabel);
predST.add("opPrec", precedence(alt));
predST.add("ruleName", ruleName);
altST.add("pred", predST);
altST.add("alt", altText);
altST.add("alt", altInfo.altText);
altST.add("startAction", altActionST);
ruleST.add("opAlts", altST);
}
ruleST.add("primaryAlts", prefixAlts);
ruleST.add("primaryAlts", otherAlts);
ruleST.add("leftRecursiveRuleRefLabels", leftRecursiveRuleRefLabels);
tool.log("left-recursion", ruleST.render());
return ruleST.render();
}
// public String getArtificialPrimaryRule() {
// ST ruleST = recRuleTemplates.getInstanceOf("recPrimaryRule");
// ruleST.add("ruleName", ruleName);
// ruleST.add("alts", prefixAlts);
// ruleST.add("alts", otherAlts);
// ruleST.add("userRetvals", retvals);
// tool.log("left-recursion", ruleST.render());
// return ruleST.render();
// }
public GrammarAST replaceRuleRefs(GrammarAST t, String name) {
public GrammarAST addPrecedenceArgToRules(GrammarAST t, int prec) {
if ( t==null ) return null;
for (GrammarAST rref : t.getNodesWithType(RULE_REF)) {
if ( rref.getText().equals(ruleName) ) rref.setText(name);
if ( rref.getText().equals(ruleName) ) {
rref.setText(ruleName+"["+prec+"]");
}
}
return t;
}
@ -299,41 +280,51 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
/**
* Match (RULE RULE_REF (BLOCK (ALT .*) (ALT RULE_REF[self] .*) (ALT .*)))
* Match (RULE RULE_REF (BLOCK (ALT .*) (ALT (ASSIGN ID RULE_REF[self]) .*) (ALT .*)))
*/
public static boolean hasImmediateRecursiveRuleRefs(GrammarAST t, String ruleName) {
if ( t==null ) return false;
GrammarAST blk = (GrammarAST)t.getFirstChildWithType(BLOCK);
if ( blk==null ) return false;
List<GrammarAST> ruleRefs = blk.getNodesWithType(RULE_REF);
if ( ruleRefs==null ) return false;
for (GrammarAST rref : ruleRefs) {
if ( rref.getChildIndex()==0 && rref.getText().equals(ruleName) ) return true;
int n = blk.getChildren().size();
for (int i = 0; i < n; i++) {
GrammarAST alt = (GrammarAST)blk.getChildren().get(i);
Tree first = alt.getChild(0);
if ( first.getType()==RULE_REF && first.getText().equals(ruleName) ) return true;
Tree rref = first.getChild(1);
if ( rref!=null && rref.getType()==RULE_REF && rref.getText().equals(ruleName) ) return true;
}
return false;
}
public GrammarAST replaceLastRuleRef(GrammarAST t, String name) {
public GrammarAST addPrecedenceArgToLastRule(GrammarAST t, int prec) {
if ( t==null ) return null;
GrammarAST last = null;
for (GrammarAST rref : t.getNodesWithType(RULE_REF)) { last = rref; }
if ( last !=null && last.getText().equals(ruleName) ) last.setText(name);
if ( last !=null && last.getText().equals(ruleName) ) {
last.setText(ruleName+"["+prec+"]");
}
return t;
}
// TODO: this strips the tree properly, but since text()
// uses the start of stop token index and gets text from that
// ineffectively ignores this routine.
public void stripLeftRecursion(GrammarAST altAST) {
GrammarAST rref = (GrammarAST)altAST.getChild(0);
if ( rref.getType()== ANTLRParser.RULE_REF &&
rref.getText().equals(ruleName))
public String stripLeftRecursion(GrammarAST altAST) {
String label=null;
GrammarAST first = (GrammarAST)altAST.getChild(0);
Tree rref = first.getChild(1); // if label=rule
if ( (first.getType()==RULE_REF && first.getText().equals(ruleName)) ||
(rref!=null && rref.getType()==RULE_REF && rref.getText().equals(ruleName)) )
{
// remove rule ref
if ( first.getType()==ASSIGN ) label = first.getChild(0).getText();
// remove rule ref (first child)
altAST.deleteChild(0);
// reset index so it prints properly
GrammarAST newFirstChild = (GrammarAST) altAST.getChild(0);
GrammarAST newFirstChild = (GrammarAST)altAST.getChild(0);
altAST.setTokenStartIndex(newFirstChild.getTokenStartIndex());
}
return label;
}
public String text(GrammarAST t) {
@ -354,21 +345,6 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
return p;
}
public void fillRetValAssignments(ST ruleST) {
if ( retvals==null ) return;
// complicated since we must be target-independent
AttributeDict args = ScopeParser.parseTypedArgList(retvals.token.getText(), tool.errMgr);
for (String name : args.attributes.keySet()) {
ST setRetValST =
codegenTemplates.getInstanceOf("recRuleSetReturnAction");
setRetValST.add("src", ruleName);
setRetValST.add("name", name);
ruleST.add("userRetvalAssignments",setRetValST);
}
}
@Override
public String toString() {
return "PrecRuleOperatorCollector{" +

View File

@ -52,11 +52,11 @@ private int currentOuterAltNumber; // which outer alt of rule?
public int numAlts; // how many alts for this rule total?
public void setTokenPrec(GrammarAST t, int alt) {}
public void binaryAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {}
public void ternaryAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {}
public void prefixAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {}
public void suffixAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {}
public void otherAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {}
public void binaryAlt(GrammarAST altTree, int alt) {}
public void ternaryAlt(GrammarAST altTree, int alt) {}
public void prefixAlt(GrammarAST altTree, int alt) {}
public void suffixAlt(GrammarAST altTree, int alt) {}
public void otherAlt(GrammarAST altTree, int alt) {}
public void setReturnValues(GrammarAST t) {}
}
@ -105,7 +105,7 @@ ruleBlock returns [boolean isLeftRec]
@init{boolean lr=false; this.numAlts = $start.getChildCount();}
: ^( BLOCK
(
o=outerAlternative[null]
o=outerAlternative
{if ($o.isLeftRec) $isLeftRec = true;}
{currentOuterAltNumber++;}
)+
@ -113,31 +113,31 @@ ruleBlock returns [boolean isLeftRec]
;
/** An alt is either prefix, suffix, binary, or ternary operation or "other" */
outerAlternative[GrammarAST rew] returns [boolean isLeftRec]
outerAlternative returns [boolean isLeftRec]
: (binaryMultipleOp)=> binaryMultipleOp
{binaryAlt($start, $rew, currentOuterAltNumber); $isLeftRec=true;}
{binaryAlt($start, currentOuterAltNumber); $isLeftRec=true;}
| (binary)=> binary
{binaryAlt($start, $rew, currentOuterAltNumber); $isLeftRec=true;}
{binaryAlt($start, currentOuterAltNumber); $isLeftRec=true;}
| (ternary)=> ternary
{ternaryAlt($start, $rew, currentOuterAltNumber); $isLeftRec=true;}
{ternaryAlt($start, currentOuterAltNumber); $isLeftRec=true;}
| (prefix)=> prefix
{prefixAlt($start, $rew, currentOuterAltNumber);}
{prefixAlt($start, currentOuterAltNumber);}
| (suffix)=> suffix
{suffixAlt($start, $rew, currentOuterAltNumber); $isLeftRec=true;}
{suffixAlt($start, currentOuterAltNumber); $isLeftRec=true;}
| ^(ALT element+) // "other" case
{otherAlt($start, $rew, currentOuterAltNumber);}
{otherAlt($start, currentOuterAltNumber);}
;
binary
: ^( ALT recurse (op=token)+ {setTokenPrec($op.t, currentOuterAltNumber);} recurse )
: ^( ALT recurse (op=token)+ {setTokenPrec($op.t, currentOuterAltNumber);} recurse ACTION? )
;
binaryMultipleOp
: ^( ALT recurse ^( BLOCK ( ^( ALT (op=token)+ {setTokenPrec($op.t, currentOuterAltNumber);} ) )+ ) recurse )
: ^( ALT recurse ^( BLOCK ( ^( ALT (op=token)+ {setTokenPrec($op.t, currentOuterAltNumber);} ) )+ ) recurse ACTION? )
;
ternary
: ^( ALT recurse op=token recurse token recurse ) {setTokenPrec($op.t, currentOuterAltNumber);}
: ^( ALT recurse op=token recurse token recurse ACTION? ) {setTokenPrec($op.t, currentOuterAltNumber);}
;
prefix

View File

@ -202,33 +202,18 @@ public class TestLeftRecursion extends BaseTest {
"grammar T;\n" +
"s : e {System.out.println($e.v);} ;\n" +
"e returns [int v, List<String> ignored]\n" +
" : e '*' b=e {$v *= $b.v;}\n" +
" | e '+' b=e {$v += $b.v;}\n" +
" : a=e '*' b=e {$v = $a.v * $b.v;}\n" +
" | a=e '+' b=e {$v = $a.v + $b.v;}\n" +
" | INT {$v = $INT.int;}\n" +
" | '(' x=e ')' {$v = $x.v;}\n" +
" ;\n" +
"INT : '0'..'9'+ ;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String[] tests = {
"4", "4",
"1+2", "3",
};
runTests(grammar, tests, "s");
}
@Test public void testReturnValueAndActionsAndASTs() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {System.out.println($ctx.toStringTree(this));} : e {System.out.print(\"v=\"+$e.v+\", \");} ;\n" +
"e returns [int v, List<String> ignored]\n" +
" : e '*' b=e {$v *= $b.v;}\n" +
" | e '+' b=e {$v += $b.v;}\n" +
" | INT {$v = $INT.int;}\n" +
" ;\n" +
"INT : '0'..'9'+ ;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String[] tests = {
"4", "v=4, 4",
"1+2", "v=3, (+ 1 2)",
"1+2", "3",
"1+2*3", "7",
"(1+2)*3", "9",
};
runTests(grammar, tests, "s");
}