forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

WOOT! all left-recursive tests pass 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8972]
This commit is contained in:
parrt 2011-07-31 18:14:47 -08:00
parent ec5d74c83e
commit d926ec9661
16 changed files with 129 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
View File

@ -48,7 +48,7 @@ public class RuleContext {
* Not used during ATN simulation; only used during parse that updates
* current location in ATN.
*/
public int s;
public int s = -1;
/** What state invoked the rule associated with this context?
* The "return address" is the followState of invokingState
@ -209,7 +209,7 @@ public class RuleContext {
}
public String toString(BaseRecognizer recog) {
return toString(recog, null);
return toString(recog, RuleContext.EMPTY);
}
public String toString(BaseRecognizer recog, RuleContext stop) {

13
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java
View File

@ -72,10 +72,12 @@ public class ATNConfig {
* We cannot execute predicates dependent upon local context unless
* we know for sure we are in the correct context. Because there is
* no way to do this efficiently, we simply cannot evaluate
* dependent predicates if we are pursuing a global FOLLOW
* operation. closure() tracks the depth of how far we dip into the
* outer context. We then avoid executing predicates that are
* dependent on local context if this depth is > 0.
* dependent predicates unless we are in the rule that initially
* invokes the ATN simulator.
*
* closure() tracks the depth of how far we dip into the
* outer context: depth > 0. Note that it may not be totally
* accurate depth since I don't ever decrement. TODO: make it a boolean then
*/
public int reachesIntoOuterContext;
@ -155,6 +157,9 @@ public class ATNConfig {
buf.append("|");
buf.append(context);
}
if ( reachesIntoOuterContext>0 ) {
buf.append("|up="+reachesIntoOuterContext);
}
// if (isAccept) {
// buf.append("|=>"+alt);
// }

34
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -108,6 +108,9 @@ public class ParserATNSimulator extends ATNSimulator {
{
if ( outerContext==null ) outerContext = RuleContext.EMPTY;
this.outerContext = outerContext;
if ( debug ) System.out.println("ATN decision "+dfa.decision+
" exec LA(1)=="+input.LT(1)+
", outerContext="+outerContext.toString(parser));
RuleContext ctx = RuleContext.EMPTY;
if ( useContext ) ctx = outerContext;
OrderedHashSet<ATNConfig> s0_closure =
@ -134,16 +137,18 @@ public class ParserATNSimulator extends ATNSimulator {
// doesn't create DFA when matching
public int matchATN(TokenStream input, ATNState startState) {
DFA dfa = new DFA(startState);
RuleContext ctx = new ParserRuleContext();
RuleContext ctx = RuleContext.EMPTY;
OrderedHashSet<ATNConfig> s0_closure = computeStartState(startState, ctx);
return execATN(input, dfa, input.index(), s0_closure, false);
}
public int execDFA(TokenStream input, DFA dfa, DFAState s0, RuleContext outerContext) {
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+" exec LA(1)=="+input.LT(1));
// dump(dfa);
if ( outerContext==null ) outerContext = RuleContext.EMPTY;
this.outerContext = outerContext;
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" exec LA(1)=="+input.LT(1)+
", outerContext="+outerContext.toString(parser));
DFAState prevAcceptState = null;
DFAState s = s0;
int t = input.LA(1);
@ -469,15 +474,17 @@ public class ParserATNSimulator extends ATNSimulator {
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
ATNConfig c = new ATNConfig(retState, config.alt, newContext);
// While we have context to pop back from, we may have
// gotten that context AFTER having fallen off a rule.
// Make sure we track that we are now out of context.
c.reachesIntoOuterContext = config.reachesIntoOuterContext;
closure(c, configs, closureBusy);
return;
}
else {
// else if we have no context info, just chase follow links
// but track how far we dip into outer context. Might
// come in handy and we avoid evaluating context dependent
// preds if this is > 0.
config.reachesIntoOuterContext++;
if ( debug ) System.out.println("FALLING off rule "+
parser.getRuleNames()[config.state.ruleIndex]);
}
}
@ -489,7 +496,16 @@ public class ParserATNSimulator extends ATNSimulator {
Transition t = p.transition(i);
boolean ignorePreds = config.traversedAction;
ATNConfig c = getEpsilonTarget(config, t, ignorePreds);
if ( c!=null ) closure(c, configs, closureBusy);
if ( c!=null ) {
if ( config.state instanceof RuleStopState ) {
// fell off end of rule.
// track how far we dip into outer context. Might
// come in handy and we avoid evaluating context dependent
// preds if this is > 0.
c.reachesIntoOuterContext++;
}
closure(c, configs, closureBusy);
}
}
}
@ -559,6 +575,10 @@ public class ParserATNSimulator extends ATNSimulator {
}
public ATNConfig ruleTransition(ATNConfig config, Transition t) {
if ( debug ) {
System.out.println("CALL rule "+parser.getRuleNames()[t.target.ruleIndex]+
", ctx="+config.context);
}
ATNState p = config.state;
RuleContext newContext =
new RuleContext(config.context, p.stateNumber, t.target.stateNumber);

22
tool/playground/T.g
View File

@ -1,23 +1,17 @@
grammar T;
options {output=AST;}
s : e EOF ;
e : e_[0] ;
s : e_[0] EOF ;
e_[int _p]
: e_primary
( {$_p <= 5}? '*'^ e_[6]{}
| {$_p <= 4}? '+'^ e_[5]{}
| {$_p <= 2}? '='<assoc=right>^ e_[2]{}
| {$_p <= 3}? '?'<assoc=right>^ e ':'! e_[3]{}
)*
: e_primary { }
( {19 >= $_p}? '['^ e_[0] ']'! )*
;
e_primary
: ID
: INT
| 'new'^ ID ('[' INT ']')+
;
ID : 'a'..'z'+;
WS : (' '|'\n')+ {skip();} ;
ID : ('a'..'z'|'A'..'Z'|'_'|'$')+;
INT : '0'..'9'+ ;
WS : (' '|'\n') {skip();} ;

8
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -131,7 +131,7 @@ public QStack\<<currentRule.ctxType>\> <currentRule.name>_stk = new QStack\<<cur
_ctx = _localctx;
<currentRule.name>_stk.push(_localctx);
_localctx.start = input.LT(1);
//System.out.println("enter "+ruleNames[<currentRule.index>]);
//System.out.println("enter "+ruleNames[<currentRule.index>]+", LT(1)="+input.LT(1).getText());
<namedActions.init>
<locals; separator="\n">
try {
@ -148,7 +148,7 @@ public QStack\<<currentRule.ctxType>\> <currentRule.name>_stk = new QStack\<<cur
<currentRule.name>_stk.pop();
_ctx = (ParserRuleContext)_ctx.parent;
<finallyAction>
//System.out.println("exit "+ruleNames[<currentRule.index>]);
// System.out.println("exit "+ruleNames[<currentRule.index>]);
}
return _localctx;
}
@ -424,8 +424,8 @@ labelref(x) ::= "<if(!x.isLocal)>_localctx.<endif><x.name>"
// used for left-recursive rules
recRuleDefArg() ::= "int _p"
recRuleArg() ::= "$_p"
recRuleAltPredicate(ruleName,opPrec) ::= "<recRuleArg()> \<= <opPrec>"
recRuleSetResultAction() ::= "root_0=$<ruleName>_primary.tree;"
recRuleAltPredicate(ruleName,opPrec) ::= "<opPrec> >= <recRuleArg()>"
recRuleSetResultAction() ::= "$tree=$<ruleName>_primary.tree;"
recRuleSetReturnAction(src,name) ::= "$<name>=$<src>.<name>;"
// AST stuff (TODO: separate?)

4
tool/src/org/antlr/v4/codegen/ActionTranslator.java
View File

@ -183,7 +183,9 @@ public class ActionTranslator implements ActionSplitterListener {
chunks.add(new QRetValueRef(getRuleLabel(x.getText()), y.getText())); break;
}
case PREDEFINED_RULE:
if ( a.dict == Rule.predefinedRulePropertiesDict ) {
if ( factory.getCurrentRuleFunction()!=null &&
factory.getCurrentRuleFunction().name.equals(x.getText()) )
{
chunks.add(getRulePropertyRef(y));
}
else {

7
tool/src/org/antlr/v4/codegen/ParserFactory.java
View File

@ -240,9 +240,10 @@ public class ParserFactory extends DefaultOutputModelFactory {
}
public boolean needsImplicitLabel(GrammarAST ID, LabeledOp op) {
return op.getLabels().size()==0 &&
(getCurrentOuterMostAlt().tokenRefsInActions.containsKey(ID.getText()) ||
getCurrentOuterMostAlt().ruleRefsInActions.containsKey(ID.getText()));
Alternative currentOuterMostAlt = getCurrentOuterMostAlt();
boolean actionRefsAsToken = currentOuterMostAlt.tokenRefsInActions.containsKey(ID.getText());
boolean actionRefsAsRule = currentOuterMostAlt.ruleRefsInActions.containsKey(ID.getText());
return op.getLabels().size()==0 && (actionRefsAsToken || actionRefsAsRule);
}
// AST REWRITE

2
tool/src/org/antlr/v4/codegen/model/ParserFile.java
View File

@ -48,7 +48,7 @@ public class ParserFile extends OutputModelObject {
this.fileName = fileName;
Grammar g = factory.getGrammar();
TokenLabelType = g.getOption("TokenLabelType");
ASTLabelType = g.getOption("ASTLabelType");
ASTLabelType = g.getOption("ASTLabelType", "CommonTree");
namedActions = new HashMap<String, Action>();
for (String name : g.namedActions.keySet()) {
GrammarAST ast = g.namedActions.get(name);

17
tool/src/org/antlr/v4/parse/ActionSplitter.g
View File

@ -42,18 +42,22 @@ LINE_COMMENT
: '//' ~('\n'|'\r')* '\r'? '\n' {delegate.text($text);}
;
SET_DYNAMIC_SCOPE_ATTR
SET_NONLOCAL_ATTR
: '$' x=ID '::' y=ID WS? '=' expr=ATTR_VALUE_EXPR ';'
{delegate.setNonLocalAttr($text, $x, $y, $expr);}
{
delegate.setNonLocalAttr($text, $x, $y, $expr);
}
;
DYNAMIC_SCOPE_ATTR
NONLOCAL_ATTR
: '$' x=ID '::' y=ID {delegate.nonLocalAttr($text, $x, $y);}
;
SET_QUALIFIED_ATTR
: '$' x=ID '.' y=ID WS? '=' expr=ATTR_VALUE_EXPR ';'
{delegate.setQualifiedAttr($text, $x, $y, $expr);}
{
delegate.setQualifiedAttr($text, $x, $y, $expr);
}
;
QUALIFIED_ATTR
@ -61,7 +65,10 @@ QUALIFIED_ATTR
;
SET_ATTR
: '$' x=ID WS? '=' expr=ATTR_VALUE_EXPR ';' {delegate.setAttr($text, $x, $expr);}
: '$' x=ID WS? '=' expr=ATTR_VALUE_EXPR ';'
{
delegate.setAttr($text, $x, $expr);
}
;
ATTR

8
tool/src/org/antlr/v4/parse/LeftRecursiveRuleAnalyzer.java
View File

@ -83,7 +83,6 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
@Override
public void setReturnValues(GrammarAST t) {
System.out.println(t);
retvals = t;
}
@ -97,6 +96,9 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
if ( a.equals(ASSOC.right.toString()) ) {
assoc = ASSOC.right;
}
else if ( a.equals(ASSOC.left.toString()) ) {
assoc = ASSOC.left;
}
else {
tool.errMgr.toolError(ErrorType.ILLEGAL_OPTION_VALUE, "assoc", assoc);
}
@ -234,10 +236,10 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
return ruleST.render();
}
public String getArtificialOpPrecRule() {
public String getArtificialOpPrecRule(boolean buildAST) {
ST ruleST = recRuleTemplates.getInstanceOf("recRule");
ruleST.add("ruleName", ruleName);
// TODO: ruleST.add("buildAST", grammar.hasASTOption());
ruleST.add("buildAST", buildAST);
ST argDefST =
codegenTemplates.getInstanceOf("recRuleDefArg");
ruleST.add("precArgDef", argDefST);

11
tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g
View File

@ -59,6 +59,7 @@ public void otherAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) {}
public void setReturnValues(GrammarAST t) {}
}
// TODO: can get parser errors for not matching pattern; make them go away
public
rec_rule returns [boolean isLeftRec]
@init
@ -67,12 +68,12 @@ rec_rule returns [boolean isLeftRec]
}
: ^( r=RULE id=ID {ruleName=$id.getText();}
ruleModifier?
(^(ARG ARG_ACTION))?
(^(RET ARG_ACTION))?
( ^(THROWS .+) )?
( ^(LOCALS ARG_ACTION) )?
// (ARG_ACTION)? shouldn't allow args, right?
(^(RETURNS a=ARG_ACTION {setReturnValues($a);}))?
// ( ^(THROWS .+) )? don't allow
( ^(LOCALS ARG_ACTION) )? // TODO: copy these to gen'd code
( ^(OPTIONS .*)
| ^(AT ID ACTION)
| ^(AT ID ACTION) // TODO: copy
)*
ruleBlock {$isLeftRec = $ruleBlock.isLeftRec;}
exceptionGroup

43
tool/src/org/antlr/v4/semantics/ActionSniffer.java
View File

@ -35,7 +35,9 @@ import org.antlr.v4.tool.*;
import java.util.List;
/** Find token and rule refs, side-effect: update Alternatives */
/** Find token and rule refs plus refs to them in actions;
* side-effect: update Alternatives
*/
public class ActionSniffer extends BlankActionSplitterListener {
public Grammar g;
public Rule r; // null if action outside of rule
@ -64,7 +66,40 @@ public class ActionSniffer extends BlankActionSplitterListener {
System.out.println(node.chunks);
}
public void attr(String expr, Token x) {
public void processNested(Token actionToken) {
ANTLRStringStream in = new ANTLRStringStream(actionToken.getText());
in.setLine(actionToken.getLine());
in.setCharPositionInLine(actionToken.getCharPositionInLine());
ActionSplitter splitter = new ActionSplitter(in, this);
// forces eval, triggers listener methods
splitter.getActionTokens();
}
@Override
public void attr(String expr, Token x) { trackRef(x); }
@Override
public void qualifiedAttr(String expr, Token x, Token y) { trackRef(x); }
@Override
public void setAttr(String expr, Token x, Token rhs) {
trackRef(x);
processNested(rhs);
}
@Override
public void setQualifiedAttr(String expr, Token x, Token y, Token rhs) {
trackRef(x);
processNested(rhs);
}
@Override
public void setNonLocalAttr(String expr, Token x, Token y, Token rhs) {
processNested(rhs);
}
public void trackRef(Token x) {
List<TerminalAST> xRefs = alt.tokenRefs.get(x.getText());
if ( alt!=null && xRefs!=null ) {
alt.tokenRefsInActions.map(x.getText(), node);
@ -74,8 +109,4 @@ public class ActionSniffer extends BlankActionSplitterListener {
alt.ruleRefsInActions.map(x.getText(), node);
}
}
public void qualifiedAttr(String expr, Token x, Token y) {
attr(expr, x);
}
}

6
tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java
View File

@ -108,7 +108,11 @@ public class GrammarTransformPipeline {
List<String> rules = new ArrayList<String>();
rules.add( leftRecursiveRuleWalker.getArtificialPrecStartRule() ) ;
rules.add( leftRecursiveRuleWalker.getArtificialOpPrecRule() );
String outputOption = ast.getOption("output");
boolean buildAST = outputOption!=null && outputOption.equals("AST");
rules.add( leftRecursiveRuleWalker.getArtificialOpPrecRule(buildAST) );
rules.add( leftRecursiveRuleWalker.getArtificialPrimaryRule() );
for (String ruleText : rules) {
// System.out.println("created: "+ruleText);

1
tool/src/org/antlr/v4/tool/Rule.java
View File

@ -110,6 +110,7 @@ public class Rule implements AttributeResolver {
public boolean isStartRule = true; // nobody calls us
/** 1..n alts */
public Alternative[] alt;
/** All rules have unique index 0..n-1 */

43
tool/test/org/antlr/v4/test/TestAttributeChecks.java
View File

@ -23,27 +23,6 @@ public class TestAttributeChecks extends BaseTest {
"c : ;\n" +
"d : ;\n";
String scopeTemplate =
"parser grammar A;\n"+
"@members {\n" +
"<members>\n" +
"}\n" +
"scope S { int i; }\n" +
"a[int x] returns [int y]\n" +
"scope { int z; }\n" +
"scope S;\n" +
"@init {<init>}\n" +
" : lab=b[34] {\n" +
" <inline>" +
" }\n" +
" ;\n" +
" finally {<finally>}\n" +
"b[int d] returns [int e]\n" +
"scope { int f; }\n" +
" : {<inline2>}\n" +
" ;\n" +
"c : ;";
String[] membersChecks = {
"$a", "error(29): A.g:2:11: unknown attribute reference a in $a\n",
"$a.y", "error(29): A.g:2:11: unknown attribute reference a in $a.y\n",
@ -56,7 +35,7 @@ public class TestAttributeChecks extends BaseTest {
"$y = $x", "",
"$lab.e", "",
"$ids", "",
"$a", "error(33): A.g:4:8: missing attribute access on rule reference a in $a\n",
"$c", "error(29): A.g:4:8: unknown attribute reference c in $c\n",
"$a.q", "error(31): A.g:4:10: unknown attribute q for rule a in $a.q\n",
@ -210,7 +189,7 @@ public class TestAttributeChecks extends BaseTest {
@Test public void testInitActions() throws RecognitionException {
testActions("init", initChecks, attributeTemplate);
}
}
@Test public void testInlineActions() throws RecognitionException {
testActions("inline", inlineChecks, attributeTemplate);
@ -224,22 +203,6 @@ public class TestAttributeChecks extends BaseTest {
testActions("finally", finallyChecks, attributeTemplate);
}
@Test public void testDynMembersActions() throws RecognitionException {
testActions("members", dynMembersChecks, scopeTemplate);
}
@Test public void testDynInitActions() throws RecognitionException {
testActions("init", dynInitChecks, scopeTemplate);
}
@Test public void testDynInlineActions() throws RecognitionException {
testActions("inline", dynInlineChecks, scopeTemplate);
}
@Test public void testDynFinallyActions() throws RecognitionException {
testActions("finally", dynFinallyChecks, scopeTemplate);
}
@Test public void testTokenRef() throws RecognitionException {
String grammar =
"parser grammar S;\n" +
@ -266,7 +229,7 @@ public class TestAttributeChecks extends BaseTest {
String action = "$Symbols::x";
}
public void testActions(String location, String[] pairs, String template) {
for (int i = 0; i < pairs.length; i+=2) {
String action = pairs[i];

4
tool/test/org/antlr/v4/test/TestLeftRecursion.java
View File

@ -261,7 +261,7 @@ public class TestLeftRecursion extends BaseTest {
" | ('~'^|'!'^) e\n" +
" | e ('*'^|'/'^|'%'^) e\n" +
" | e ('+'^|'-'^) e\n" +
" | e ('<'^ '<' | '>'^ '>' '>' | '>'^ '>') e\n" +
" | e ('<<'^ | '>>>'^ | '>>'^) e\n" +
" | e ('<='^ | '>='^ | '>'^ | '<'^) e\n" +
" | e 'instanceof'^ e\n" +
" | e ('=='^ | '!='^) e\n" +
@ -311,7 +311,7 @@ public class TestLeftRecursion extends BaseTest {
"a|b&c", "(| a (& b c))",
"(a|b)&c", "(& (| a b) c)",
"a > b", "(> a b)",
"a >> b", "(> a b)", // text is from one token
"a >> b", "(>> a b)", // text is from one token
"a < b", "(< a b)",
"(T)x", "(( T x)",