From 46d5458617cd44ac54a188898620ef8c7663a456 Mon Sep 17 00:00:00 2001 From: sharwell Date: Thu, 9 Feb 2012 01:11:09 -0600 Subject: [PATCH 01/25] Fix handling of alts with some configs predicated and some configs unpredicated (should be treated as always true) --- .../v4/runtime/atn/ParserATNSimulator.java | 63 +++++++++++-------- .../antlr/v4/runtime/atn/SemanticContext.java | 9 +-- .../antlr/v4/test/TestSemPredEvalParser.java | 2 +- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 51b31e458..b01bbea3e 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -565,21 +565,28 @@ public class ParserATNSimulator extends ATNSimulator { int nalts = decState.getNumberOfTransitions(); List predPredictions = predicateDFAState(D, D.configset, outerContext, nalts); - IntervalSet conflictingAlts = getConflictingAltsFromConfigSet(D.configset); - if ( D.predicates.size() < conflictingAlts.size() ) { - reportInsufficientPredicates(dfa, startIndex, input.index(), - conflictingAlts, - decState, - getPredsForAmbigAlts(conflictingAlts, D.configset, nalts), - D.configset, - false); + if ( predPredictions!=null ) { + IntervalSet conflictingAlts = getConflictingAltsFromConfigSet(D.configset); + if ( D.predicates.size() < conflictingAlts.size() ) { + reportInsufficientPredicates(dfa, startIndex, input.index(), + conflictingAlts, + decState, + getPredsForAmbigAlts(conflictingAlts, D.configset, nalts), + D.configset, + false); + } + input.seek(startIndex); + predictedAlt = evalSemanticContext(predPredictions, outerContext); + if ( predictedAlt!=ATN.INVALID_ALT_NUMBER ) { + return predictedAlt; + } + + if (D.prediction == ATN.INVALID_ALT_NUMBER) { + throw noViableAlt(input, outerContext, D.configset, startIndex); + } + + predictedAlt = D.prediction; } - input.seek(startIndex); - predictedAlt = evalSemanticContext(predPredictions, outerContext); - if ( predictedAlt!=ATN.INVALID_ALT_NUMBER ) { - return predictedAlt; - } - throw noViableAlt(input, outerContext, D.configset, startIndex); } if ( D.isAcceptState ) return predictedAlt; @@ -736,14 +743,9 @@ public class ParserATNSimulator extends ATNSimulator { // we have a validating predicate; test it // Update DFA so reach becomes accept state with predicate predPredictions = getPredicatePredictions(conflictingAlts, altToPred); - if ( D.isCtxSensitive ) { -// D.ctxToPredicates.put(outerContext, predPredictions); - } - else { - D.predicates = predPredictions; - } + D.predicates = predPredictions; + D.prediction = ATN.INVALID_ALT_NUMBER; // make sure we use preds } - D.prediction = ATN.INVALID_ALT_NUMBER; // make sure we use preds return predPredictions; } @@ -754,11 +756,18 @@ public class ParserATNSimulator extends ATNSimulator { // REACH=[1|1|[]|0:0, 1|2|[]|0:1] SemanticContext[] altToPred = new SemanticContext[nalts +1]; int n = altToPred.length; - for (int i = 0; i < n; i++) altToPred[i] = SemanticContext.NONE; - int nPredAlts = 0; for (ATNConfig c : configs) { - if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) { + if ( ambigAlts.contains(c.alt) ) { altToPred[c.alt] = SemanticContext.or(altToPred[c.alt], c.semanticContext); + } + } + + int nPredAlts = 0; + for (int i = 0; i < n; i++) { + if (altToPred[i] == null) { + altToPred[i] = SemanticContext.NONE; + } + else if (altToPred[i] != SemanticContext.NONE) { nPredAlts++; } } @@ -817,10 +826,12 @@ public class ParserATNSimulator extends ATNSimulator { predictedAlt = pair.alt; // default prediction break; } + + boolean evaluatedResult = pair.pred.eval(parser, outerContext); if ( debug || dfa_debug ) { - System.out.println("eval pred "+pair+"="+pair.pred.eval(parser, outerContext)); + System.out.println("eval pred "+pair+"="+evaluatedResult); } - if ( pair.pred.eval(parser, outerContext) ) { + if ( evaluatedResult ) { if ( debug || dfa_debug ) System.out.println("PREDICT "+pair.alt); predictedAlt = pair.alt; break; diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java index 505011579..5c1c64ba2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java @@ -189,14 +189,15 @@ public abstract class SemanticContext { } public static SemanticContext and(SemanticContext a, SemanticContext b) { - if ( a == NONE ) return b; - if ( b == NONE ) return a; + if ( a == null || a == NONE ) return b; + if ( b == null || b == NONE ) return a; return new AND(a, b); } public static SemanticContext or(SemanticContext a, SemanticContext b) { - if ( a == NONE ) return b; - if ( b == NONE ) return a; + if ( a == null ) return b; + if ( b == null ) return a; + if ( a == NONE || b == NONE ) return NONE; return new OR(a, b); } } diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java index 776e711f1..5467c898a 100644 --- a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java @@ -274,7 +274,7 @@ public class TestSemPredEvalParser extends BaseTest { "alt 1\n"; assertEquals(expecting, found); - expecting = ""; + expecting = null; assertEquals(expecting, stderrDuringParse); } From 6304a562ea4a5205afe6c6273c6b77e5225460cf Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 08:45:19 -0600 Subject: [PATCH 02/25] Fix unit tests --- .../v4/test/TestATNLexerInterpreter.java | 12 ++--- .../antlr/v4/test/TestATNSerialization.java | 44 +++++++++++-------- .../antlr/v4/test/TestActionTranslation.java | 4 +- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java index 3e021d8f0..947d6a6c1 100644 --- a/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java @@ -2,7 +2,7 @@ package org.antlr.v4.test; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.LexerRecognitionExeption; +import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.Utils; @@ -72,7 +72,7 @@ public class TestATNLexerInterpreter extends BaseTest { " | 'xy' .\n" + // should not pursue '.' since xy already hit stop " ;\n"); checkLexerMatches(lg, "xy", "A, EOF"); - LexerRecognitionExeption e = checkLexerMatches(lg, "xyz", "A, EOF"); + RecognitionException e = checkLexerMatches(lg, "xyz", "A, EOF"); assertEquals("NoViableAltException('z')", e.toString()); } @@ -83,7 +83,7 @@ public class TestATNLexerInterpreter extends BaseTest { " | 'xy' . 'z'\n" + // will not pursue '.' since xy already hit stop (prior alt) " ;\n"); // checkLexerMatches(lg, "xy", "A, EOF"); - LexerRecognitionExeption e = checkLexerMatches(lg, "xyqz", "A, EOF"); + RecognitionException e = checkLexerMatches(lg, "xyqz", "A, EOF"); assertEquals("NoViableAltException('q')", e.toString()); } @@ -244,7 +244,7 @@ public class TestATNLexerInterpreter extends BaseTest { checkLexerMatches(lg, "a", expecting); } - protected LexerRecognitionExeption checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { + protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { ATN atn = createATN(lg); CharStream input = new ANTLRInputStream(inputString); ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE"); @@ -252,11 +252,11 @@ public class TestATNLexerInterpreter extends BaseTest { System.out.println(dot.getDOT(startState, true)); List tokenTypes = null; - LexerRecognitionExeption retException = null; + RecognitionException retException = null; try { tokenTypes = getTokenTypes(lg, atn, input, false); } - catch (LexerRecognitionExeption lre) { retException = lre; } + catch (RecognitionException lre) { retException = lre; } if ( retException!=null ) return retException; String result = Utils.join(tokenTypes.iterator(), ", "); diff --git a/tool/test/org/antlr/v4/test/TestATNSerialization.java b/tool/test/org/antlr/v4/test/TestATNSerialization.java index 0eac6f8d4..c3d2558e1 100644 --- a/tool/test/org/antlr/v4/test/TestATNSerialization.java +++ b/tool/test/org/antlr/v4/test/TestATNSerialization.java @@ -406,7 +406,7 @@ public class TestATNSerialization extends BaseTest { "5:PLUS_BLOCK_START 0\n" + "6:BLOCK_END 0\n" + "7:PLUS_LOOP_BACK 0\n" + - "8:LOOP_END 0 6\n" + + "8:LOOP_END 0 7\n" + "rule 0:1 3,-1\n" + "mode 0:0\n" + "0->1 EPSILON 0,0,0\n" + @@ -441,10 +441,12 @@ public class TestATNSerialization extends BaseTest { "6:RULE_STOP 2\n" + "7:BASIC 0\n" + "9:BASIC 0\n" + - "10:BASIC 1\n" + + "10:BASIC 0\n" + "11:BASIC 1\n" + - "12:BASIC 2\n" + - "14:BASIC 2\n" + + "12:BASIC 1\n" + + "13:BASIC 2\n" + + "15:BASIC 2\n" + + "16:BASIC 2\n" + "rule 0:1 3,0\n" + "rule 1:3 4,-1\n" + "rule 2:5 5,1\n" + @@ -453,14 +455,16 @@ public class TestATNSerialization extends BaseTest { "0->3 EPSILON 0,0,0\n" + "0->5 EPSILON 0,0,0\n" + "1->7 EPSILON 0,0,0\n" + - "3->10 EPSILON 0,0,0\n" + - "5->12 EPSILON 0,0,0\n" + + "3->11 EPSILON 0,0,0\n" + + "5->13 EPSILON 0,0,0\n" + "7->9 ATOM 97,0,0\n" + - "9->2 EPSILON 0,0,0\n" + - "10->11 ATOM 98,0,0\n" + - "11->4 EPSILON 0,0,0\n" + - "12->14 ATOM 99,0,0\n" + - "14->6 EPSILON 0,0,0\n" + + "9->10 ACTION 0,0,0\n" + + "10->2 EPSILON 0,0,0\n" + + "11->12 ATOM 98,0,0\n" + + "12->4 EPSILON 0,0,0\n" + + "13->15 ATOM 99,0,0\n" + + "15->16 ACTION 2,1,0\n" + + "16->6 EPSILON 0,0,0\n" + "0:0 1\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); @@ -560,13 +564,15 @@ public class TestATNSerialization extends BaseTest { "10:PLUS_BLOCK_START 0\n" + "11:BLOCK_END 0\n" + "12:PLUS_LOOP_BACK 0\n" + - "13:LOOP_END 0\n" + + "13:LOOP_END 0 12\n" + "14:BASIC 1\n" + "15:BASIC 1\n" + "16:BASIC 1\n" + "17:BASIC 1\n" + - "18:BASIC 2\n" + - "20:BASIC 2\n" + + "18:BASIC 1\n" + + "19:BASIC 2\n" + + "21:BASIC 2\n" + + "22:BASIC 2\n" + "rule 0:2 3,-1\n" + "rule 1:4 4,0\n" + "rule 2:6 5,1\n" + @@ -577,7 +583,7 @@ public class TestATNSerialization extends BaseTest { "1->6 EPSILON 0,0,0\n" + "2->10 EPSILON 0,0,0\n" + "4->14 EPSILON 0,0,0\n" + - "6->18 EPSILON 0,0,0\n" + + "6->19 EPSILON 0,0,0\n" + "8->11 RANGE 97,122,0\n" + "10->8 EPSILON 0,0,0\n" + "11->12 EPSILON 0,0,0\n" + @@ -587,9 +593,11 @@ public class TestATNSerialization extends BaseTest { "14->15 ATOM 42,0,0\n" + "15->16 ATOM 47,0,0\n" + "16->17 EPSILON 0,0,0\n" + - "17->5 EPSILON 0,0,0\n" + - "18->20 WILDCARD 0,0,0\n" + - "20->7 EPSILON 0,0,0\n" + + "17->18 ACTION 1,0,0\n" + + "18->5 EPSILON 0,0,0\n" + + "19->21 WILDCARD 0,0,0\n" + + "21->22 ACTION 2,1,0\n" + + "22->7 EPSILON 0,0,0\n" + "0:0 1\n" + "1:1 1\n" + "2:12 1\n"; diff --git a/tool/test/org/antlr/v4/test/TestActionTranslation.java b/tool/test/org/antlr/v4/test/TestActionTranslation.java index 9fecaad6e..778b01a1a 100644 --- a/tool/test/org/antlr/v4/test/TestActionTranslation.java +++ b/tool/test/org/antlr/v4/test/TestActionTranslation.java @@ -116,8 +116,8 @@ public class TestActionTranslation extends BaseTest { } @Test public void testRuleRefs() throws Exception { - String action = "$lab.start; $c.tree;"; - String expected = "(_localctx.lab!=null?((Token)_localctx.lab.start):null); (_localctx._rc!=null?((CommonAST)_localctx._rc.tree):null);"; + String action = "$lab.start;"; + String expected = "(_localctx.lab!=null?(_localctx.lab.start):null);"; testActions(attributeTemplate, "inline", action, expected); } From b686efb293e030ed88f0ddbb5d5dc0d3824e3045 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 12:42:48 -0600 Subject: [PATCH 03/25] Code cleanup in unit tests --- .../antlr/v4/test/TestATNParserPrediction.java | 5 +++-- .../antlr/v4/test/TestCommonTokenStream.java | 17 +++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 4e9ba3ead..417c239d4 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -33,6 +33,7 @@ import org.antlr.v4.Tool; import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.runtime.NoViableAltException; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; @@ -523,7 +524,7 @@ public class TestATNParserPrediction extends BaseTest { } public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName, - String inputString, ParserRuleContext ctx) + String inputString, ParserRuleContext ctx) { Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); @@ -541,7 +542,7 @@ public class TestATNParserPrediction extends BaseTest { // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b")))); // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); - ParserATNSimulator interp = new ParserATNSimulator(atn); + ParserATNSimulator interp = new ParserATNSimulator(atn); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); TokenStream input = new IntTokenStream(types); diff --git a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java index 636ca98dc..dd42e2bc7 100644 --- a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java @@ -178,24 +178,17 @@ public class TestCommonTokenStream extends BaseTest { new TokenSource() { int i = 0; WritableToken[] tokens = { - new CommonToken(1," "), + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1,"x"), - new CommonToken(1," "), + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1,"="), new CommonToken(1,"34"), - new CommonToken(1," "), - new CommonToken(1," "), + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1,";"), - new CommonToken(1,"\n"), + new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}}, new CommonToken(Token.EOF,"") }; - { - tokens[0].setChannel(Lexer.HIDDEN); - tokens[2].setChannel(Lexer.HIDDEN); - tokens[5].setChannel(Lexer.HIDDEN); - tokens[6].setChannel(Lexer.HIDDEN); - tokens[8].setChannel(Lexer.HIDDEN); - } public Token nextToken() { return tokens[i++]; } From bf5df307081c18805ec49f7007aff04de8a19334 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 12:45:46 -0600 Subject: [PATCH 04/25] Remove unnecessary initializations, use isEmpty() instead of size()==0 --- .../org/antlr/v4/runtime/atn/ParserATNSimulator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 51b31e458..2bd500ce0 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -367,7 +367,7 @@ public class ParserATNSimulator extends ATNSimulator { // if no edge, pop over to ATN interpreter, update DFA and return if ( s.edges == null || t >= s.edges.length || t < -1 || s.edges[t+1] == null ) { if ( dfa_debug && t>=0 ) System.out.println("no edge for "+parser.getTokenNames()[t]); - int alt = -1; + int alt; if ( dfa_debug ) { System.out.println("ATN exec upon "+ parser.getInputString(startIndex) + @@ -480,8 +480,8 @@ public class ParserATNSimulator extends ATNSimulator { ATN_failover++; ATNConfigSet previous = s0.configset; - DFAState D = null; - ATNConfigSet fullCtxSet = null; + DFAState D; + ATNConfigSet fullCtxSet; if ( debug ) System.out.println("s0 = "+s0); @@ -631,7 +631,7 @@ public class ParserATNSimulator extends ATNSimulator { if ( reach.hasSemanticContext ) { SemanticContext[] altToPred = getPredsForAmbigAlts(reach.conflictingAlts, reach, nalts); // altToPred[uniqueAlt] is now our validating predicate (if any) - List predPredictions = null; + List predPredictions; if ( altToPred!=null ) { // we have a validating predicate; test it predPredictions = getPredicatePredictions(reach.conflictingAlts, altToPred); @@ -793,7 +793,7 @@ public class ParserATNSimulator extends ATNSimulator { pairs.add(new DFAState.PredPrediction(pred, i)); } } - if ( pairs.size()==0 ) pairs = null; + if ( pairs.isEmpty() ) pairs = null; else if ( firstUnpredicated!=ATN.INVALID_ALT_NUMBER ) { // add default prediction if we found null predicate pairs.add(new DFAState.PredPrediction(null, firstUnpredicated)); From d589011ad82968ff6c6b8e70e4eb090eeddfd257 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 13:00:27 -0600 Subject: [PATCH 05/25] Small fixes to the Java grammars used in the performance test --- tool/test/org/antlr/v4/test/Java-LR.g | 47 ++++++++++++++++----------- tool/test/org/antlr/v4/test/Java.g | 4 ++- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/tool/test/org/antlr/v4/test/Java-LR.g b/tool/test/org/antlr/v4/test/Java-LR.g index 97e5facbf..75fcb36ac 100644 --- a/tool/test/org/antlr/v4/test/Java-LR.g +++ b/tool/test/org/antlr/v4/test/Java-LR.g @@ -169,7 +169,6 @@ * Character.isJavaIdentifierPart(int) returns true." */ grammar Java; -options {backtrack=true; memoize=true;} @lexer::members { protected boolean enumIsKeyword = true; @@ -184,7 +183,9 @@ compilationUnit ( packageDeclaration importDeclaration* typeDeclaration* | classOrInterfaceDeclaration typeDeclaration* ) + EOF | packageDeclaration? importDeclaration* typeDeclaration* + EOF ; packageDeclaration @@ -499,7 +500,7 @@ constructorBody explicitConstructorInvocation : nonWildcardTypeArguments? ('this' | 'super') arguments ';' - | expression '.' nonWildcardTypeArguments? 'super' arguments ';' + | primary '.' nonWildcardTypeArguments? 'super' arguments ';' ; @@ -711,17 +712,12 @@ constantExpression ; expression - : parExpression - | 'this' - | 'super' - | literal - | Identifier + : primary | expression '.' Identifier - | expression '.' 'class' // should be type.class but causes backtracking | expression '.' 'this' | expression '.' 'super' '(' expressionList? ')' - | expression '.' 'super' '.' Identifier arguments? | expression '.' 'new' Identifier '(' expressionList? ')' + | expression '.' 'super' '.' Identifier arguments? | expression '.' explicitGenericInvocation | 'new' creator | expression '[' expression ']' @@ -754,8 +750,20 @@ expression | '>' '>' '=' | '>' '>' '>' '=' | '<' '<' '=' - | '%=') expression - ; + | '%=' + ) + expression + ; + +primary + : '(' expression ')' + | 'this' + | 'super' + | literal + | Identifier + | type '.' 'class' + | 'void' '.' 'class' + ; creator : nonWildcardTypeArguments createdName classCreatorRest @@ -827,15 +835,16 @@ FloatingPointLiteral | ('0'..'9')+ Exponent FloatTypeSuffix? | ('0'..'9')+ FloatTypeSuffix | '0' ('x'|'X') - ( HexDigit+ '.' HexDigit* Exponent? FloatTypeSuffix? - | '.' HexDigit+ Exponent? FloatTypeSuffix? - | HexDigit+ Exponent FloatTypeSuffix? - | HexDigit+ FloatTypeSuffix + ( HexDigit+ ('.' HexDigit*)? HexExponent FloatTypeSuffix? + | '.' HexDigit+ HexExponent FloatTypeSuffix? ) ; fragment -Exponent : ('e'|'E'|'p'|'P') ('+'|'-')? ('0'..'9')+ ; +Exponent : ('e'|'E') ('+'|'-')? ('0'..'9')+ ; + +fragment +HexExponent : ('p'|'P') ('+'|'-')? ('0'..'9')+ ; fragment FloatTypeSuffix : ('f'|'F'|'d'|'D') ; @@ -917,13 +926,13 @@ JavaIDDigit '\u1040'..'\u1049' ; -WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} +WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;}//-> channel(HIDDEN) ; COMMENT - : '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} + : '/*' .* '*/' {$channel=HIDDEN;}//-> channel(HIDDEN) ; LINE_COMMENT - : '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;} + : '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;}//-> channel(HIDDEN) ; diff --git a/tool/test/org/antlr/v4/test/Java.g b/tool/test/org/antlr/v4/test/Java.g index e5b174fda..dcfb75673 100644 --- a/tool/test/org/antlr/v4/test/Java.g +++ b/tool/test/org/antlr/v4/test/Java.g @@ -181,7 +181,9 @@ compilationUnit ( packageDeclaration importDeclaration* typeDeclaration* | classOrInterfaceDeclaration typeDeclaration* ) + EOF | packageDeclaration? importDeclaration* typeDeclaration* + EOF ; packageDeclaration @@ -1021,7 +1023,7 @@ WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} ; COMMENT - : '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} + : '/*' .* '*/' {$channel=HIDDEN;} ; LINE_COMMENT From 7fb73a38493492a27f4ba8c5924c01e48a6c6d1e Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 13:29:14 -0600 Subject: [PATCH 06/25] Construct DFA instances with the decision number --- .../org/antlr/v4/runtime/atn/ParserATNSimulator.java | 3 +-- runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java | 12 ++++++++++-- .../org/antlr/v4/test/TestATNParserPrediction.java | 3 +-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 2bd500ce0..36b881f1b 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -268,8 +268,7 @@ public class ParserATNSimulator extends ATNSimulator { DFA dfa = decisionToDFA[decision]; if ( dfa==null || dfa.s0==null ) { DecisionState startState = atn.decisionToState.get(decision); - decisionToDFA[decision] = dfa = new DFA(startState); - dfa.decision = decision; + decisionToDFA[decision] = dfa = new DFA(startState, decision); return predictATN(dfa, input, outerContext); } else { diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java index b3b41bd70..edaf8b93e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java @@ -43,7 +43,8 @@ public class DFA { public final Map states = new LinkedHashMap(); @Nullable public DFAState s0; - public int decision; + + public final int decision; /** From which ATN state did we create this DFA? */ @NotNull @@ -54,7 +55,14 @@ public class DFA { */ // public OrderedHashSet conflictSet; - public DFA(@NotNull DecisionState atnStartState) { this.atnStartState = atnStartState; } + public DFA(@NotNull DecisionState atnStartState) { + this(atnStartState, 0); + } + + public DFA(@NotNull DecisionState atnStartState, int decision) { + this.atnStartState = atnStartState; + this.decision = decision; + } /** Find the path in DFA from s0 to s, returning list of states encountered (inclusively) */ // public List getPathToState(DFAState finalState, TokenStream input, int start, int stop) { diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 417c239d4..4393a5a1b 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -505,8 +505,7 @@ public class TestATNParserPrediction extends BaseTest { TokenStream input = new IntTokenStream(types); ParserInterpreter interp = new ParserInterpreter(g, input); DecisionState startState = atn.decisionToState.get(decision); - DFA dfa = new DFA(startState); - dfa.decision = decision; + DFA dfa = new DFA(startState, decision); int alt = interp.predictATN(dfa, input, ParserRuleContext.EMPTY, false); System.out.println(dot.getDOT(dfa, false)); From 702b91fab746b2001ef5c8d9762ae817e8c80812 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 13:30:44 -0600 Subject: [PATCH 07/25] Ignore TestPerformance by default since it fails without special configuration --- tool/test/org/antlr/v4/test/TestPerformance.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java index 98622ed83..f2d26daf8 100644 --- a/tool/test/org/antlr/v4/test/TestPerformance.java +++ b/tool/test/org/antlr/v4/test/TestPerformance.java @@ -6,6 +6,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.io.*; @@ -75,7 +76,7 @@ public class TestPerformance extends BaseTest { private int tokenCount; @Test -// @Ignore + @Ignore public void compileJdk() throws IOException { compileParser(USE_LR_GRAMMAR); JavaParserFactory factory = getParserFactory(); From ddf946b067247a3040d78fe3cf535b4f6418e380 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 15:33:03 -0600 Subject: [PATCH 08/25] Token.EOF instead of -1, add to generic type usage, make getUniqueAlt an instance method (need to override it for some apps), @NotNull annotations --- .../src/org/antlr/v4/runtime/atn/ParserATNSimulator.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 36b881f1b..8b1143ec7 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -1200,7 +1200,7 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull public String getTokenName(int t) { - if ( t==-1 ) return "EOF"; + if ( t==Token.EOF ) return "EOF"; if ( parser!=null && parser.getTokenNames()!=null ) { String[] tokensNames = parser.getTokenNames(); if ( t>=tokensNames.length ) { @@ -1240,7 +1240,7 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull public NoViableAltException noViableAlt(@NotNull SymbolStream input, - @NotNull ParserRuleContext outerContext, + @NotNull ParserRuleContext outerContext, @NotNull ATNConfigSet configs, int startIndex) { @@ -1250,7 +1250,7 @@ public class ParserATNSimulator extends ATNSimulator { configs, outerContext); } - public static int getUniqueAlt(@NotNull Collection configs) { + public int getUniqueAlt(@NotNull Collection configs) { int alt = ATN.INVALID_ALT_NUMBER; for (ATNConfig c : configs) { if ( alt == ATN.INVALID_ALT_NUMBER ) { @@ -1275,6 +1275,7 @@ public class ParserATNSimulator extends ATNSimulator { return false; } + @NotNull protected DFAState addDFAEdge(@NotNull DFA dfa, @NotNull ATNConfigSet p, int t, @@ -1297,7 +1298,7 @@ public class ParserATNSimulator extends ATNSimulator { } /** See comment on LexerInterpreter.addDFAState. */ - @Nullable + @NotNull protected DFAState addDFAState(@NotNull DFA dfa, @NotNull ATNConfigSet configs) { DFAState proposed = new DFAState(configs); DFAState existing = dfa.states.get(proposed); From 467797785f30dd44041162334344cbd1cf60394e Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 15:37:56 -0600 Subject: [PATCH 09/25] Fix generics in ParserATNSimulator --- .../v4/runtime/atn/ParserATNSimulator.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 8b1143ec7..e63982fea 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -225,7 +225,7 @@ import java.util.*; * when closure operations fall off the end of the rule that * holds the decision were evaluating */ -public class ParserATNSimulator extends ATNSimulator { +public class ParserATNSimulator extends ATNSimulator { public static boolean debug = false; public static boolean dfa_debug = false; public static boolean retry_debug = false; @@ -261,7 +261,7 @@ public class ParserATNSimulator extends ATNSimulator { public void reset() { } - public int adaptivePredict(@NotNull SymbolStream input, int decision, + public int adaptivePredict(@NotNull SymbolStream input, int decision, @Nullable ParserRuleContext outerContext) { predict_calls++; @@ -287,7 +287,7 @@ public class ParserATNSimulator extends ATNSimulator { } } - public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream input, + public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream input, @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; @@ -319,7 +319,7 @@ public class ParserATNSimulator extends ATNSimulator { } public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0, - @NotNull SymbolStream input, int startIndex, + @NotNull SymbolStream input, int startIndex, @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; @@ -472,7 +472,7 @@ public class ParserATNSimulator extends ATNSimulator { */ public int execATN(@NotNull DFA dfa, @NotNull DFAState s0, - @NotNull SymbolStream input, int startIndex, + @NotNull SymbolStream input, int startIndex, ParserRuleContext outerContext) { if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input)); @@ -593,7 +593,7 @@ public class ParserATNSimulator extends ATNSimulator { public ATNConfigSet execATNWithFullContext(DFA dfa, DFAState D, // how far we got before failing over @NotNull ATNConfigSet s0, - @NotNull SymbolStream input, int startIndex, + @NotNull SymbolStream input, int startIndex, ParserRuleContext outerContext, int nalts, boolean greedy) @@ -1214,7 +1214,7 @@ public class ParserATNSimulator extends ATNSimulator { return String.valueOf(t); } - public String getLookaheadName(SymbolStream input) { + public String getLookaheadName(SymbolStream input) { return getTokenName(input.LA(1)); } @@ -1239,14 +1239,14 @@ public class ParserATNSimulator extends ATNSimulator { } @NotNull - public NoViableAltException noViableAlt(@NotNull SymbolStream input, + public NoViableAltException noViableAlt(@NotNull SymbolStream input, @NotNull ParserRuleContext outerContext, @NotNull ATNConfigSet configs, int startIndex) { return new NoViableAltException(parser, input, - (Token)input.get(startIndex), - (Token)input.LT(1), + input.get(startIndex), + input.LT(1), configs, outerContext); } From 99ce3cba5cfe2ecaa81244e59400fa28f2104be3 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 19:47:56 -0600 Subject: [PATCH 10/25] Fix ParserATNSimulator.closure not tracking proper context for context-sensitive predicates during full-context parsing --- .../v4/runtime/atn/ParserATNSimulator.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index e63982fea..d4328e4dc 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -844,6 +844,17 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull Set closureBusy, boolean collectPredicates, boolean greedy, boolean loopsSimulateTailRecursion) + { + final int initialDepth = 0; + closure(config, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, initialDepth); + } + + protected void closure(@NotNull ATNConfig config, + @NotNull ATNConfigSet configs, + @NotNull Set closureBusy, + boolean collectPredicates, + boolean greedy, boolean loopsSimulateTailRecursion, + int depth) { if ( debug ) System.out.println("closure("+config.toString(parser,true)+")"); @@ -868,7 +879,7 @@ public class ParserATNSimulator extends ATNSimulator { // gotten that context AFTER having falling off a rule. // Make sure we track that we are now out of context. c.reachesIntoOuterContext = config.reachesIntoOuterContext; - closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion); + closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, depth - 1); return; } else { @@ -912,8 +923,9 @@ public class ParserATNSimulator extends ATNSimulator { Transition t = p.transition(i); boolean continueCollecting = !(t instanceof ActionTransition) && collectPredicates; - ATNConfig c = getEpsilonTarget(config, t, continueCollecting); + ATNConfig c = getEpsilonTarget(config, t, continueCollecting, depth == 0); if ( c!=null ) { + int newDepth = depth; if ( config.state instanceof RuleStopState ) { // target fell off end of rule; mark resulting c as having dipped into outer context // We can't get here if incoming config was rule stop and we had context @@ -922,9 +934,16 @@ public class ParserATNSimulator extends ATNSimulator { // preds if this is > 0. c.reachesIntoOuterContext++; configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method + newDepth--; if ( debug ) System.out.println("dips into outer ctx: "+c); } - closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion); + else if (t instanceof RuleTransition) { + if (newDepth >= 0) { + newDepth++; + } + } + + closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion, newDepth); } } } @@ -936,12 +955,12 @@ public class ParserATNSimulator extends ATNSimulator { } @Nullable - public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates) { + public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates, boolean inContext) { if ( t instanceof RuleTransition ) { return ruleTransition(config, t); } else if ( t instanceof PredicateTransition ) { - return predTransition(config, (PredicateTransition)t, collectPredicates); + return predTransition(config, (PredicateTransition)t, collectPredicates, inContext); } else if ( t instanceof ActionTransition ) { return actionTransition(config, (ActionTransition)t); @@ -961,7 +980,8 @@ public class ParserATNSimulator extends ATNSimulator { @Nullable public ATNConfig predTransition(@NotNull ATNConfig config, @NotNull PredicateTransition pt, - boolean collectPredicates) + boolean collectPredicates, + boolean inContext) { if ( debug ) { System.out.println("PRED (collectPredicates="+collectPredicates+") "+ @@ -972,12 +992,6 @@ public class ParserATNSimulator extends ATNSimulator { parser.getRuleInvocationStack()); } } - // We know the correct context in exactly one spot: in the original - // rule that invokes the ATN simulation. We know we are in this rule - // when the context stack is empty and we've not dipped into - // the outer context. - boolean inContext = - config.context==ParserRuleContext.EMPTY && config.reachesIntoOuterContext==0; ATNConfig c; if ( collectPredicates && From 30c58bf8ebbd1b98f42f84888346249c4332d84b Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 10 Feb 2012 00:51:23 -0600 Subject: [PATCH 11/25] Specify type parameter on raw types --- .../org/antlr/v4/runtime/atn/ParserATNSimulator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index d4328e4dc..eb02799ae 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -262,7 +262,7 @@ public class ParserATNSimulator extends ATNSimulator { } public int adaptivePredict(@NotNull SymbolStream input, int decision, - @Nullable ParserRuleContext outerContext) + @Nullable ParserRuleContext outerContext) { predict_calls++; DFA dfa = decisionToDFA[decision]; @@ -288,7 +288,7 @@ public class ParserATNSimulator extends ATNSimulator { } public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream input, - @Nullable ParserRuleContext outerContext) + @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( debug ) System.out.println("ATN decision "+dfa.decision+ @@ -320,7 +320,7 @@ public class ParserATNSimulator extends ATNSimulator { public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0, @NotNull SymbolStream input, int startIndex, - @Nullable ParserRuleContext outerContext) + @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ @@ -473,7 +473,7 @@ public class ParserATNSimulator extends ATNSimulator { */ public int execATN(@NotNull DFA dfa, @NotNull DFAState s0, @NotNull SymbolStream input, int startIndex, - ParserRuleContext outerContext) + ParserRuleContext outerContext) { if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input)); ATN_failover++; @@ -594,7 +594,7 @@ public class ParserATNSimulator extends ATNSimulator { DFAState D, // how far we got before failing over @NotNull ATNConfigSet s0, @NotNull SymbolStream input, int startIndex, - ParserRuleContext outerContext, + ParserRuleContext outerContext, int nalts, boolean greedy) { @@ -806,7 +806,7 @@ public class ParserATNSimulator extends ATNSimulator { * prediction for disambiguating predicates. */ public int evalSemanticContext(List predPredictions, - ParserRuleContext outerContext) + ParserRuleContext outerContext) { int predictedAlt = ATN.INVALID_ALT_NUMBER; // List predPredictions = D.predicates; From cd3adb140d7308082171e0640a15e312caee84be Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 10 Feb 2012 00:52:24 -0600 Subject: [PATCH 12/25] Fix ParserATNSimulator.execDFA adding incorrect error edges in the DFA --- .../v4/runtime/atn/ParserATNSimulator.java | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index eb02799ae..e7d9cec0c 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -372,31 +372,26 @@ public class ParserATNSimulator extends ATNSimulator { parser.getInputString(startIndex) + " at DFA state "+s.stateNumber); } - try { - alt = execATN(dfa, s, input, startIndex, outerContext); - // this adds edge even if next state is accept for - // same alt; e.g., s0-A->:s1=>2-B->:s2=>2 - // TODO: This next stuff kills edge, but extra states remain. :( - if ( s.isAcceptState && alt!=-1 ) { - DFAState d = s.edges[input.LA(1)+1]; - if ( d.isAcceptState && d.prediction==s.prediction ) { - // we can carve it out. - s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error - } + + alt = execATN(dfa, s, input, startIndex, outerContext); + // this adds edge even if next state is accept for + // same alt; e.g., s0-A->:s1=>2-B->:s2=>2 + // TODO: This next stuff kills edge, but extra states remain. :( + if ( s.isAcceptState && alt!=-1 ) { + DFAState d = s.edges[input.LA(1)+1]; + if ( d.isAcceptState && d.prediction==s.prediction ) { + // we can carve it out. + s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error } - if ( dfa_debug ) { - System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames())); - //dump(dfa); - } - // action already executed - if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ - " predicts "+alt); - return alt; // we've updated DFA, exec'd action, and have our deepest answer } - catch (NoViableAltException nvae) { - addDFAEdge(s, t, ERROR); - throw nvae; + if ( dfa_debug ) { + System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames())); + //dump(dfa); } + // action already executed + if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ + " predicts "+alt); + return alt; // we've updated DFA, exec'd action, and have our deepest answer } DFAState target = s.edges[t+1]; if ( target == ERROR ) { From 7287f5a2d3719f992f34bfea5071c8d7d9c16ab5 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 20 Feb 2012 10:51:34 -0800 Subject: [PATCH 13/25] prefix left-recursive alts weren't recognized with actions on end. --- .../antlr/v4/parse/LeftRecursiveRuleWalker.g | 2 +- .../org/antlr/v4/test/TestLeftRecursion.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g b/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g index db7ace07e..b02745302 100644 --- a/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g +++ b/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g @@ -148,7 +148,7 @@ ternary prefix : ^( ALT {setTokenPrec((GrammarAST)input.LT(1), currentOuterAltNumber);} ({!((CommonTree)input.LT(1)).getText().equals(ruleName)}? element)+ - recurse + recurse ACTION? ) ; diff --git a/tool/test/org/antlr/v4/test/TestLeftRecursion.java b/tool/test/org/antlr/v4/test/TestLeftRecursion.java index ffdb45941..62d970172 100644 --- a/tool/test/org/antlr/v4/test/TestLeftRecursion.java +++ b/tool/test/org/antlr/v4/test/TestLeftRecursion.java @@ -255,6 +255,27 @@ public class TestLeftRecursion extends BaseTest { runTests(grammar, tests, "s"); } + @Test public void testPrefixOpWithActionAndLabel() throws Exception { + String grammar = + "grammar T;\n" + + "s : e {System.out.println($e.result);} ;\n" + + "\n" + + "e returns [String result]\n" + + " : ID '=' e1=e { $result = \"(\" + $ID.getText() + \"=\" + $e1.result + \")\"; }\n" + + " | ID { $result = $ID.getText(); }\n" + + " | e1=e '+' e2=e { $result = \"(\" + $e1.result + \"+\" + $e2.result + \")\"; }\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+ ;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + String[] tests = { + "a", "a", + "a+b", "(a+b)", + "a=b+c", "((a=b)+c)", + }; + runTests(grammar, tests, "s"); + } + public void runTests(String grammar, String[] tests, String startRule) { rawGenerateAndBuildRecognizer("T.g", grammar, "TParser", "TLexer"); writeRecognizerAndCompile("TParser", From 97660e21ae82ac6eda744c879c16aa725cb4c9fe Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 20 Feb 2012 12:26:32 -0800 Subject: [PATCH 14/25] add alt labels to all alts --- tool/test/org/antlr/v4/test/TestLeftRecursion.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tool/test/org/antlr/v4/test/TestLeftRecursion.java b/tool/test/org/antlr/v4/test/TestLeftRecursion.java index 62d970172..f5b8400f9 100644 --- a/tool/test/org/antlr/v4/test/TestLeftRecursion.java +++ b/tool/test/org/antlr/v4/test/TestLeftRecursion.java @@ -236,10 +236,10 @@ public class TestLeftRecursion extends BaseTest { "e returns [int v]\n" + " : a=e op='*' b=e {$v = $a.v * $b.v;} -> mult\n" + " | a=e '+' b=e {$v = $a.v + $b.v;} -> add\n" + - " | INT {$v = $INT.int;}\n" + - " | '(' x=e ')' {$v = $x.v;}\n" + + " | INT {$v = $INT.int;} -> anInt\n" + + " | '(' x=e ')' {$v = $x.v;} -> parens\n" + " | x=e '++' {$v = $x.v+1;} -> inc\n" + - " | e '--'\n" + + " | e '--' -> dec\n" + " | ID {$v = 3;} -> anID\n" + " ; \n" + "\n" + From 16fa8ba4f4eaefe681e1e0c55a8b9a1e97ebbd93 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 15:25:58 -0600 Subject: [PATCH 15/25] Add comments describing the context depth latch in ParserATNSimulator.closure --- .../Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index c5939706d..dcafb6c44 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -891,6 +891,7 @@ public class ParserATNSimulator extends ATNSimulator { // gotten that context AFTER having falling off a rule. // Make sure we track that we are now out of context. c.reachesIntoOuterContext = config.reachesIntoOuterContext; + assert depth > Integer.MIN_VALUE; closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, depth - 1); return; } @@ -946,10 +947,12 @@ public class ParserATNSimulator extends ATNSimulator { // preds if this is > 0. c.reachesIntoOuterContext++; configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method + assert newDepth > Integer.MIN_VALUE; newDepth--; if ( debug ) System.out.println("dips into outer ctx: "+c); } else if (t instanceof RuleTransition) { + // latch when newDepth goes negative - once we step out of the entry context we can't return if (newDepth >= 0) { newDepth++; } From 2982f4b5a607684a6f58fa58a4e4e6ad700517a3 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 15:43:02 -0600 Subject: [PATCH 16/25] Add comments describing the semantics of an ATNConfig set with partially predicated alts in ParserATNSimulator --- .../v4/runtime/atn/ParserATNSimulator.java | 20 +++++++++++++++---- .../antlr/v4/runtime/atn/SemanticContext.java | 4 ++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index dcafb6c44..5d3d05489 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -581,11 +581,11 @@ public class ParserATNSimulator extends ATNSimulator { return predictedAlt; } - if (D.prediction == ATN.INVALID_ALT_NUMBER) { - throw noViableAlt(input, outerContext, D.configset, startIndex); - } + // Consistency check - the DFAState should not have a "fallback" + // prediction specified for the case where no predicates succeed. + assert D.prediction == ATN.INVALID_ALT_NUMBER; - predictedAlt = D.prediction; + throw noViableAlt(input, outerContext, D.configset, startIndex); } } @@ -754,6 +754,18 @@ public class ParserATNSimulator extends ATNSimulator { int nalts) { // REACH=[1|1|[]|0:0, 1|2|[]|0:1] + + /* altToPred starts as an array of all null contexts. The entry at index i + * corresponds to alternative i. altToPred[i] may have one of three values: + * 1. null: no ATNConfig c is found such that c.alt==i + * 2. SemanticContext.NONE: At least one ATNConfig c exists such that + * c.alt==i and c.semanticContext==SemanticContext.NONE. In other words, + * alt i has at least one unpredicated config. + * 3. Non-NONE Semantic Context: There exists at least one, and for all + * ATNConfig c such that c.alt==i, c.semanticContext!=SemanticContext.NONE. + * + * From this, it is clear that NONE||anything==NONE. + */ SemanticContext[] altToPred = new SemanticContext[nalts +1]; int n = altToPred.length; for (ATNConfig c : configs) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java index 5c1c64ba2..85a9e997b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java @@ -194,6 +194,10 @@ public abstract class SemanticContext { return new AND(a, b); } + /** + * + * @see ParserATNSimulator#getPredsForAmbigAlts + */ public static SemanticContext or(SemanticContext a, SemanticContext b) { if ( a == null ) return b; if ( b == null ) return a; From 8773c34375566ffdf6a0802cc827435cf3603e3c Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 15:58:21 -0600 Subject: [PATCH 17/25] Fix unit tests --- .../org/antlr/v4/test/TestActionTranslation.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tool/test/org/antlr/v4/test/TestActionTranslation.java b/tool/test/org/antlr/v4/test/TestActionTranslation.java index 20932f9a1..1bba58098 100644 --- a/tool/test/org/antlr/v4/test/TestActionTranslation.java +++ b/tool/test/org/antlr/v4/test/TestActionTranslation.java @@ -69,8 +69,8 @@ public class TestActionTranslation extends BaseTest { String action = "x, $ID.text+\"3242\", (*$ID).foo(21,33), 3.2+1, '\\n', "+ "\"a,oo\\nick\", {bl, \"fdkj\"eck}"; String expected = - "x, (((aContext)_localctx).ID!=null?((aContext)_localctx).ID.getText():null)+\"3242\", " + - "(*((aContext)_localctx).ID).foo(21,33), 3.2+1, '\\n', \"a,oo\\nick\", {bl, \"fdkj\"eck}"; + "x, (((AContext)_localctx).ID!=null?((AContext)_localctx).ID.getText():null)+\"3242\", " + + "(*((AContext)_localctx).ID).foo(21,33), 3.2+1, '\\n', \"a,oo\\nick\", {bl, \"fdkj\"eck}"; testActions(attributeTemplate, "inline", action, expected); } @@ -100,25 +100,25 @@ public class TestActionTranslation extends BaseTest { @Test public void testReturnValues() throws Exception { String action = "$lab.e; $b.e;"; - String expected = "((aContext)_localctx).lab.e; ((aContext)_localctx).b.e;"; + String expected = "((AContext)_localctx).lab.e; ((AContext)_localctx).b.e;"; testActions(attributeTemplate, "inline", action, expected); } @Test public void testReturnWithMultipleRuleRefs() throws Exception { String action = "$c.x; $c.y;"; - String expected = "((aContext)_localctx).c.x; ((aContext)_localctx).c.y;"; + String expected = "((AContext)_localctx).c.x; ((AContext)_localctx).c.y;"; testActions(attributeTemplate, "inline", action, expected); } @Test public void testTokenRefs() throws Exception { String action = "$id; $ID; $id.text; $id.getText(); $id.line;"; - String expected = "((aContext)_localctx).id; ((aContext)_localctx).ID; (((aContext)_localctx).id!=null?((aContext)_localctx).id.getText():null); ((aContext)_localctx).id.getText(); (((aContext)_localctx).id!=null?((aContext)_localctx).id.getLine():0);"; + String expected = "((AContext)_localctx).id; ((AContext)_localctx).ID; (((AContext)_localctx).id!=null?((AContext)_localctx).id.getText():null); ((AContext)_localctx).id.getText(); (((AContext)_localctx).id!=null?((AContext)_localctx).id.getLine():0);"; testActions(attributeTemplate, "inline", action, expected); } @Test public void testRuleRefs() throws Exception { String action = "$lab.start; $c.text;"; - String expected = "(((aContext)_localctx).lab!=null?(((aContext)_localctx).lab.start):null); (((aContext)_localctx).c!=null?_input.toString(((aContext)_localctx).c.start,((aContext)_localctx).c.stop):null);"; + String expected = "(((AContext)_localctx).lab!=null?(((AContext)_localctx).lab.start):null); (((AContext)_localctx).c!=null?_input.toString(((AContext)_localctx).c.start,((AContext)_localctx).c.stop):null);"; testActions(attributeTemplate, "inline", action, expected); } From d166e6d5d12b1f7dbe567a5a0715e383b1f432bb Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 17:14:19 -0600 Subject: [PATCH 18/25] Emit grammar file name instead of generated class file name in generated implementation of getGrammarFileName(). Implement getGrammarFileName() in generated parser. --- .../org/antlr/v4/tool/templates/codegen/Java/Java.stg | 5 ++++- tool/src/org/antlr/v4/codegen/model/Lexer.java | 3 +++ tool/src/org/antlr/v4/codegen/model/Parser.java | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 6805d5820..02d03003e 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -133,6 +133,9 @@ public class extends { + @Override + public String getGrammarFileName() { return ""; } + @Override public String[] getTokenNames() { return tokenNames; } @Override @@ -720,7 +723,7 @@ public class extends Lexer { _interp = new LexerATNSimulator(this,_ATN); } - public String getGrammarFileName() { return ""; } + public String getGrammarFileName() { return ""; } @Override public String[] getTokenNames() { return tokenNames; } diff --git a/tool/src/org/antlr/v4/codegen/model/Lexer.java b/tool/src/org/antlr/v4/codegen/model/Lexer.java index 051a97f20..45edc6193 100644 --- a/tool/src/org/antlr/v4/codegen/model/Lexer.java +++ b/tool/src/org/antlr/v4/codegen/model/Lexer.java @@ -35,6 +35,7 @@ import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; +import java.io.File; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -42,6 +43,7 @@ import java.util.Set; public class Lexer extends OutputModelObject { public String name; + public String grammarFileName; public Map tokens; public LexerFile file; public String[] tokenNames; @@ -58,6 +60,7 @@ public class Lexer extends OutputModelObject { this.factory = factory; this.file = file; // who contains us? Grammar g = factory.getGrammar(); + grammarFileName = new File(g.fileName).getName(); name = g.getRecognizerName(); tokens = new LinkedHashMap(); LexerGrammar lg = (LexerGrammar)g; diff --git a/tool/src/org/antlr/v4/codegen/model/Parser.java b/tool/src/org/antlr/v4/codegen/model/Parser.java index 91be7ac53..76c693ddf 100644 --- a/tool/src/org/antlr/v4/codegen/model/Parser.java +++ b/tool/src/org/antlr/v4/codegen/model/Parser.java @@ -33,11 +33,13 @@ import org.antlr.v4.codegen.*; import org.antlr.v4.codegen.model.chunk.*; import org.antlr.v4.tool.*; +import java.io.File; import java.util.*; /** */ public class Parser extends OutputModelObject { public String name; + public String grammarFileName; public String grammarName; @ModelElement public ActionChunk superclass; public Map tokens; @@ -55,6 +57,7 @@ public class Parser extends OutputModelObject { this.factory = factory; this.file = file; // who contains us? Grammar g = factory.getGrammar(); + grammarFileName = new File(g.fileName).getName(); grammarName = g.name; name = g.getRecognizerName(); tokens = new LinkedHashMap(); From a81b1a17eb92749cb3e5a321581080bd8e2deb0f Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 17:18:57 -0600 Subject: [PATCH 19/25] Extract abstract method in Recognizer for Parser.getGrammarFileName() and generated lexer's getGrammarFileName(). Make Recognizer.getTokenNames(), getRuleNames(), and getATN() abstract - implementations are always generated. --- runtime/Java/src/org/antlr/v4/runtime/Parser.java | 7 ------- .../Java/src/org/antlr/v4/runtime/Recognizer.java | 15 ++++++++------- .../antlr/v4/tool/templates/codegen/Java/Java.stg | 1 + .../antlr/v4/tool/interp/ParserInterpreter.java | 10 ++++++++++ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index 14f5bcf7d..e7af03393 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -476,13 +476,6 @@ public abstract class Parser extends Recognizer return stack; } - /** For debugging and other purposes, might want the grammar name. - * Have ANTLR generate an implementation for this method. - */ - public String getGrammarFileName() { - return null; - } - /** For debugging and other purposes */ public List getDFAStrings() { List s = new ArrayList(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java index f3a31bde3..db187a1b3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java @@ -53,15 +53,16 @@ public abstract class Recognizer { * error reporting. The generated parsers implement a method * that overrides this to point to their String[] tokenNames. */ - public String[] getTokenNames() { - return null; - } + public abstract String[] getTokenNames(); - public String[] getRuleNames() { - return null; - } + public abstract String[] getRuleNames(); - public ATN getATN() { return null; } + /** For debugging and other purposes, might want the grammar name. + * Have ANTLR generate an implementation for this method. + */ + public abstract String getGrammarFileName(); + + public abstract ATN getATN(); public ATNInterpreter getInterpreter() { return _interp; } diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 02d03003e..5007d9180 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -723,6 +723,7 @@ public class extends Lexer { _interp = new LexerATNSimulator(this,_ATN); } + @Override public String getGrammarFileName() { return ""; } @Override diff --git a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java index 55683426f..aaf5d2d4d 100644 --- a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java +++ b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java @@ -45,6 +45,11 @@ public class ParserInterpreter { this.g = g; } + @Override + public String getGrammarFileName() { + return null; + } + @Override public String[] getRuleNames() { return g.rules.keySet().toArray(new String[g.rules.size()]); @@ -54,6 +59,11 @@ public class ParserInterpreter { public String[] getTokenNames() { return g.getTokenNames(); } + + @Override + public ATN getATN() { + return null; + } } protected Grammar g; From ca0b1d399c56b6395a1f8df386140dd55b55c58e Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 17:21:44 -0600 Subject: [PATCH 20/25] Move "metadata" methods to top of generated parser --- .../org/antlr/v4/tool/templates/codegen/Java/Java.stg | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 5007d9180..56d6a320d 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -128,21 +128,24 @@ public class extends { public static final String[] ruleNames = { "}; separator=", ", wrap, anchor> }; - - - - @Override public String getGrammarFileName() { return ""; } @Override public String[] getTokenNames() { return tokenNames; } + @Override public String[] getRuleNames() { return ruleNames; } + @Override public ATN getATN() { return _ATN; } + + + + + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch ( ruleIndex ) { From f5edb7653a48ce8181af2870e68fe75a70abc940 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 17:23:38 -0600 Subject: [PATCH 21/25] Add missing @Override attributes in generated code, use hard tabs instead of spaces --- .../v4/tool/templates/codegen/Java/Java.stg | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 56d6a320d..aaf22a044 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -47,12 +47,12 @@ import org.antlr.v4.runtime.Token; public class BaseListener implements Listener { (.Context ctx) { \} -public void exit(.Context ctx) { \}}; separator="\n"> +@Override public void enter(.Context ctx) { \} +@Override public void exit(.Context ctx) { \}}; separator="\n"> - public void enterEveryRule(ParserRuleContext\<\> ctx) { } - public void exitEveryRule(ParserRuleContext\<\> ctx) { } - public void visitTerminal(ParserRuleContext\<\> ctx, symbol) { } + @Override public void enterEveryRule(ParserRuleContext\<\> ctx) { } + @Override public void exitEveryRule(ParserRuleContext\<\> ctx) { } + @Override public void visitTerminal(ParserRuleContext\<\> ctx, symbol) { } } >> @@ -76,13 +76,13 @@ import org.antlr.v4.runtime.*; public class BaseParseListener implements ParseListener { (ParserRuleContext\<\> ctx) { \}}; separator="\n"> +@Override public void enter(ParserRuleContext\<\> ctx) { \}}; separator="\n"> (.Context ctx) { \}}; separator="\n"> +@Override public void exit(.Context ctx) { \}}; separator="\n"> - public void enterNonLRRule(ParserRuleContext\<\> ctx) { } - public void exitEveryRule(ParserRuleContext\<\> ctx) { } - public void visitTerminal(ParserRuleContext\<\> ctx, symbol) { } + @Override public void enterNonLRRule(ParserRuleContext\<\> ctx) { } + @Override public void exitEveryRule(ParserRuleContext\<\> ctx) { } + @Override public void visitTerminal(ParserRuleContext\<\> ctx, symbol) { } } >> @@ -104,7 +104,7 @@ import org.antlr.v4.runtime.Token; public class BaseVisitor\ extends ParseTreeVisitor\ implements Visitor\ { (.Context ctx) { visitChildren(ctx); return null; \}}; separator="\n"> +@Override public T visit(.Context ctx) { visitChildren(ctx); return null; \}}; separator="\n"> } >> @@ -623,19 +623,22 @@ public static class extends Contex >> ListenerDispatchMethod(method) ::= << +@Override public void enterexitRule(ParseTreeListener\<\> listener) { if ( listener instanceof Listener ) ((Listener)listener).enterexit(this); } >> VisitorDispatchMethod(method) ::= << +@Override public \ T accept(ParseTreeVisitor\ visitor) { - if ( visitor instanceof Visitor ) return ((Visitor\)visitor).visit(this); - else return null; + if ( visitor instanceof Visitor ) return ((Visitor\)visitor).visit(this); + else return null; } >> ParseListenerDispatchMethod(method) ::= << +@Override public void enterexitRule(ParseListener\<\> listener) { if ( listener instanceof ParseListener ) ((ParseListener)listener).enterexit(this); } From 611728a3806d6a8c9fe44a538d34b448784986c1 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 18:51:26 -0600 Subject: [PATCH 22/25] Update testPerformance to show information about DFA states in the lexer --- .../v4/runtime/atn/LexerATNSimulator.java | 2 +- .../org/antlr/v4/test/TestPerformance.java | 25 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 3000c99b5..8aef0e84d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -114,7 +114,7 @@ public class LexerATNSimulator extends ATNSimulator { protected int charPositionInLine = 0; @NotNull - protected DFA[] dfa; + public final DFA[] dfa; protected int mode = Lexer.DEFAULT_MODE; /** Used during DFA/ATN exec to record the most recent accept configuration info */ diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java index 92a6e5a87..a41543415 100644 --- a/tool/test/org/antlr/v4/test/TestPerformance.java +++ b/tool/test/org/antlr/v4/test/TestPerformance.java @@ -284,6 +284,29 @@ public class TestPerformance extends BaseTest { tokenCount, System.currentTimeMillis() - startTime); + final LexerATNSimulator lexerInterpreter = sharedLexer.getInterpreter(); + final DFA[] modeToDFA = lexerInterpreter.dfa; + if (SHOW_DFA_STATE_STATS) { + int states = 0; + int configs = 0; + Set uniqueConfigs = new HashSet(); + + for (int i = 0; i < modeToDFA.length; i++) { + DFA dfa = modeToDFA[i]; + if (dfa == null || dfa.states == null) { + continue; + } + + states += dfa.states.size(); + for (DFAState state : dfa.states.values()) { + configs += state.configset.size(); + uniqueConfigs.addAll(state.configset); + } + } + + System.out.format("There are %d lexer DFAState instances, %d configs (%d unique).\n", states, configs, uniqueConfigs.size()); + } + if (RUN_PARSER) { // make sure the individual DFAState objects actually have unique ATNConfig arrays final ParserATNSimulator interpreter = sharedParser.getInterpreter(); @@ -307,7 +330,7 @@ public class TestPerformance extends BaseTest { } } - System.out.format("There are %d DFAState instances, %d configs (%d unique).\n", states, configs, uniqueConfigs.size()); + System.out.format("There are %d parser DFAState instances, %d configs (%d unique).\n", states, configs, uniqueConfigs.size()); } int localDfaCount = 0; From 5fb389b6e417918ebaef2780ce89b35b57498755 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 20 Feb 2012 19:20:50 -0600 Subject: [PATCH 23/25] Improve lexer performance during warm-up by tracking DFA states within exec (ATN) and using existing edges when available --- .../v4/runtime/atn/LexerATNSimulator.java | 76 +++++++++++-------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 3000c99b5..fb14642ba 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -201,7 +201,7 @@ public class LexerATNSimulator extends ATNSimulator { ATNConfigSet s0_closure = computeStartState(input, startState); int old_mode = mode; dfa[mode].s0 = addDFAState(s0_closure); - int predict = exec(input, s0_closure); + int predict = exec(input, s0_closure, dfa[mode].s0); if ( debug ) { System.out.format("DFA after matchATN: %s\n", dfa[old_mode].toLexerString()); @@ -282,7 +282,7 @@ public class LexerATNSimulator extends ATNSimulator { return dfaPrevAccept.state.prediction; } - protected int exec(@NotNull CharStream input, @NotNull ATNConfigSet s0) { + protected int exec(@NotNull CharStream input, @NotNull ATNConfigSet s0, @Nullable DFAState ds0) { //System.out.println("enter exec index "+input.index()+" from "+s0); @NotNull ATNConfigSet closure = new ATNConfigSet(); @@ -297,23 +297,37 @@ public class LexerATNSimulator extends ATNSimulator { traceLookahead1(); int t = input.LA(1); + DFAState s = ds0; while ( true ) { // while more work if ( debug ) { System.out.format("in reach starting closure: %s\n", closure); } - for (ATNConfig c : closure) { - if ( debug ) { - System.out.format("testing %s at %s\n", getTokenName(t), c.toString(recog, true)); + DFAState next = null; + if (s != null) { + if ( s.edges != null && t < s.edges.length && t > CharStream.EOF ) { + closure = s.configset; + next = s.edges[t]; + if (next != null) { + reach = next.configset; + } } + } - int n = c.state.getNumberOfTransitions(); - for (int ti=0; ti MAX_DFA_EDGE) { - return; + return to; } // System.out.println("MOVE "+p+" -> "+q+" upon "+getTokenName(t)); - DFAState from = addDFAState(p); - if (from == null) { - return; - } - - DFAState to = addDFAState(q); - if (to == null) { - return; + if (from == null || to == null) { + return to; } addDFAEdge(from, t, to); + return to; } protected void addDFAEdge(@NotNull DFAState p, int t, @NotNull DFAState q) { From 58ef729be58d986017215528d6141d9697b64e24 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Tue, 21 Feb 2012 12:47:34 -0800 Subject: [PATCH 24/25] renamed next to target. factored out to methods, commented code. --- .../v4/runtime/atn/LexerATNSimulator.java | 79 +++++++++++++------ 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index f29319fa9..749a0a0a1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -297,42 +297,47 @@ public class LexerATNSimulator extends ATNSimulator { traceLookahead1(); int t = input.LA(1); - DFAState s = ds0; + DFAState s = ds0; // s is current/from DFA state while ( true ) { // while more work if ( debug ) { System.out.format("in reach starting closure: %s\n", closure); } - DFAState next = null; + // As we move src->trg, src->trg, we keep track of the previous trg to + // avoid looking up the DFA state again, which is expensive. + // If the previous target was already part of the DFA, we might + // be able to avoid doing a reach operation upon t. If s!=null, + // it means that semantic predicates didn't prevent us from + // creating a DFA state. Once we know s!=null, we check to see if + // the DFA state has an edge already for t. If so, we can just reuse + // it's configuration set; there's no point in re-computing it. + // This is kind of like doing DFA simulation within the ATN + // simulation because DFA simulation is really just a way to avoid + // computing reach/closure sets. Technically, once we know that + // we have a previously added DFA state, we could jump over to + // the DFA simulator. But, that would mean popping back and forth + // a lot and making things more complicated algorithmically. + // This optimization makes a lot of sense for loops within DFA. + // A character will take us back to an existing DFA state + // that already has lots of edges out of it. e.g., .* in comments. + DFAState target = null; if (s != null) { if ( s.edges != null && t < s.edges.length && t > CharStream.EOF ) { closure = s.configset; - next = s.edges[t]; - if (next != null) { - reach = next.configset; + target = s.edges[t]; + if (target != null) { + reach = target.configset; } } } - if (next == null) { - for (ATNConfig c : closure) { - if ( debug ) { - System.out.format("testing %s at %s\n", getTokenName(t), c.toString(recog, true)); - } - - int n = c.state.getNumberOfTransitions(); - for (int ti=0; ti Date: Wed, 22 Feb 2012 12:44:33 -0800 Subject: [PATCH 25/25] added listener unit tests. fixed bug that didn't create ctx getters properly for recursive rules. added Symbol extends Token to parse tree stuff. added visitTerminal to Visitor. recursive alts now track their original, unedited AltAST subtree so we can properly count rule refs etc... later. dup of RuleRefAST was making wrong node. don't gen dispatch methods if no listener. --- .../org/antlr/v4/runtime/ParseListener.java | 2 +- .../antlr/v4/runtime/ParserRuleContext.java | 45 +++--- .../org/antlr/v4/runtime/tree/ParseTree.java | 25 +-- .../v4/runtime/tree/ParseTreeListener.java | 3 +- .../v4/runtime/tree/ParseTreeVisitor.java | 24 ++- .../v4/runtime/tree/ParseTreeWalker.java | 9 +- tool/playground/Errors.g | 56 ------- tool/playground/T.g | 40 +++-- tool/playground/TestT.java | 4 +- .../v4/tool/templates/codegen/Java/Java.stg | 4 +- tool/src/org/antlr/v4/Tool.java | 47 ++++-- .../v4/analysis/LeftRecursiveRuleAltInfo.java | 12 +- .../analysis/LeftRecursiveRuleAnalyzer.java | 58 ++++--- .../LeftRecursiveRuleTransformer.java | 39 ++++- .../model/LeftRecursiveRuleFunction.java | 9 +- .../antlr/v4/codegen/model/RuleFunction.java | 66 +++++--- .../v4/codegen/model/decl/StructDecl.java | 21 ++- .../org/antlr/v4/tool/LeftRecursiveRule.java | 12 +- tool/src/org/antlr/v4/tool/ast/AltAST.java | 1 + .../src/org/antlr/v4/tool/ast/RuleRefAST.java | 8 +- .../test/org/antlr/v4/test/TestListeners.java | 149 ++++++++++++++++++ 21 files changed, 430 insertions(+), 204 deletions(-) delete mode 100644 tool/playground/Errors.g create mode 100644 tool/test/org/antlr/v4/test/TestListeners.java diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParseListener.java b/runtime/Java/src/org/antlr/v4/runtime/ParseListener.java index 6821bdbe2..73c11688d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParseListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParseListener.java @@ -8,7 +8,7 @@ package org.antlr.v4.runtime; * make the interface clear these semantics up. If you need the ctx, * use Parser.getRuleContext(). */ -public interface ParseListener { +public interface ParseListener { void visitTerminal(ParserRuleContext ctx, Symbol symbol); /** Enter all but left-recursive rules */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java index b5df15044..ac68aebd8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java @@ -28,11 +28,17 @@ */ package org.antlr.v4.runtime; -import org.antlr.v4.runtime.atn.*; -import org.antlr.v4.runtime.misc.*; -import org.antlr.v4.runtime.tree.*; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNState; +import org.antlr.v4.runtime.misc.NotNull; +import org.antlr.v4.runtime.misc.Nullable; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.ParseTreeVisitor; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** A rule invocation record for parsing and tree parsing. * @@ -57,8 +63,8 @@ import java.util.*; * group values such as this aggregate. The getters/setters are there to * satisfy the superclass interface. */ -public class ParserRuleContext extends RuleContext { - public static final ParserRuleContext EMPTY = new ParserRuleContext(); +public class ParserRuleContext extends RuleContext { + public static final ParserRuleContext EMPTY = new ParserRuleContext(); /** If we are debugging or building a parse tree for a visitor, * we need to track all of the tokens and rule invocations associated @@ -137,7 +143,7 @@ public class ParserRuleContext extends RuleContext { public void exitRule(ParseTreeListener listener) { } // visitor - public T accept(ParseTreeVisitor visitor) { visitor.visitChildren(this); return null; } + public T accept(ParseTreeVisitor visitor) { return visitor.visitChildren(this); } /** Does not set parent link; other add methods do */ @@ -209,13 +215,11 @@ public class ParserRuleContext extends RuleContext { for (ParseTree o : children) { if ( o instanceof TerminalNode ) { TerminalNode tnode = (TerminalNode)o; - if ( tnode.getSymbol() instanceof Token ) { - Token symbol = (Token)tnode.getSymbol(); - if ( symbol.getType()==ttype ) { - j++; - if ( j == i ) { - return symbol; - } + Token symbol = tnode.getSymbol(); + if ( symbol.getType()==ttype ) { + j++; + if ( j == i ) { + return symbol; } } } @@ -233,13 +237,11 @@ public class ParserRuleContext extends RuleContext { for (ParseTree o : children) { if ( o instanceof TerminalNode ) { TerminalNode tnode = (TerminalNode)o; - if ( tnode.getSymbol() instanceof Token ) { - Token symbol = (Token)tnode.getSymbol(); - if ( tokens==null ) { - tokens = new ArrayList(); - } - tokens.add(symbol); + Token symbol = tnode.getSymbol(); + if ( tokens==null ) { + tokens = new ArrayList(); } + tokens.add(symbol); } } @@ -298,9 +300,6 @@ public class ParserRuleContext extends RuleContext { String ruleName = recog.getRuleNames()[s.ruleIndex]; buf.append(ruleName); if ( p.parent != null ) buf.append(" "); -// ATNState invoker = atn.states.get(ctx.invokingState); -// RuleTransition rt = (RuleTransition)invoker.transition(0); -// buf.append(recog.getRuleNames()[rt.target.ruleIndex]); p = (ParserRuleContext)p.parent; } buf.append("]"); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java index 53e7864c0..7b6adbb9a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java @@ -45,11 +45,11 @@ public interface ParseTree extends SyntaxTree { RuleContext getRuleContext(); } - public interface TerminalNode extends ParseTree { + public interface TerminalNode extends ParseTree { Symbol getSymbol(); } - public static class TerminalNodeImpl implements TerminalNode { + public static class TerminalNodeImpl implements TerminalNode { public Symbol symbol; public ParseTree parent; /** Which ATN node matched this token? */ @@ -72,13 +72,7 @@ public interface ParseTree extends SyntaxTree { public Interval getSourceInterval() { if ( symbol ==null ) return Interval.INVALID; - if (symbol instanceof Token) { - return new Interval(((Token)symbol).getStartIndex(), ((Token)symbol).getStopIndex()); - } else if (symbol instanceof SyntaxTree) { - return ((SyntaxTree)symbol).getSourceInterval(); - } else { - throw new UnsupportedOperationException("This symbol type is not supported by the default implementation."); - } + return new Interval(symbol.getStartIndex(), symbol.getStopIndex()); } @Override @@ -86,13 +80,8 @@ public interface ParseTree extends SyntaxTree { @Override public String toString() { - if (symbol instanceof Token) { - if ( ((Token)symbol).getType() == Token.EOF ) return ""; - return ((Token)symbol).getText(); - } - else { - throw new UnsupportedOperationException("This symbol type is not supported by the default implementation."); - } + if ( symbol.getType() == Token.EOF ) return ""; + return symbol.getText(); } @Override @@ -107,12 +96,10 @@ public interface ParseTree extends SyntaxTree { * and deletion as well as during "consume until error recovery set" * upon no viable alternative exceptions. */ - public static class ErrorNodeImpl extends TerminalNodeImpl { + public static class ErrorNodeImpl extends TerminalNodeImpl { public ErrorNodeImpl(Symbol token) { super(token); } -// @Override -// public String toString() { return ""; } } // the following methods narrow the return type; they are not additional methods diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java index fa6872c62..bebf8b7f7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java @@ -30,8 +30,9 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; -public interface ParseTreeListener { +public interface ParseTreeListener { void visitTerminal(ParserRuleContext ctx, Symbol symbol); void enterEveryRule(ParserRuleContext ctx); void exitEveryRule(ParserRuleContext ctx); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java index 5dd260312..d07820523 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java @@ -1,6 +1,7 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; /** T is return type of visit methods. Use T=Void for no return type. */ public class ParseTreeVisitor { @@ -8,19 +9,30 @@ public class ParseTreeVisitor { return ctx.accept(this); } - /** Visit all rule, nonleaf children. Not useful if you are using T as - * non-Void. This returns nothing, losing all computations from below. - * But handy if you are just walking the tree with a visitor and only + /** Visit all rule, nonleaf children. Not that useful if you are using T as + * non-Void. This returns value returned from last child visited, + * losing all computations from first n-1 children. Works fine for + * ctxs with one child then. + * Handy if you are just walking the tree with a visitor and only * care about some nodes. The ParserRuleContext.accept() method * walks all children by default; i.e., calls this method. */ - public void visitChildren(ParserRuleContext ctx) { + public T visitChildren(ParserRuleContext ctx) { + T result = null; for (ParseTree c : ctx.children) { if ( c instanceof ParseTree.RuleNode) { ParseTree.RuleNode r = (ParseTree.RuleNode)c; - ParserRuleContext rctx = (ParserRuleContext)r.getRuleContext(); - visit(rctx); + ParserRuleContext rctx = (ParserRuleContext)r.getRuleContext(); + result = visit(rctx); + } + else { + result = visitTerminal(ctx, ((ParseTree.TerminalNode)c).getSymbol()); } } + return result; + } + + public T visitTerminal(ParserRuleContext ctx, Token symbol) { + return null; } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java index d25361a7b..1f152448d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java @@ -30,12 +30,13 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; public class ParseTreeWalker { public static final ParseTreeWalker DEFAULT = new ParseTreeWalker(); @SuppressWarnings("unchecked") - public void walk(ParseTreeListener listener, ParseTree t) { + public void walk(ParseTreeListener listener, ParseTree t) { if ( t instanceof ParseTree.TerminalNode) { visitTerminal(listener, (ParseTree.TerminalNode) t); return; @@ -50,7 +51,7 @@ public class ParseTreeWalker { } @SuppressWarnings("unchecked") - protected void visitTerminal(ParseTreeListener listener, + protected void visitTerminal(ParseTreeListener listener, ParseTree.TerminalNode t) { ParseTree.RuleNode r = (ParseTree.RuleNode)t.getParent(); @@ -66,14 +67,14 @@ public class ParseTreeWalker { * First we trigger the generic and then the rule specific. * We to them in reverse order upon finishing the node. */ - protected void enterRule(ParseTreeListener listener, ParseTree.RuleNode r) { + protected void enterRule(ParseTreeListener listener, ParseTree.RuleNode r) { @SuppressWarnings("unchecked") ParserRuleContext ctx = (ParserRuleContext)r.getRuleContext(); listener.enterEveryRule(ctx); ctx.enterRule(listener); } - protected void exitRule(ParseTreeListener listener, ParseTree.RuleNode r) { + protected void exitRule(ParseTreeListener listener, ParseTree.RuleNode r) { @SuppressWarnings("unchecked") ParserRuleContext ctx = (ParserRuleContext)r.getRuleContext(); ctx.exitRule(listener); diff --git a/tool/playground/Errors.g b/tool/playground/Errors.g deleted file mode 100644 index a15bbdf1c..000000000 --- a/tool/playground/Errors.g +++ /dev/null @@ -1,56 +0,0 @@ -grammar Errors; - -stat: 'return' INT - | ID '=' expr ';' - | ID '(' expr (',' expr)* ')' ';' - / ID .* '(' expr (',' expr)* ')' ';' - / ID '=' .* ';' // bad assignment - / .* ';' // bad stat - / .* // match anything else? when to stop? - / // match anything else? - ; - catch[Exception e] { } - finally { } - -// error to match might be diff than how to resynch? maybe just -// include resynch pattern on end of error alt. - -/* -Traps any recog exception in anything called from rule or matched in that rule. -a : expr ';' - / '--' ID ';' // catches any problem in expr or matching ';' - ; - -If no err alt matches, defaults to normal error mechanism at rule level. -report. resync. -*/ - -atom: '(' expr ')' - | INT - / '(' expr // missing RP; how to resync? - / '(' ')' - ; - -// do error alts affect FOLLOW sync sets? nope. - -// foo -> bar says how to make resulting tree for bad alts - -expr: atom ('*' atom)* ; - -atom: INT ; - -ID : 'a'..'z'+ ; - -WS : (' '|'\n')* ; - -/* -Stop .* when it sees any viable following token, even if it uses FOLLOW. So, -err alt - - / .* - -would match until it sees something in FOLLOW (but not context-sensitive follow). -actually maybe it would be sensitive; just use real outer context when matching -error alts. who cares about speed. - -*/ diff --git a/tool/playground/T.g b/tool/playground/T.g index 2659bfaac..0c89ee18e 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,19 +1,27 @@ grammar T; -/* This is ambig too. -s_ : s EOF ; -s : a s - | - ; +@members { +public static class LeafListener extends TBaseListener { + public void exitA(TParser.EContext ctx) { +/* + if (ctx.getChildCount()==3) { + System.out.printf("%s %s %s",ctx.e(0).start.getText(), + ctx.e(1).start.getText(),ctx.e().get(0).start.getText()); + } + else System.out.println(ctx.INT(0).start.getText()); */ - -s : (a)* EOF ; // ambig; can match A B in alt 3 or alt 2 then alt 1 -a : e '!' - | e + } + }} +s +@init {setBuildParseTree(true);} +@after { System.out.println($r.ctx.toStringTree(this)); ParseTreeWalker walker = new ParseTreeWalker(); + walker.walk(new LeafListener(), $r.ctx);} + : r=e ; +e : e op='*' e + | e op='+' e + | e '++' + | INT ; -e : B - | A // both alts 2,3 can reach end of s upon abEOF - | A B - ; -A : 'a' ; -B : 'b' ; -WS : (' '|'\n')+ {skip();} ; +MULT: '*' ; +ADD : '+' ; +INT : [0-9]+ ; +WS : [ \t\n]+ -> skip ; diff --git a/tool/playground/TestT.java b/tool/playground/TestT.java index 7e801a892..04333c34f 100644 --- a/tool/playground/TestT.java +++ b/tool/playground/TestT.java @@ -40,8 +40,8 @@ public class TestT { } TParser p = new TParser(tokens); p.setBuildParseTree(true); - final TParser.sContext tree = p.s(); - System.out.println(tree.toStringTree(p)); +// final TParser.sContext tree = p.s(); +// System.out.println(tree.toStringTree(p)); // TreeViewer v = new TreeViewer(p, tree); // v.setHighlightedBoxColor(TreeViewer.LIGHT_RED); // v.addHighlightedNodes(new ArrayList() {{ diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index aaf22a044..09ee1977f 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -101,10 +101,12 @@ BaseVisitorFile(file, header) ::= <<
import org.antlr.v4.runtime.tree.*; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.ParserRuleContext; public class BaseVisitor\ extends ParseTreeVisitor\ implements Visitor\ { (.Context ctx) { visitChildren(ctx); return null; \}}; separator="\n"> +@Override public T visit(.Context ctx) { return visitChildren(ctx); \}}; separator="\n"> + @Override public T visitTerminal(ParserRuleContext\ ctx, Token symbol) { return null; } } >> diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 14b336fad..8492a9e63 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -29,20 +29,50 @@ package org.antlr.v4; -import org.antlr.runtime.*; +import org.antlr.runtime.ANTLRFileStream; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.ParserRuleReturnScope; +import org.antlr.runtime.RecognitionException; import org.antlr.v4.analysis.AnalysisPipeline; -import org.antlr.v4.automata.*; +import org.antlr.v4.automata.ATNFactory; +import org.antlr.v4.automata.LexerATNFactory; +import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.codegen.CodeGenPipeline; -import org.antlr.v4.parse.*; -import org.antlr.v4.runtime.misc.*; +import org.antlr.v4.parse.ANTLRLexer; +import org.antlr.v4.parse.ANTLRParser; +import org.antlr.v4.parse.GrammarASTAdaptor; +import org.antlr.v4.parse.ToolANTLRParser; +import org.antlr.v4.runtime.misc.LogManager; +import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.tool.*; -import org.antlr.v4.tool.ast.*; +import org.antlr.v4.tool.ANTLRMessage; +import org.antlr.v4.tool.ANTLRToolListener; +import org.antlr.v4.tool.DOTGenerator; +import org.antlr.v4.tool.DefaultToolListener; +import org.antlr.v4.tool.ErrorManager; +import org.antlr.v4.tool.ErrorType; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.GrammarTransformPipeline; +import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.Rule; +import org.antlr.v4.tool.ast.GrammarAST; +import org.antlr.v4.tool.ast.GrammarASTErrorNode; +import org.antlr.v4.tool.ast.GrammarRootAST; import org.stringtemplate.v4.STGroup; -import java.io.*; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; import java.lang.reflect.Field; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; public class Tool { public String VERSION = "4.0-"+new Date(); @@ -83,7 +113,6 @@ public class Tool { public boolean force_atn = false; public boolean log = false; public boolean verbose_dfa = false; - public boolean no_auto_element_labels = false; public boolean gen_listener = true; public boolean gen_parse_listener = false; public boolean gen_visitor = false; diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java index 9e30945b0..c8d3b5df3 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java @@ -36,17 +36,23 @@ public class LeftRecursiveRuleAltInfo { public String leftRecursiveRuleRefLabel; public String altLabel; public String altText; - public AltAST altAST; + public AltAST altAST; // transformed ALT + public AltAST originalAltAST; public int nextPrec; public LeftRecursiveRuleAltInfo(int altNum, String altText) { - this(altNum, altText, null, null); + this(altNum, altText, null, null, null); } - public LeftRecursiveRuleAltInfo(int altNum, String altText, String leftRecursiveRuleRefLabel, String altLabel) { + public LeftRecursiveRuleAltInfo(int altNum, String altText, + String leftRecursiveRuleRefLabel, + String altLabel, + AltAST originalAltAST) + { this.altNum = altNum; this.altText = altText; this.leftRecursiveRuleRefLabel = leftRecursiveRuleRefLabel; this.altLabel = altLabel; + this.originalAltAST = originalAltAST; } } diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java index 57a71a442..170d01379 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java @@ -29,17 +29,28 @@ package org.antlr.v4.analysis; -import org.antlr.runtime.*; -import org.antlr.runtime.tree.*; +import org.antlr.runtime.CommonToken; +import org.antlr.runtime.TokenStream; +import org.antlr.runtime.tree.CommonTreeNodeStream; +import org.antlr.runtime.tree.Tree; import org.antlr.v4.Tool; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.misc.Pair; -import org.antlr.v4.parse.*; +import org.antlr.v4.parse.GrammarASTAdaptor; +import org.antlr.v4.parse.LeftRecursiveRuleWalker; import org.antlr.v4.tool.ErrorType; -import org.antlr.v4.tool.ast.*; -import org.stringtemplate.v4.*; +import org.antlr.v4.tool.ast.AltAST; +import org.antlr.v4.tool.ast.GrammarAST; +import org.antlr.v4.tool.ast.GrammarASTWithOptions; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupFile; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** Using a tree walker on the rules, determine if a rule is directly left-recursive and if it follows * our pattern. @@ -124,8 +135,8 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { } @Override - public void binaryAlt(AltAST altTree, int alt) { - altTree = (AltAST)altTree.dupTree(); + public void binaryAlt(AltAST originalAltTree, int alt) { + AltAST altTree = (AltAST)originalAltTree.dupTree(); String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null; GrammarAST lrlabel = stripLeftRecursion(altTree); @@ -143,7 +154,8 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { stripAltLabel(altTree); String altText = text(altTree); altText = altText.trim(); - LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel); + LeftRecursiveRuleAltInfo a = + new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, originalAltTree); a.nextPrec = nextPrec; binaryAlts.put(alt, a); //System.out.println("binaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); @@ -151,8 +163,8 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { /** Convert e ? e : e -> ? e : e_[nextPrec] */ @Override - public void ternaryAlt(AltAST altTree, int alt) { - altTree = (AltAST)altTree.dupTree(); + public void ternaryAlt(AltAST originalAltTree, int alt) { + AltAST altTree = (AltAST)originalAltTree.dupTree(); String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null; GrammarAST lrlabel = stripLeftRecursion(altTree); @@ -168,15 +180,16 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { String altText = text(altTree); altText = altText.trim(); - LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel); + LeftRecursiveRuleAltInfo a = + new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, originalAltTree); a.nextPrec = nextPrec; ternaryAlts.put(alt, a); //System.out.println("ternaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); } @Override - public void prefixAlt(AltAST altTree, int alt) { - altTree = (AltAST)altTree.dupTree(); + public void prefixAlt(AltAST originalAltTree, int alt) { + AltAST altTree = (AltAST)originalAltTree.dupTree(); stripAltLabel(altTree); int nextPrec = precedence(alt); @@ -185,15 +198,16 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { String altText = text(altTree); altText = altText.trim(); String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null; - LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel); + LeftRecursiveRuleAltInfo a = + new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, originalAltTree); a.nextPrec = nextPrec; prefixAlts.add(a); //System.out.println("prefixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); } @Override - public void suffixAlt(AltAST altTree, int alt) { - altTree = (AltAST)altTree.dupTree(); + public void suffixAlt(AltAST originalAltTree, int alt) { + AltAST altTree = (AltAST)originalAltTree.dupTree(); String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null; GrammarAST lrlabel = stripLeftRecursion(altTree); @@ -204,18 +218,20 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { stripAltLabel(altTree); String altText = text(altTree); altText = altText.trim(); - LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel); + LeftRecursiveRuleAltInfo a = + new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, originalAltTree); suffixAlts.put(alt, a); // System.out.println("suffixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); } @Override - public void otherAlt(AltAST altTree, int alt) { - altTree = (AltAST)altTree.dupTree(); + public void otherAlt(AltAST originalAltTree, int alt) { + AltAST altTree = (AltAST)originalAltTree.dupTree(); stripAltLabel(altTree); String altText = text(altTree); String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null; - LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel); + LeftRecursiveRuleAltInfo a = + new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, originalAltTree); otherAlts.add(a); // System.out.println("otherAlt " + alt + ": " + altText); } diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java index 3f100f383..172f3b93f 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java @@ -29,14 +29,37 @@ package org.antlr.v4.analysis; -import org.antlr.runtime.*; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CommonToken; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.ParserRuleReturnScope; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.TokenStream; import org.antlr.v4.Tool; -import org.antlr.v4.misc.*; -import org.antlr.v4.parse.*; -import org.antlr.v4.tool.*; -import org.antlr.v4.tool.ast.*; +import org.antlr.v4.misc.OrderedHashMap; +import org.antlr.v4.misc.Pair; +import org.antlr.v4.parse.ANTLRLexer; +import org.antlr.v4.parse.ANTLRParser; +import org.antlr.v4.parse.GrammarASTAdaptor; +import org.antlr.v4.parse.ScopeParser; +import org.antlr.v4.parse.ToolANTLRParser; +import org.antlr.v4.tool.AttributeDict; +import org.antlr.v4.tool.ErrorType; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.GrammarTransformPipeline; +import org.antlr.v4.tool.LabelElementPair; +import org.antlr.v4.tool.LeftRecursiveRule; +import org.antlr.v4.tool.Rule; +import org.antlr.v4.tool.ast.ActionAST; +import org.antlr.v4.tool.ast.AltAST; +import org.antlr.v4.tool.ast.BlockAST; +import org.antlr.v4.tool.ast.GrammarAST; +import org.antlr.v4.tool.ast.GrammarRootAST; +import org.antlr.v4.tool.ast.RuleAST; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; /** Remove left-recursive rule refs, add precedence args to recursive rule refs. * Rewrite rule so we can create ATN. @@ -201,12 +224,16 @@ public class LeftRecursiveRuleTransformer { LeftRecursiveRuleAltInfo altInfo = r.recPrimaryAlts.get(i); altInfo.altAST = (AltAST)primaryBlk.getChild(i); altInfo.altAST.leftRecursiveAltInfo = altInfo; + altInfo.originalAltAST.leftRecursiveAltInfo = altInfo; + altInfo.originalAltAST.parent = altInfo.altAST.parent; // System.out.println(altInfo.altAST.toStringTree()); } for (int i = 0; i < r.recOpAlts.size(); i++) { LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.getElement(i); altInfo.altAST = (AltAST)opsBlk.getChild(i); altInfo.altAST.leftRecursiveAltInfo = altInfo; + altInfo.originalAltAST.leftRecursiveAltInfo = altInfo; + altInfo.originalAltAST.parent = altInfo.altAST.parent; // System.out.println(altInfo.altAST.toStringTree()); } } diff --git a/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java b/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java index 4d5d34f12..318970f17 100644 --- a/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java @@ -29,11 +29,14 @@ package org.antlr.v4.codegen.model; -import org.antlr.v4.codegen.*; -import org.antlr.v4.codegen.model.decl.*; +import org.antlr.v4.codegen.CodeGenerator; +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.codegen.model.decl.RuleContextDecl; +import org.antlr.v4.codegen.model.decl.StructDecl; import org.antlr.v4.misc.Pair; import org.antlr.v4.parse.ANTLRParser; -import org.antlr.v4.tool.*; +import org.antlr.v4.tool.LeftRecursiveRule; +import org.antlr.v4.tool.Rule; import org.antlr.v4.tool.ast.GrammarAST; public class LeftRecursiveRuleFunction extends RuleFunction { diff --git a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java index 1bafda149..bb40d2d83 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java @@ -30,7 +30,15 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.codegen.model.decl.*; +import org.antlr.v4.codegen.model.decl.AltLabelStructDecl; +import org.antlr.v4.codegen.model.decl.ContextRuleGetterDecl; +import org.antlr.v4.codegen.model.decl.ContextRuleListGetterDecl; +import org.antlr.v4.codegen.model.decl.ContextRuleListIndexedGetterDecl; +import org.antlr.v4.codegen.model.decl.ContextTokenGetterDecl; +import org.antlr.v4.codegen.model.decl.ContextTokenListGetterDecl; +import org.antlr.v4.codegen.model.decl.ContextTokenListIndexedGetterDecl; +import org.antlr.v4.codegen.model.decl.Decl; +import org.antlr.v4.codegen.model.decl.StructDecl; import org.antlr.v4.misc.FrequencySet; import org.antlr.v4.misc.Triple; import org.antlr.v4.misc.Utils; @@ -42,9 +50,18 @@ import org.antlr.v4.tool.Rule; import org.antlr.v4.tool.ast.AltAST; import org.antlr.v4.tool.ast.GrammarAST; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; -import static org.antlr.v4.parse.ANTLRParser.*; +import static org.antlr.v4.parse.ANTLRParser.CLOSURE; +import static org.antlr.v4.parse.ANTLRParser.POSITIVE_CLOSURE; +import static org.antlr.v4.parse.ANTLRParser.RULE_REF; +import static org.antlr.v4.parse.ANTLRParser.TOKEN_REF; /** */ public class RuleFunction extends OutputModelObject { @@ -84,13 +101,11 @@ public class RuleFunction extends OutputModelObject { altToContext = new AltLabelStructDecl[r.getOriginalNumberOfAlts()+1]; // Add ctx labels for elements in alts with no -> label - if ( !factory.getGrammar().tool.no_auto_element_labels ) { - List altsNoLabels = r.getUnlabeledAltASTs(); - if ( altsNoLabels!=null ) { - Set decls = getDeclsForAllElements(altsNoLabels); - // we know to put in rule ctx, so do it directly - for (Decl d : decls) ruleCtx.addDecl(d); - } + List altsNoLabels = r.getUnlabeledAltASTs(); + if ( altsNoLabels!=null ) { + Set decls = getDeclsForAllElements(altsNoLabels); + // we know to put in rule ctx, so do it directly + for (Decl d : decls) ruleCtx.addDecl(d); } // make structs for -> labeled alts, define ctx labels for elements @@ -103,11 +118,9 @@ public class RuleFunction extends OutputModelObject { String label = pair.c; altToContext[altNum] = new AltLabelStructDecl(factory, r, altNum, label); altLabelCtxs.put(label, altToContext[altNum]); - if ( !factory.getGrammar().tool.no_auto_element_labels ) { - Set decls = getDeclsForAltElements(altAST); - // we know which ctx to put in, so do it directly - for (Decl d : decls) altToContext[altNum].addDecl(d); - } + Set decls = getDeclsForAltElements(altAST); + // we know which ctx to put in, so do it directly + for (Decl d : decls) altToContext[altNum].addDecl(d); } } @@ -139,23 +152,18 @@ public class RuleFunction extends OutputModelObject { } } - /** for all alts, find which ref X or r in way which needs List + /** for all alts, find which ref X or r needs List Must see across alts. If any alt needs X or r as list, then define as list. */ public Set getDeclsForAllElements(List altASTs) { Set needsList = new HashSet(); List allRefs = new ArrayList(); -// for (Alternative a :alts) { for (AltAST ast : altASTs) { IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF); List refs = ast.getNodesWithType(reftypes); - FrequencySet altFreq = new FrequencySet(); - for (GrammarAST t : refs) { - String refLabelName = t.getText(); - altFreq.add(refLabelName); - allRefs.add(t); - } + allRefs.addAll(refs); + FrequencySet altFreq = getElementFrequenciesForAlt(ast); for (GrammarAST t : refs) { String refLabelName = t.getText(); if ( altFreq.count(t.getText())>1 ) needsList.add(refLabelName); @@ -172,6 +180,18 @@ public class RuleFunction extends OutputModelObject { return decls; } + /** Given list of X and r refs in alt, compute how many of each there are */ + protected FrequencySet getElementFrequenciesForAlt(AltAST ast) { + IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF); + List refs = ast.getNodesWithType(reftypes); + FrequencySet altFreq = new FrequencySet(); + for (GrammarAST t : refs) { + String refLabelName = t.getText(); + altFreq.add(refLabelName); + } + return altFreq; + } + /** Get list of decls for token/rule refs. * Single ref X becomes X() getter * Multiple refs to X becomes List X() method, X(int i) method. diff --git a/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java index c24352e00..49ef47356 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java @@ -30,11 +30,20 @@ package org.antlr.v4.codegen.model.decl; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.codegen.model.*; +import org.antlr.v4.codegen.model.DispatchMethod; +import org.antlr.v4.codegen.model.ListenerDispatchMethod; +import org.antlr.v4.codegen.model.ModelElement; +import org.antlr.v4.codegen.model.OutputModelObject; +import org.antlr.v4.codegen.model.ParseListenerDispatchMethod; +import org.antlr.v4.codegen.model.VisitorDispatchMethod; import org.antlr.v4.runtime.misc.OrderedHashSet; -import org.antlr.v4.tool.*; +import org.antlr.v4.tool.Attribute; +import org.antlr.v4.tool.LeftRecursiveRule; +import org.antlr.v4.tool.Rule; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; /** This object models the structure holding all of the parameters, * return values, local variables, and labels associated with a rule. @@ -60,8 +69,10 @@ public class StructDecl extends Decl { dispatchMethods = new ArrayList(); if ( !r.hasAltSpecificContexts() ) { // no enter/exit for this ruleContext if rule has labels - dispatchMethods.add(new ListenerDispatchMethod(factory, true)); - dispatchMethods.add(new ListenerDispatchMethod(factory, false)); + if ( factory.getGrammar().tool.gen_listener ) { + dispatchMethods.add(new ListenerDispatchMethod(factory, true)); + dispatchMethods.add(new ListenerDispatchMethod(factory, false)); + } if ( factory.getGrammar().tool.gen_visitor ) { dispatchMethods.add(new VisitorDispatchMethod(factory)); } diff --git a/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java b/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java index 5e694928d..855eaa41b 100644 --- a/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java +++ b/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java @@ -72,11 +72,11 @@ public class LeftRecursiveRule extends Rule { List alts = new ArrayList(); for (int i = 0; i < recPrimaryAlts.size(); i++) { LeftRecursiveRuleAltInfo altInfo = recPrimaryAlts.get(i); - if ( altInfo.altLabel==null ) alts.add(altInfo.altAST); + if ( altInfo.altLabel==null ) alts.add(altInfo.originalAltAST); } for (int i = 0; i < recOpAlts.size(); i++) { LeftRecursiveRuleAltInfo altInfo = recOpAlts.getElement(i); - if ( altInfo.altLabel==null ) alts.add(altInfo.altAST); + if ( altInfo.altLabel==null ) alts.add(altInfo.originalAltAST); } if ( alts.size()==0 ) return null; return alts; @@ -91,13 +91,17 @@ public class LeftRecursiveRule extends Rule { for (int i = 0; i < recPrimaryAlts.size(); i++) { LeftRecursiveRuleAltInfo altInfo = recPrimaryAlts.get(i); if ( altInfo.altLabel!=null ) { - labels.add(new Triple(altInfo.altNum,altInfo.altAST,altInfo.altLabel)); + labels.add(new Triple(altInfo.altNum, + altInfo.originalAltAST, + altInfo.altLabel)); } } for (int i = 0; i < recOpAlts.size(); i++) { LeftRecursiveRuleAltInfo altInfo = recOpAlts.getElement(i); if ( altInfo.altLabel!=null ) { - labels.add(new Triple(altInfo.altNum,altInfo.altAST,altInfo.altLabel)); + labels.add(new Triple(altInfo.altNum, + altInfo.originalAltAST, + altInfo.altLabel)); } } if ( labels.size()==0 ) return null; diff --git a/tool/src/org/antlr/v4/tool/ast/AltAST.java b/tool/src/org/antlr/v4/tool/ast/AltAST.java index da897fbee..37902e1b1 100644 --- a/tool/src/org/antlr/v4/tool/ast/AltAST.java +++ b/tool/src/org/antlr/v4/tool/ast/AltAST.java @@ -50,6 +50,7 @@ public class AltAST extends GrammarAST { super(node); this.alt = ((AltAST)node).alt; this.altLabel = ((AltAST)node).altLabel; + this.leftRecursiveAltInfo = ((AltAST)node).leftRecursiveAltInfo; } public AltAST(Token t) { super(t); } diff --git a/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java b/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java index 730412cee..97d71d808 100644 --- a/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java @@ -29,6 +29,7 @@ package org.antlr.v4.tool.ast; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; import org.antlr.runtime.tree.Tree; @@ -41,8 +42,13 @@ public class RuleRefAST extends GrammarASTWithOptions implements RuleElementAST public RuleRefAST(int type) { super(type); } public RuleRefAST(int type, Token t) { super(type, t); } + /** Dup token too since we overwrite during LR rule transform */ @Override - public Tree dupNode() { return new TerminalAST(this); } + public Tree dupNode() { + RuleRefAST r = new RuleRefAST(this); + r.token = new CommonToken(r.token); + return r; + } @Override public Object visit(GrammarASTVisitor v) { return v.visit(this); } diff --git a/tool/test/org/antlr/v4/test/TestListeners.java b/tool/test/org/antlr/v4/test/TestListeners.java new file mode 100644 index 000000000..3d91b1fe7 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestListeners.java @@ -0,0 +1,149 @@ +package org.antlr.v4.test; + +import org.junit.Test; + +public class TestListeners extends BaseTest { + @Test public void testBasic() throws Exception { + String grammar = + "grammar T;\n" + + "@members {\n" + + "public static class LeafListener extends TBaseListener {\n" + + " public void visitTerminal(ParserRuleContext ctx, Token symbol) {\n" + + " System.out.println(symbol.getText());\n" + + " }\n" + + " }}\n" + + "s\n" + + "@init {setBuildParseTree(true);}\n" + + "@after {" + + " System.out.println($r.ctx.toStringTree(this));" + + " ParseTreeWalker walker = new ParseTreeWalker();\n" + + " walker.walk(new LeafListener(), $r.ctx);" + + "}\n" + + " : r=a ;\n" + + "a : INT INT" + + " | ID" + + " ;\n" + + "MULT: '*' ;\n" + + "ADD : '+' ;\n" + + "INT : [0-9]+ ;\n" + + "ID : [a-z]+ ;\n" + + "WS : [ \\t\\n]+ -> skip ;\n"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1 2", false); + String expecting = "(a 1 2)\n" + + "1\n" + + "2\n"; + assertEquals(expecting, result); + } + + @Test public void testTokenGetters() throws Exception { + String grammar = + "grammar T;\n" + + "@members {\n" + + "public static class LeafListener extends TBaseListener {\n" + + " public void exitA(TParser.AContext ctx) {\n" + + " if (ctx.getChildCount()==2) System.out.printf(\"%s %s %s\",ctx.INT(0).getText(),ctx.INT(1).getText(),ctx.INT());\n" + + " else System.out.println(ctx.ID());\n" + + " }\n" + + " }}\n" + + "s\n" + + "@init {setBuildParseTree(true);}\n" + + "@after {" + + " System.out.println($r.ctx.toStringTree(this));" + + " ParseTreeWalker walker = new ParseTreeWalker();\n" + + " walker.walk(new LeafListener(), $r.ctx);" + + "}\n" + + " : r=a ;\n" + + "a : INT INT" + + " | ID" + + " ;\n" + + "MULT: '*' ;\n" + + "ADD : '+' ;\n" + + "INT : [0-9]+ ;\n" + + "ID : [a-z]+ ;\n" + + "WS : [ \\t\\n]+ -> skip ;\n"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1 2", false); + String expecting = "(a 1 2)\n" + + "1 2 [[@0,0:0='1',<5>,1:0], [@1,2:2='2',<5>,1:2]]\n"; + assertEquals(expecting, result); + + result = execParser("T.g", grammar, "TParser", "TLexer", "s", "abc", false); + expecting = "(a abc)\n" + + "[@0,0:2='abc',<6>,1:0]\n"; + assertEquals(expecting, result); + } + + @Test public void testRuleGetters() throws Exception { + String grammar = + "grammar T;\n" + + "@members {\n" + + "public static class LeafListener extends TBaseListener {\n" + + " public void exitA(TParser.AContext ctx) {\n" + + " if (ctx.getChildCount()==2) {\n" + + " System.out.printf(\"%s %s %s\",ctx.b(0).start.getText(),\n" + + " ctx.b(1).start.getText(),ctx.b().get(0).start.getText());\n" + + " }\n" + + " else System.out.println(ctx.b(0).start.getText());\n" + + " }\n" + + " }}\n" + + "s\n" + + "@init {setBuildParseTree(true);}\n" + + "@after {" + + " System.out.println($r.ctx.toStringTree(this));" + + " ParseTreeWalker walker = new ParseTreeWalker();\n" + + " walker.walk(new LeafListener(), $r.ctx);" + + "}\n" + + " : r=a ;\n" + + "a : b b" + // forces list + " | b" + // a list still + " ;\n" + + "b : ID | INT ;\n" + + "MULT: '*' ;\n" + + "ADD : '+' ;\n" + + "INT : [0-9]+ ;\n" + + "ID : [a-z]+ ;\n" + + "WS : [ \\t\\n]+ -> skip ;\n"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1 2", false); + String expecting = "(a (b 1) (b 2))\n" + + "1 2 1\n"; + assertEquals(expecting, result); + + result = execParser("T.g", grammar, "TParser", "TLexer", "s", "abc", false); + expecting = "(a (b abc))\n" + + "abc\n"; + assertEquals(expecting, result); + } + + @Test public void testLR() throws Exception { + String grammar = + "grammar T;\n" + + "@members {\n" + + "public static class LeafListener extends TBaseListener {\n" + + " public void exitA(TParser.EContext ctx) {\n" + + " if (ctx.getChildCount()==3) {\n" + + " System.out.printf(\"%s %s %s\",ctx.e(0).start.getText(),\n" + + " ctx.e(1).start.getText(),ctx.e().get(0).start.getText());\n" + + " }\n" + + " else System.out.println(ctx.INT().getText());\n" + + " }\n" + + " }}\n" + + "s\n" + + "@init {setBuildParseTree(true);}\n" + + "@after {" + + " System.out.println($r.ctx.toStringTree(this));" + + " ParseTreeWalker walker = new ParseTreeWalker();\n" + + " walker.walk(new LeafListener(), $r.ctx);" + + "}\n" + + " : r=e ;\n" + + "e : e op='*' e\n" + + " | e op='+' e\n" + + " | INT\n" + + " ;\n" + + "MULT: '*' ;\n" + + "ADD : '+' ;\n" + + "INT : [0-9]+ ;\n" + + "WS : [ \\t\\n]+ -> skip ;\n"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", "1+2*3", false); + String expecting = "(e (e 1) + (e (e 2) * (e 3)))\n"; + assertEquals(expecting, result); + } +}