From 6304a562ea4a5205afe6c6273c6b77e5225460cf Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 08:45:19 -0600 Subject: [PATCH 01/11] Fix unit tests --- .../v4/test/TestATNLexerInterpreter.java | 12 ++--- .../antlr/v4/test/TestATNSerialization.java | 44 +++++++++++-------- .../antlr/v4/test/TestActionTranslation.java | 4 +- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java index 3e021d8f0..947d6a6c1 100644 --- a/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java @@ -2,7 +2,7 @@ package org.antlr.v4.test; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.LexerRecognitionExeption; +import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.Utils; @@ -72,7 +72,7 @@ public class TestATNLexerInterpreter extends BaseTest { " | 'xy' .\n" + // should not pursue '.' since xy already hit stop " ;\n"); checkLexerMatches(lg, "xy", "A, EOF"); - LexerRecognitionExeption e = checkLexerMatches(lg, "xyz", "A, EOF"); + RecognitionException e = checkLexerMatches(lg, "xyz", "A, EOF"); assertEquals("NoViableAltException('z')", e.toString()); } @@ -83,7 +83,7 @@ public class TestATNLexerInterpreter extends BaseTest { " | 'xy' . 'z'\n" + // will not pursue '.' since xy already hit stop (prior alt) " ;\n"); // checkLexerMatches(lg, "xy", "A, EOF"); - LexerRecognitionExeption e = checkLexerMatches(lg, "xyqz", "A, EOF"); + RecognitionException e = checkLexerMatches(lg, "xyqz", "A, EOF"); assertEquals("NoViableAltException('q')", e.toString()); } @@ -244,7 +244,7 @@ public class TestATNLexerInterpreter extends BaseTest { checkLexerMatches(lg, "a", expecting); } - protected LexerRecognitionExeption checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { + protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { ATN atn = createATN(lg); CharStream input = new ANTLRInputStream(inputString); ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE"); @@ -252,11 +252,11 @@ public class TestATNLexerInterpreter extends BaseTest { System.out.println(dot.getDOT(startState, true)); List tokenTypes = null; - LexerRecognitionExeption retException = null; + RecognitionException retException = null; try { tokenTypes = getTokenTypes(lg, atn, input, false); } - catch (LexerRecognitionExeption lre) { retException = lre; } + catch (RecognitionException lre) { retException = lre; } if ( retException!=null ) return retException; String result = Utils.join(tokenTypes.iterator(), ", "); diff --git a/tool/test/org/antlr/v4/test/TestATNSerialization.java b/tool/test/org/antlr/v4/test/TestATNSerialization.java index 0eac6f8d4..c3d2558e1 100644 --- a/tool/test/org/antlr/v4/test/TestATNSerialization.java +++ b/tool/test/org/antlr/v4/test/TestATNSerialization.java @@ -406,7 +406,7 @@ public class TestATNSerialization extends BaseTest { "5:PLUS_BLOCK_START 0\n" + "6:BLOCK_END 0\n" + "7:PLUS_LOOP_BACK 0\n" + - "8:LOOP_END 0 6\n" + + "8:LOOP_END 0 7\n" + "rule 0:1 3,-1\n" + "mode 0:0\n" + "0->1 EPSILON 0,0,0\n" + @@ -441,10 +441,12 @@ public class TestATNSerialization extends BaseTest { "6:RULE_STOP 2\n" + "7:BASIC 0\n" + "9:BASIC 0\n" + - "10:BASIC 1\n" + + "10:BASIC 0\n" + "11:BASIC 1\n" + - "12:BASIC 2\n" + - "14:BASIC 2\n" + + "12:BASIC 1\n" + + "13:BASIC 2\n" + + "15:BASIC 2\n" + + "16:BASIC 2\n" + "rule 0:1 3,0\n" + "rule 1:3 4,-1\n" + "rule 2:5 5,1\n" + @@ -453,14 +455,16 @@ public class TestATNSerialization extends BaseTest { "0->3 EPSILON 0,0,0\n" + "0->5 EPSILON 0,0,0\n" + "1->7 EPSILON 0,0,0\n" + - "3->10 EPSILON 0,0,0\n" + - "5->12 EPSILON 0,0,0\n" + + "3->11 EPSILON 0,0,0\n" + + "5->13 EPSILON 0,0,0\n" + "7->9 ATOM 97,0,0\n" + - "9->2 EPSILON 0,0,0\n" + - "10->11 ATOM 98,0,0\n" + - "11->4 EPSILON 0,0,0\n" + - "12->14 ATOM 99,0,0\n" + - "14->6 EPSILON 0,0,0\n" + + "9->10 ACTION 0,0,0\n" + + "10->2 EPSILON 0,0,0\n" + + "11->12 ATOM 98,0,0\n" + + "12->4 EPSILON 0,0,0\n" + + "13->15 ATOM 99,0,0\n" + + "15->16 ACTION 2,1,0\n" + + "16->6 EPSILON 0,0,0\n" + "0:0 1\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); @@ -560,13 +564,15 @@ public class TestATNSerialization extends BaseTest { "10:PLUS_BLOCK_START 0\n" + "11:BLOCK_END 0\n" + "12:PLUS_LOOP_BACK 0\n" + - "13:LOOP_END 0\n" + + "13:LOOP_END 0 12\n" + "14:BASIC 1\n" + "15:BASIC 1\n" + "16:BASIC 1\n" + "17:BASIC 1\n" + - "18:BASIC 2\n" + - "20:BASIC 2\n" + + "18:BASIC 1\n" + + "19:BASIC 2\n" + + "21:BASIC 2\n" + + "22:BASIC 2\n" + "rule 0:2 3,-1\n" + "rule 1:4 4,0\n" + "rule 2:6 5,1\n" + @@ -577,7 +583,7 @@ public class TestATNSerialization extends BaseTest { "1->6 EPSILON 0,0,0\n" + "2->10 EPSILON 0,0,0\n" + "4->14 EPSILON 0,0,0\n" + - "6->18 EPSILON 0,0,0\n" + + "6->19 EPSILON 0,0,0\n" + "8->11 RANGE 97,122,0\n" + "10->8 EPSILON 0,0,0\n" + "11->12 EPSILON 0,0,0\n" + @@ -587,9 +593,11 @@ public class TestATNSerialization extends BaseTest { "14->15 ATOM 42,0,0\n" + "15->16 ATOM 47,0,0\n" + "16->17 EPSILON 0,0,0\n" + - "17->5 EPSILON 0,0,0\n" + - "18->20 WILDCARD 0,0,0\n" + - "20->7 EPSILON 0,0,0\n" + + "17->18 ACTION 1,0,0\n" + + "18->5 EPSILON 0,0,0\n" + + "19->21 WILDCARD 0,0,0\n" + + "21->22 ACTION 2,1,0\n" + + "22->7 EPSILON 0,0,0\n" + "0:0 1\n" + "1:1 1\n" + "2:12 1\n"; diff --git a/tool/test/org/antlr/v4/test/TestActionTranslation.java b/tool/test/org/antlr/v4/test/TestActionTranslation.java index 9fecaad6e..778b01a1a 100644 --- a/tool/test/org/antlr/v4/test/TestActionTranslation.java +++ b/tool/test/org/antlr/v4/test/TestActionTranslation.java @@ -116,8 +116,8 @@ public class TestActionTranslation extends BaseTest { } @Test public void testRuleRefs() throws Exception { - String action = "$lab.start; $c.tree;"; - String expected = "(_localctx.lab!=null?((Token)_localctx.lab.start):null); (_localctx._rc!=null?((CommonAST)_localctx._rc.tree):null);"; + String action = "$lab.start;"; + String expected = "(_localctx.lab!=null?(_localctx.lab.start):null);"; testActions(attributeTemplate, "inline", action, expected); } From b686efb293e030ed88f0ddbb5d5dc0d3824e3045 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 12:42:48 -0600 Subject: [PATCH 02/11] Code cleanup in unit tests --- .../antlr/v4/test/TestATNParserPrediction.java | 5 +++-- .../antlr/v4/test/TestCommonTokenStream.java | 17 +++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 4e9ba3ead..417c239d4 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -33,6 +33,7 @@ import org.antlr.v4.Tool; import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.runtime.NoViableAltException; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; @@ -523,7 +524,7 @@ public class TestATNParserPrediction extends BaseTest { } public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName, - String inputString, ParserRuleContext ctx) + String inputString, ParserRuleContext ctx) { Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); @@ -541,7 +542,7 @@ public class TestATNParserPrediction extends BaseTest { // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b")))); // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); - ParserATNSimulator interp = new ParserATNSimulator(atn); + ParserATNSimulator interp = new ParserATNSimulator(atn); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); TokenStream input = new IntTokenStream(types); diff --git a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java index 636ca98dc..dd42e2bc7 100644 --- a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java @@ -178,24 +178,17 @@ public class TestCommonTokenStream extends BaseTest { new TokenSource() { int i = 0; WritableToken[] tokens = { - new CommonToken(1," "), + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1,"x"), - new CommonToken(1," "), + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1,"="), new CommonToken(1,"34"), - new CommonToken(1," "), - new CommonToken(1," "), + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1,";"), - new CommonToken(1,"\n"), + new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}}, new CommonToken(Token.EOF,"") }; - { - tokens[0].setChannel(Lexer.HIDDEN); - tokens[2].setChannel(Lexer.HIDDEN); - tokens[5].setChannel(Lexer.HIDDEN); - tokens[6].setChannel(Lexer.HIDDEN); - tokens[8].setChannel(Lexer.HIDDEN); - } public Token nextToken() { return tokens[i++]; } From bf5df307081c18805ec49f7007aff04de8a19334 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 12:45:46 -0600 Subject: [PATCH 03/11] Remove unnecessary initializations, use isEmpty() instead of size()==0 --- .../org/antlr/v4/runtime/atn/ParserATNSimulator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 51b31e458..2bd500ce0 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -367,7 +367,7 @@ public class ParserATNSimulator extends ATNSimulator { // if no edge, pop over to ATN interpreter, update DFA and return if ( s.edges == null || t >= s.edges.length || t < -1 || s.edges[t+1] == null ) { if ( dfa_debug && t>=0 ) System.out.println("no edge for "+parser.getTokenNames()[t]); - int alt = -1; + int alt; if ( dfa_debug ) { System.out.println("ATN exec upon "+ parser.getInputString(startIndex) + @@ -480,8 +480,8 @@ public class ParserATNSimulator extends ATNSimulator { ATN_failover++; ATNConfigSet previous = s0.configset; - DFAState D = null; - ATNConfigSet fullCtxSet = null; + DFAState D; + ATNConfigSet fullCtxSet; if ( debug ) System.out.println("s0 = "+s0); @@ -631,7 +631,7 @@ public class ParserATNSimulator extends ATNSimulator { if ( reach.hasSemanticContext ) { SemanticContext[] altToPred = getPredsForAmbigAlts(reach.conflictingAlts, reach, nalts); // altToPred[uniqueAlt] is now our validating predicate (if any) - List predPredictions = null; + List predPredictions; if ( altToPred!=null ) { // we have a validating predicate; test it predPredictions = getPredicatePredictions(reach.conflictingAlts, altToPred); @@ -793,7 +793,7 @@ public class ParserATNSimulator extends ATNSimulator { pairs.add(new DFAState.PredPrediction(pred, i)); } } - if ( pairs.size()==0 ) pairs = null; + if ( pairs.isEmpty() ) pairs = null; else if ( firstUnpredicated!=ATN.INVALID_ALT_NUMBER ) { // add default prediction if we found null predicate pairs.add(new DFAState.PredPrediction(null, firstUnpredicated)); From d589011ad82968ff6c6b8e70e4eb090eeddfd257 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 13:00:27 -0600 Subject: [PATCH 04/11] Small fixes to the Java grammars used in the performance test --- tool/test/org/antlr/v4/test/Java-LR.g | 47 ++++++++++++++++----------- tool/test/org/antlr/v4/test/Java.g | 4 ++- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/tool/test/org/antlr/v4/test/Java-LR.g b/tool/test/org/antlr/v4/test/Java-LR.g index 97e5facbf..75fcb36ac 100644 --- a/tool/test/org/antlr/v4/test/Java-LR.g +++ b/tool/test/org/antlr/v4/test/Java-LR.g @@ -169,7 +169,6 @@ * Character.isJavaIdentifierPart(int) returns true." */ grammar Java; -options {backtrack=true; memoize=true;} @lexer::members { protected boolean enumIsKeyword = true; @@ -184,7 +183,9 @@ compilationUnit ( packageDeclaration importDeclaration* typeDeclaration* | classOrInterfaceDeclaration typeDeclaration* ) + EOF | packageDeclaration? importDeclaration* typeDeclaration* + EOF ; packageDeclaration @@ -499,7 +500,7 @@ constructorBody explicitConstructorInvocation : nonWildcardTypeArguments? ('this' | 'super') arguments ';' - | expression '.' nonWildcardTypeArguments? 'super' arguments ';' + | primary '.' nonWildcardTypeArguments? 'super' arguments ';' ; @@ -711,17 +712,12 @@ constantExpression ; expression - : parExpression - | 'this' - | 'super' - | literal - | Identifier + : primary | expression '.' Identifier - | expression '.' 'class' // should be type.class but causes backtracking | expression '.' 'this' | expression '.' 'super' '(' expressionList? ')' - | expression '.' 'super' '.' Identifier arguments? | expression '.' 'new' Identifier '(' expressionList? ')' + | expression '.' 'super' '.' Identifier arguments? | expression '.' explicitGenericInvocation | 'new' creator | expression '[' expression ']' @@ -754,8 +750,20 @@ expression | '>' '>' '=' | '>' '>' '>' '=' | '<' '<' '=' - | '%=') expression - ; + | '%=' + ) + expression + ; + +primary + : '(' expression ')' + | 'this' + | 'super' + | literal + | Identifier + | type '.' 'class' + | 'void' '.' 'class' + ; creator : nonWildcardTypeArguments createdName classCreatorRest @@ -827,15 +835,16 @@ FloatingPointLiteral | ('0'..'9')+ Exponent FloatTypeSuffix? | ('0'..'9')+ FloatTypeSuffix | '0' ('x'|'X') - ( HexDigit+ '.' HexDigit* Exponent? FloatTypeSuffix? - | '.' HexDigit+ Exponent? FloatTypeSuffix? - | HexDigit+ Exponent FloatTypeSuffix? - | HexDigit+ FloatTypeSuffix + ( HexDigit+ ('.' HexDigit*)? HexExponent FloatTypeSuffix? + | '.' HexDigit+ HexExponent FloatTypeSuffix? ) ; fragment -Exponent : ('e'|'E'|'p'|'P') ('+'|'-')? ('0'..'9')+ ; +Exponent : ('e'|'E') ('+'|'-')? ('0'..'9')+ ; + +fragment +HexExponent : ('p'|'P') ('+'|'-')? ('0'..'9')+ ; fragment FloatTypeSuffix : ('f'|'F'|'d'|'D') ; @@ -917,13 +926,13 @@ JavaIDDigit '\u1040'..'\u1049' ; -WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} +WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;}//-> channel(HIDDEN) ; COMMENT - : '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} + : '/*' .* '*/' {$channel=HIDDEN;}//-> channel(HIDDEN) ; LINE_COMMENT - : '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;} + : '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;}//-> channel(HIDDEN) ; diff --git a/tool/test/org/antlr/v4/test/Java.g b/tool/test/org/antlr/v4/test/Java.g index e5b174fda..dcfb75673 100644 --- a/tool/test/org/antlr/v4/test/Java.g +++ b/tool/test/org/antlr/v4/test/Java.g @@ -181,7 +181,9 @@ compilationUnit ( packageDeclaration importDeclaration* typeDeclaration* | classOrInterfaceDeclaration typeDeclaration* ) + EOF | packageDeclaration? importDeclaration* typeDeclaration* + EOF ; packageDeclaration @@ -1021,7 +1023,7 @@ WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} ; COMMENT - : '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} + : '/*' .* '*/' {$channel=HIDDEN;} ; LINE_COMMENT From 7fb73a38493492a27f4ba8c5924c01e48a6c6d1e Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 13:29:14 -0600 Subject: [PATCH 05/11] Construct DFA instances with the decision number --- .../org/antlr/v4/runtime/atn/ParserATNSimulator.java | 3 +-- runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java | 12 ++++++++++-- .../org/antlr/v4/test/TestATNParserPrediction.java | 3 +-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 2bd500ce0..36b881f1b 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -268,8 +268,7 @@ public class ParserATNSimulator extends ATNSimulator { DFA dfa = decisionToDFA[decision]; if ( dfa==null || dfa.s0==null ) { DecisionState startState = atn.decisionToState.get(decision); - decisionToDFA[decision] = dfa = new DFA(startState); - dfa.decision = decision; + decisionToDFA[decision] = dfa = new DFA(startState, decision); return predictATN(dfa, input, outerContext); } else { diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java index b3b41bd70..edaf8b93e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java @@ -43,7 +43,8 @@ public class DFA { public final Map states = new LinkedHashMap(); @Nullable public DFAState s0; - public int decision; + + public final int decision; /** From which ATN state did we create this DFA? */ @NotNull @@ -54,7 +55,14 @@ public class DFA { */ // public OrderedHashSet conflictSet; - public DFA(@NotNull DecisionState atnStartState) { this.atnStartState = atnStartState; } + public DFA(@NotNull DecisionState atnStartState) { + this(atnStartState, 0); + } + + public DFA(@NotNull DecisionState atnStartState, int decision) { + this.atnStartState = atnStartState; + this.decision = decision; + } /** Find the path in DFA from s0 to s, returning list of states encountered (inclusively) */ // public List getPathToState(DFAState finalState, TokenStream input, int start, int stop) { diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 417c239d4..4393a5a1b 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -505,8 +505,7 @@ public class TestATNParserPrediction extends BaseTest { TokenStream input = new IntTokenStream(types); ParserInterpreter interp = new ParserInterpreter(g, input); DecisionState startState = atn.decisionToState.get(decision); - DFA dfa = new DFA(startState); - dfa.decision = decision; + DFA dfa = new DFA(startState, decision); int alt = interp.predictATN(dfa, input, ParserRuleContext.EMPTY, false); System.out.println(dot.getDOT(dfa, false)); From 702b91fab746b2001ef5c8d9762ae817e8c80812 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 13:30:44 -0600 Subject: [PATCH 06/11] Ignore TestPerformance by default since it fails without special configuration --- tool/test/org/antlr/v4/test/TestPerformance.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java index 98622ed83..f2d26daf8 100644 --- a/tool/test/org/antlr/v4/test/TestPerformance.java +++ b/tool/test/org/antlr/v4/test/TestPerformance.java @@ -6,6 +6,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.io.*; @@ -75,7 +76,7 @@ public class TestPerformance extends BaseTest { private int tokenCount; @Test -// @Ignore + @Ignore public void compileJdk() throws IOException { compileParser(USE_LR_GRAMMAR); JavaParserFactory factory = getParserFactory(); From ddf946b067247a3040d78fe3cf535b4f6418e380 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 15:33:03 -0600 Subject: [PATCH 07/11] Token.EOF instead of -1, add to generic type usage, make getUniqueAlt an instance method (need to override it for some apps), @NotNull annotations --- .../src/org/antlr/v4/runtime/atn/ParserATNSimulator.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 36b881f1b..8b1143ec7 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -1200,7 +1200,7 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull public String getTokenName(int t) { - if ( t==-1 ) return "EOF"; + if ( t==Token.EOF ) return "EOF"; if ( parser!=null && parser.getTokenNames()!=null ) { String[] tokensNames = parser.getTokenNames(); if ( t>=tokensNames.length ) { @@ -1240,7 +1240,7 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull public NoViableAltException noViableAlt(@NotNull SymbolStream input, - @NotNull ParserRuleContext outerContext, + @NotNull ParserRuleContext outerContext, @NotNull ATNConfigSet configs, int startIndex) { @@ -1250,7 +1250,7 @@ public class ParserATNSimulator extends ATNSimulator { configs, outerContext); } - public static int getUniqueAlt(@NotNull Collection configs) { + public int getUniqueAlt(@NotNull Collection configs) { int alt = ATN.INVALID_ALT_NUMBER; for (ATNConfig c : configs) { if ( alt == ATN.INVALID_ALT_NUMBER ) { @@ -1275,6 +1275,7 @@ public class ParserATNSimulator extends ATNSimulator { return false; } + @NotNull protected DFAState addDFAEdge(@NotNull DFA dfa, @NotNull ATNConfigSet p, int t, @@ -1297,7 +1298,7 @@ public class ParserATNSimulator extends ATNSimulator { } /** See comment on LexerInterpreter.addDFAState. */ - @Nullable + @NotNull protected DFAState addDFAState(@NotNull DFA dfa, @NotNull ATNConfigSet configs) { DFAState proposed = new DFAState(configs); DFAState existing = dfa.states.get(proposed); From 467797785f30dd44041162334344cbd1cf60394e Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 15:37:56 -0600 Subject: [PATCH 08/11] Fix generics in ParserATNSimulator --- .../v4/runtime/atn/ParserATNSimulator.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 8b1143ec7..e63982fea 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -225,7 +225,7 @@ import java.util.*; * when closure operations fall off the end of the rule that * holds the decision were evaluating */ -public class ParserATNSimulator extends ATNSimulator { +public class ParserATNSimulator extends ATNSimulator { public static boolean debug = false; public static boolean dfa_debug = false; public static boolean retry_debug = false; @@ -261,7 +261,7 @@ public class ParserATNSimulator extends ATNSimulator { public void reset() { } - public int adaptivePredict(@NotNull SymbolStream input, int decision, + public int adaptivePredict(@NotNull SymbolStream input, int decision, @Nullable ParserRuleContext outerContext) { predict_calls++; @@ -287,7 +287,7 @@ public class ParserATNSimulator extends ATNSimulator { } } - public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream input, + public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream input, @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; @@ -319,7 +319,7 @@ public class ParserATNSimulator extends ATNSimulator { } public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0, - @NotNull SymbolStream input, int startIndex, + @NotNull SymbolStream input, int startIndex, @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; @@ -472,7 +472,7 @@ public class ParserATNSimulator extends ATNSimulator { */ public int execATN(@NotNull DFA dfa, @NotNull DFAState s0, - @NotNull SymbolStream input, int startIndex, + @NotNull SymbolStream input, int startIndex, ParserRuleContext outerContext) { if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input)); @@ -593,7 +593,7 @@ public class ParserATNSimulator extends ATNSimulator { public ATNConfigSet execATNWithFullContext(DFA dfa, DFAState D, // how far we got before failing over @NotNull ATNConfigSet s0, - @NotNull SymbolStream input, int startIndex, + @NotNull SymbolStream input, int startIndex, ParserRuleContext outerContext, int nalts, boolean greedy) @@ -1214,7 +1214,7 @@ public class ParserATNSimulator extends ATNSimulator { return String.valueOf(t); } - public String getLookaheadName(SymbolStream input) { + public String getLookaheadName(SymbolStream input) { return getTokenName(input.LA(1)); } @@ -1239,14 +1239,14 @@ public class ParserATNSimulator extends ATNSimulator { } @NotNull - public NoViableAltException noViableAlt(@NotNull SymbolStream input, + public NoViableAltException noViableAlt(@NotNull SymbolStream input, @NotNull ParserRuleContext outerContext, @NotNull ATNConfigSet configs, int startIndex) { return new NoViableAltException(parser, input, - (Token)input.get(startIndex), - (Token)input.LT(1), + input.get(startIndex), + input.LT(1), configs, outerContext); } From 99ce3cba5cfe2ecaa81244e59400fa28f2104be3 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Feb 2012 19:47:56 -0600 Subject: [PATCH 09/11] Fix ParserATNSimulator.closure not tracking proper context for context-sensitive predicates during full-context parsing --- .../v4/runtime/atn/ParserATNSimulator.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index e63982fea..d4328e4dc 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -844,6 +844,17 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull Set closureBusy, boolean collectPredicates, boolean greedy, boolean loopsSimulateTailRecursion) + { + final int initialDepth = 0; + closure(config, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, initialDepth); + } + + protected void closure(@NotNull ATNConfig config, + @NotNull ATNConfigSet configs, + @NotNull Set closureBusy, + boolean collectPredicates, + boolean greedy, boolean loopsSimulateTailRecursion, + int depth) { if ( debug ) System.out.println("closure("+config.toString(parser,true)+")"); @@ -868,7 +879,7 @@ public class ParserATNSimulator extends ATNSimulator { // gotten that context AFTER having falling off a rule. // Make sure we track that we are now out of context. c.reachesIntoOuterContext = config.reachesIntoOuterContext; - closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion); + closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, depth - 1); return; } else { @@ -912,8 +923,9 @@ public class ParserATNSimulator extends ATNSimulator { Transition t = p.transition(i); boolean continueCollecting = !(t instanceof ActionTransition) && collectPredicates; - ATNConfig c = getEpsilonTarget(config, t, continueCollecting); + ATNConfig c = getEpsilonTarget(config, t, continueCollecting, depth == 0); if ( c!=null ) { + int newDepth = depth; if ( config.state instanceof RuleStopState ) { // target fell off end of rule; mark resulting c as having dipped into outer context // We can't get here if incoming config was rule stop and we had context @@ -922,9 +934,16 @@ public class ParserATNSimulator extends ATNSimulator { // preds if this is > 0. c.reachesIntoOuterContext++; configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method + newDepth--; if ( debug ) System.out.println("dips into outer ctx: "+c); } - closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion); + else if (t instanceof RuleTransition) { + if (newDepth >= 0) { + newDepth++; + } + } + + closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion, newDepth); } } } @@ -936,12 +955,12 @@ public class ParserATNSimulator extends ATNSimulator { } @Nullable - public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates) { + public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates, boolean inContext) { if ( t instanceof RuleTransition ) { return ruleTransition(config, t); } else if ( t instanceof PredicateTransition ) { - return predTransition(config, (PredicateTransition)t, collectPredicates); + return predTransition(config, (PredicateTransition)t, collectPredicates, inContext); } else if ( t instanceof ActionTransition ) { return actionTransition(config, (ActionTransition)t); @@ -961,7 +980,8 @@ public class ParserATNSimulator extends ATNSimulator { @Nullable public ATNConfig predTransition(@NotNull ATNConfig config, @NotNull PredicateTransition pt, - boolean collectPredicates) + boolean collectPredicates, + boolean inContext) { if ( debug ) { System.out.println("PRED (collectPredicates="+collectPredicates+") "+ @@ -972,12 +992,6 @@ public class ParserATNSimulator extends ATNSimulator { parser.getRuleInvocationStack()); } } - // We know the correct context in exactly one spot: in the original - // rule that invokes the ATN simulation. We know we are in this rule - // when the context stack is empty and we've not dipped into - // the outer context. - boolean inContext = - config.context==ParserRuleContext.EMPTY && config.reachesIntoOuterContext==0; ATNConfig c; if ( collectPredicates && From 30c58bf8ebbd1b98f42f84888346249c4332d84b Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 10 Feb 2012 00:51:23 -0600 Subject: [PATCH 10/11] Specify type parameter on raw types --- .../org/antlr/v4/runtime/atn/ParserATNSimulator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index d4328e4dc..eb02799ae 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -262,7 +262,7 @@ public class ParserATNSimulator extends ATNSimulator { } public int adaptivePredict(@NotNull SymbolStream input, int decision, - @Nullable ParserRuleContext outerContext) + @Nullable ParserRuleContext outerContext) { predict_calls++; DFA dfa = decisionToDFA[decision]; @@ -288,7 +288,7 @@ public class ParserATNSimulator extends ATNSimulator { } public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream input, - @Nullable ParserRuleContext outerContext) + @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( debug ) System.out.println("ATN decision "+dfa.decision+ @@ -320,7 +320,7 @@ public class ParserATNSimulator extends ATNSimulator { public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0, @NotNull SymbolStream input, int startIndex, - @Nullable ParserRuleContext outerContext) + @Nullable ParserRuleContext outerContext) { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ @@ -473,7 +473,7 @@ public class ParserATNSimulator extends ATNSimulator { */ public int execATN(@NotNull DFA dfa, @NotNull DFAState s0, @NotNull SymbolStream input, int startIndex, - ParserRuleContext outerContext) + ParserRuleContext outerContext) { if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input)); ATN_failover++; @@ -594,7 +594,7 @@ public class ParserATNSimulator extends ATNSimulator { DFAState D, // how far we got before failing over @NotNull ATNConfigSet s0, @NotNull SymbolStream input, int startIndex, - ParserRuleContext outerContext, + ParserRuleContext outerContext, int nalts, boolean greedy) { @@ -806,7 +806,7 @@ public class ParserATNSimulator extends ATNSimulator { * prediction for disambiguating predicates. */ public int evalSemanticContext(List predPredictions, - ParserRuleContext outerContext) + ParserRuleContext outerContext) { int predictedAlt = ATN.INVALID_ALT_NUMBER; // List predPredictions = D.predicates; From cd3adb140d7308082171e0640a15e312caee84be Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 10 Feb 2012 00:52:24 -0600 Subject: [PATCH 11/11] Fix ParserATNSimulator.execDFA adding incorrect error edges in the DFA --- .../v4/runtime/atn/ParserATNSimulator.java | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index eb02799ae..e7d9cec0c 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -372,31 +372,26 @@ public class ParserATNSimulator extends ATNSimulator { parser.getInputString(startIndex) + " at DFA state "+s.stateNumber); } - try { - alt = execATN(dfa, s, input, startIndex, outerContext); - // this adds edge even if next state is accept for - // same alt; e.g., s0-A->:s1=>2-B->:s2=>2 - // TODO: This next stuff kills edge, but extra states remain. :( - if ( s.isAcceptState && alt!=-1 ) { - DFAState d = s.edges[input.LA(1)+1]; - if ( d.isAcceptState && d.prediction==s.prediction ) { - // we can carve it out. - s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error - } + + alt = execATN(dfa, s, input, startIndex, outerContext); + // this adds edge even if next state is accept for + // same alt; e.g., s0-A->:s1=>2-B->:s2=>2 + // TODO: This next stuff kills edge, but extra states remain. :( + if ( s.isAcceptState && alt!=-1 ) { + DFAState d = s.edges[input.LA(1)+1]; + if ( d.isAcceptState && d.prediction==s.prediction ) { + // we can carve it out. + s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error } - if ( dfa_debug ) { - System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames())); - //dump(dfa); - } - // action already executed - if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ - " predicts "+alt); - return alt; // we've updated DFA, exec'd action, and have our deepest answer } - catch (NoViableAltException nvae) { - addDFAEdge(s, t, ERROR); - throw nvae; + if ( dfa_debug ) { + System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames())); + //dump(dfa); } + // action already executed + if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ + " predicts "+alt); + return alt; // we've updated DFA, exec'd action, and have our deepest answer } DFAState target = s.edges[t+1]; if ( target == ERROR ) {