From 3d133e94176c7b97a7eb1a3af851116d7169ec6f Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 16 Dec 2011 09:43:29 -0800 Subject: [PATCH] broke out fullctx tests, some fixes. [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9636] --- .../v4/runtime/atn/v2ParserATNSimulator.java | 16 +- tool/playground/T.g | 16 +- .../v4/test/TestATNParserPrediction.java | 220 +---------------- .../antlr/v4/test/TestFullContextParsing.java | 229 ++++++++++++++++++ 4 files changed, 253 insertions(+), 228 deletions(-) create mode 100644 tool/test/org/antlr/v4/test/TestFullContextParsing.java diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java index dfafcec2b..1788a7031 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java @@ -149,9 +149,9 @@ import java.util.*; * no context simulation. */ public class v2ParserATNSimulator extends ATNSimulator { - public static boolean debug = false; - public static boolean dfa_debug = false; - public static boolean retry_debug = false; + public static boolean debug = true; + public static boolean dfa_debug = true; + public static boolean retry_debug = true; public static int ATN_failover = 0; public static int predict_calls = 0; @@ -453,8 +453,13 @@ public class v2ParserATNSimulator extends ATNSimulator { } else { if ( debug ) System.out.println("RETRY with outerContext="+outerContext); + int old_k = input.index(); ATNConfigSet s0_closure = computeStartState(dfa.atnStartState, outerContext, greedy); fullCtxSet = execATNWithFullContext(s0_closure, input, startIndex, greedy); + if ( old_k != input.index() ) { + parser.notifyListeners("used diff amount of k; old="+(old_k-startIndex+1)+ + ", new="+(input.index()-startIndex+1)); + } if ( fullCtxSet.conflictingAlts!=null ) { reportAmbiguity(dfa, startIndex, input.index(), fullCtxSet.conflictingAlts, fullCtxSet); predictedAlt = fullCtxSet.conflictingAlts.getMinElement(); @@ -462,6 +467,11 @@ public class v2ParserATNSimulator extends ATNSimulator { } else { D.isCtxSensitive = true; + predictedAlt = fullCtxSet.uniqueAlt; + if ( D.ctxToPrediction==null ) { + D.ctxToPrediction = new LinkedHashMap(); + } + D.ctxToPrediction.put(outerContext, predictedAlt); reportContextSensitivity(dfa, fullCtxSet, startIndex, input.index()); } } diff --git a/tool/playground/T.g b/tool/playground/T.g index 7638d5d39..174e506da 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,10 +1,8 @@ grammar T; -s : ( .* (tag {System.out.println($tag.text);} |header) )* EOF; -tag : '<' .+ '>' ; -header : 'x' 'y' ; -EQ : '=' ; -COMMA : ',' ; -ID : 'a'..'z'+ ; -STR : '"' (options {greedy=false;}:.)* '"' ; -INT : '0'..'9'+; -WS : (' '|'\n') {skip();} ; +s@after {dumpDFA();} + : '{' stat* '}' ; +stat: 'if' ID 'then' stat ('else' stat)? + | 'break' + | 'return' + ;ID : 'a'..'z'+ ; +WS : (' '|'\t'|'\n')+ {skip();} ; diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 3b29c0762..cfa1b888b 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -34,7 +34,10 @@ import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.runtime.NoViableAltException; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNState; +import org.antlr.v4.runtime.atn.LexerATNSimulator; +import org.antlr.v4.runtime.atn.ParserATNSimulator; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.Grammar; @@ -286,221 +289,6 @@ public class TestATNParserPrediction extends BaseTest { checkDFAConstruction(lg, g, decision, inputs, dfa); } - @Test public void testFullLLContextParse() throws Exception { - LexerGrammar lg = new LexerGrammar( - "lexer grammar L;\n" + - "A : 'a' ;\n" + - "B : 'b' ;\n" + - "C : 'c' ;\n"); - // AB predicted in both alts of e but in diff contexts. - Grammar g = new Grammar( - "parser grammar T;\n"+ - "tokens {A;B;C;}\n" + - "a : e B ;\n" + - "b : e A B ;\n" + - "e : A | ;\n"); // TODO: try with three alts - - ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn); - - semanticProcess(lg); - g.importVocab(lg); - semanticProcess(g); - - ParserATNFactory f = new ParserATNFactory(g); - ATN atn = f.createATN(); - - RuleStartState aStart = atn.ruleToStartState[g.getRule("a").index]; - RuleStartState bStart = atn.ruleToStartState[g.getRule("b").index]; - RuleStartState eStart = atn.ruleToStartState[g.getRule("e").index]; - ATNState a_e_invoke = aStart.transition(0).target; // - ATNState b_e_invoke = bStart.transition(0).target; // - ParserRuleContext a_ctx = new ParserRuleContext(null, -1, a_e_invoke.stateNumber); - ParserRuleContext b_ctx = new ParserRuleContext(null, -1, b_e_invoke.stateNumber); - ParserRuleContext a_e_ctx = new ParserRuleContext(a_ctx, a_e_invoke.stateNumber, bStart.stateNumber); - ParserRuleContext b_e_ctx = new ParserRuleContext(b_ctx, b_e_invoke.stateNumber, bStart.stateNumber); - - List types = getTokenTypesViaATN("ab", lexInterp); - System.out.println(types); - TokenStream input = new IntTokenStream(types); - -// ParserATNSimulator interp = new ParserATNSimulator(atn); - ParserInterpreter interp = new ParserInterpreter(g, input); -// interp.setContextSensitive(true); the default - int alt = interp.adaptivePredict(input, 0, b_e_ctx); - assertEquals(alt, 2); - DFA dfa = interp.getATNSimulator().decisionToDFA[0]; - String expecting = - "s0-'a'->s1\n" + - "s1-'b'->s2\n" + - "s2-EOF->:s3@{[10]=2}\n"; - assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); - - alt = interp.adaptivePredict(input, 0, b_e_ctx); // cached - assertEquals(alt, 2); - expecting = - "s0-'a'->s1\n" + - "s1-'b'->s2\n" + - "s2-EOF->:s3@{[10]=2}\n"; - assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); - - alt = interp.adaptivePredict(input, 0, a_e_ctx); // forces new context-sens ATN match - assertEquals(alt, 1); - expecting = - "s0-'a'->s1\n" + - "s1-'b'->s2\n" + - "s2-EOF->:s3@{[10]=2, [6]=1}\n"; - assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); - - alt = interp.adaptivePredict(input, 0, b_e_ctx); // cached - assertEquals(alt, 2); - expecting = - "s0-'a'->s1\n" + - "s1-'b'->s2\n" + - "s2-EOF->:s3@{[10]=2, [6]=1}\n"; - assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); - - alt = interp.adaptivePredict(input, 0, a_e_ctx); // cached - assertEquals(alt, 1); - expecting = - "s0-'a'->s1\n" + - "s1-'b'->s2\n" + - "s2-EOF->:s3@{[10]=2, [6]=1}\n"; - assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); - - types = getTokenTypesViaATN("b", lexInterp); - System.out.println(types); - input = new IntTokenStream(types); - alt = interp.adaptivePredict(input, 0, null); // ctx irrelevant - assertEquals(alt, 2); - expecting = - "s0-'a'->s1\n" + - "s0-'b'->:s4=>2\n" + - "s1-'b'->s2\n" + - "s2-EOF->:s3@{[10]=2, [6]=1}\n"; - assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); - - types = getTokenTypesViaATN("aab", lexInterp); - System.out.println(types); - input = new IntTokenStream(types); - alt = interp.adaptivePredict(input, 0, null); - assertEquals(alt, 1); - expecting = - "s0-'a'->s1\n" + - "s0-'b'->:s4=>2\n" + - "s1-'a'->:s5=>1\n" + - "s1-'b'->s2\n" + - "s2-EOF->:s3@{[10]=2, [6]=1}\n"; - assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); - } - - @Test public void testFullContextIF_THEN_ELSEParse() { - String grammar = - "grammar T;\n"+ - "s" + - "@after {dumpDFA();}\n" + - " : '{' stat* '}'" + - " ;\n" + - "stat: 'if' ID 'then' stat ('else' stat)?\n" + - " | 'break'\n" + - " | 'return'\n" + - " ;" + - "ID : 'a'..'z'+ ;\n"+ - "WS : (' '|'\\t'|'\\n')+ {skip();} ;\n"; - String input = "{ if x then break }"; - String result = execParser("T.g", grammar, "TParser", "TLexer", "s", - input, true); - String expecting = - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'}'->:s2=>2\n" + - "\n" + - "Decision 1:\n" + - "s0-'}'->:s1=>2\n"; - assertEquals(expecting, result); - assertEquals(null, this.stderrDuringParse); - - input = - "{ if x then if y then break else break }"; - result = execParser("T.g", grammar, "TParser", "TLexer", "s", - input, true); - expecting = - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'}'->:s2=>2\n" + - "\n" + - "Decision 1:\n" + - "s0-'else'->:s1@{[21 6]=1}\n" + - "s0-'}'->:s2=>2\n"; - assertEquals(expecting, result); - assertEquals("line 1:28 reportAmbiguity {1..2}:[1|1|[], 1|2|[]], input=else\n", - this.stderrDuringParse); - - input = "{ if x then break else return }"; - result = execParser("T.g", grammar, "TParser", "TLexer", "s", - input, true); - expecting = - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'}'->:s2=>2\n" + - "\n" + - "Decision 1:\n" + - "s0-'else'->:s1@{[6]=1}\n"; - assertEquals(expecting, result); - assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n", - this.stderrDuringParse); - - input = "{ if x then break else return }"; - result = execParser("T.g", grammar, "TParser", "TLexer", "s", - input, true); - expecting = - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'}'->:s2=>2\n" + - "\n" + - "Decision 1:\n" + - "s0-'else'->:s1@{[6]=1}\n"; - assertEquals(expecting, result); - assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n", - this.stderrDuringParse); - - input = - "{ if x then break else return\n" + - "if x then if y then break else return }"; - result = execParser("T.g", grammar, "TParser", "TLexer", "s", - input, true); - expecting = - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'}'->:s2=>2\n" + - "\n" + - "Decision 1:\n" + - "s0-'else'->:s1@{[6]=1, [21 6]=1}\n" + - "s0-'}'->:s2=>2\n"; - assertEquals(expecting, result); - assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n" + - "line 2:26 reportAmbiguity {1..2}:[1|1|[], 1|2|[]], input=else\n", - this.stderrDuringParse); - - input = - "{ if x then break else return\n" + - "if x then if y then break else return }"; - result = execParser("T.g", grammar, "TParser", "TLexer", "s", - input, true); - expecting = - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'}'->:s2=>2\n" + - "\n" + - "Decision 1:\n" + - "s0-'else'->:s1@{[6]=1, [21 6]=1}\n" + - "s0-'}'->:s2=>2\n"; - assertEquals(expecting, result); - assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n" + - "line 2:26 reportAmbiguity {1..2}:[1|1|[], 1|2|[]], input=else\n", - this.stderrDuringParse); - } - @Test public void testRecursiveLeftPrefix() throws Exception { LexerGrammar lg = new LexerGrammar( "lexer grammar L;\n" + diff --git a/tool/test/org/antlr/v4/test/TestFullContextParsing.java b/tool/test/org/antlr/v4/test/TestFullContextParsing.java new file mode 100644 index 000000000..486408530 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestFullContextParsing.java @@ -0,0 +1,229 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.test; + +import org.junit.Test; + +/* + cover these cases: + dead end + single alt + single alt + preds + conflict + conflict + preds + + */ +public class TestFullContextParsing extends BaseTest { + @Test public void testAmbigYieldsNonCtxSensitiveDFA() { + String grammar = + "grammar T;\n"+ + "s" + + "@after {dumpDFA();}\n" + + " : ID | ID {;} ;\n" + + "ID : 'a'..'z'+ ;\n"+ + "WS : (' '|'\\t'|'\\n')+ {skip();} ;\n"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", + "abc", true); + String expecting = + "Decision 0:\n" + + "s0-ID->:s1=>1\n"; // not ctx sensitive + assertEquals(expecting, result); + assertEquals("line 1:0 reportAmbiguity d=0: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=abc\n", + this.stderrDuringParse); + } + + @Test public void testCtxSensitiveDFA() { + String grammar = + "grammar T;\n"+ + "s @after {dumpDFA();}\n" + + " : '$' a | '@' b ;\n" + + "a : e ID ;\n" + + "b : e INT ID ;\n" + + "e : INT | ;\n" + + "ID : 'a'..'z'+ ;\n"+ + "INT : '0'..'9'+ ;\n"+ + "WS : (' '|'\\t'|'\\n')+ {skip();} ;\n"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", + "$ 34 abc", true); + String expecting = + "Decision 1:\n" + + "s0-INT->s1\n" + + "s1-ID->:s2@{[18 10]=1}\n"; + assertEquals(expecting, result); + assertEquals("line 1:2 reportContextSensitivity d=1: [(20,1,[10])],uniqueAlt=1, input=34\n", + this.stderrDuringParse); + + result = execParser("T.g", grammar, "TParser", "TLexer", "s", + "@ 34 abc", true); + expecting = + "Decision 1:\n" + + "s0-INT->s1\n" + + "s1-ID->:s2@{[22 14]=2}\n"; + assertEquals(expecting, result); + assertEquals("line 1:5 reportContextSensitivity d=1: [(1,2,[])],uniqueAlt=2, input=34abc\n", + this.stderrDuringParse); + } + + @Test public void testCtxSensitiveDFATwoDiffInput() { + String grammar = + "grammar T;\n"+ + "s @after {dumpDFA();}\n" + + " : ('$' a | '@' b)+ ;\n" + + "a : e ID ;\n" + + "b : e INT ID ;\n" + + "e : INT | ;\n" + + "ID : 'a'..'z'+ ;\n"+ + "INT : '0'..'9'+ ;\n"+ + "WS : (' '|'\\t'|'\\n')+ {skip();} ;\n"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", + "$ 34 abc @ 34 abc", true); + String expecting = + "Decision 1:\n" + + "s0-EOF->:s3=>2\n" + + "s0-'@'->:s2=>1\n" + + "s0-'$'->:s1=>1\n" + + "\n" + + "Decision 2:\n" + + "s0-INT->s1\n" + + "s1-ID->:s2@{[20 10]=1, [24 14]=2}\n"; + assertEquals(expecting, result); + assertEquals("line 1:2 reportContextSensitivity d=2: [(22,1,[10])],uniqueAlt=1, input=34\n" + + "line 1:14 reportContextSensitivity d=2: [(8,2,[]), (12,2,[]), (1,2,[])],uniqueAlt=2, input=34abc\n", + this.stderrDuringParse); + } + + @Test public void testFullContextIF_THEN_ELSEParse() { + String grammar = + "grammar T;\n"+ + "s" + + "@after {dumpDFA();}\n" + + " : '{' stat* '}'" + + " ;\n" + + "stat: 'if' ID 'then' stat ('else' stat)?\n" + + " | 'break'\n" + + " | 'return'\n" + + " ;" + + "ID : 'a'..'z'+ ;\n"+ + "WS : (' '|'\\t'|'\\n')+ {skip();} ;\n"; + String input = "{ if x then break }"; + String result = execParser("T.g", grammar, "TParser", "TLexer", "s", + input, true); + String expecting = + "Decision 0:\n" + + "s0-'if'->:s1=>1\n" + + "s0-'}'->:s2=>2\n" + + "\n" + + "Decision 1:\n" + + "s0-'}'->:s1=>2\n"; + assertEquals(expecting, result); + assertEquals(null, this.stderrDuringParse); + + input = + "{ if x then if y then break else break }"; + result = execParser("T.g", grammar, "TParser", "TLexer", "s", + input, true); + expecting = + "Decision 0:\n" + + "s0-'if'->:s1=>1\n" + + "s0-'}'->:s2=>2\n" + + "\n" + + "Decision 1:\n" + + "s0-'else'->:s1=>1\n" + + "s0-'}'->:s2=>2\n"; + assertEquals(expecting, result); + assertEquals("line 1:39 reportAmbiguity d=1: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=elsebreak}\n", + this.stderrDuringParse); + + input = "{ if x then break else return }"; + result = execParser("T.g", grammar, "TParser", "TLexer", "s", + input, true); + expecting = + "Decision 0:\n" + + "s0-'if'->:s1=>1\n" + + "s0-'}'->:s2=>2\n" + + "\n" + + "Decision 1:\n" + + "s0-'else'->:s1@{[6]=1}\n"; + assertEquals(expecting, result); + assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n", + this.stderrDuringParse); + + input = "{ if x then break else return }"; + result = execParser("T.g", grammar, "TParser", "TLexer", "s", + input, true); + expecting = + "Decision 0:\n" + + "s0-'if'->:s1=>1\n" + + "s0-'}'->:s2=>2\n" + + "\n" + + "Decision 1:\n" + + "s0-'else'->:s1@{[6]=1}\n"; + assertEquals(expecting, result); + assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n", + this.stderrDuringParse); + + input = + "{ if x then break else return\n" + + "if x then if y then break else return }"; + result = execParser("T.g", grammar, "TParser", "TLexer", "s", + input, true); + expecting = + "Decision 0:\n" + + "s0-'if'->:s1=>1\n" + + "s0-'}'->:s2=>2\n" + + "\n" + + "Decision 1:\n" + + "s0-'else'->:s1@{[6]=1, [21 6]=1}\n" + + "s0-'}'->:s2=>2\n"; + assertEquals(expecting, result); + assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n" + + "line 2:38 reportAmbiguity d=1: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=elsereturn}\n", + this.stderrDuringParse); + + input = + "{ if x then break else return\n" + + "if x then if y then break else return }"; + result = execParser("T.g", grammar, "TParser", "TLexer", "s", + input, true); + expecting = + "Decision 0:\n" + + "s0-'if'->:s1=>1\n" + + "s0-'}'->:s2=>2\n" + + "\n" + + "Decision 1:\n" + + "s0-'else'->:s1@{[6]=1, [21 6]=1}\n" + + "s0-'}'->:s2=>2\n"; + assertEquals(expecting, result); + assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n" + + "line 2:38 reportAmbiguity d=1: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=elsereturn}\n", + this.stderrDuringParse); + } + +}