forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

broke out fullctx tests, some fixes. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9636]
This commit is contained in:
parrt 2011-12-16 09:43:29 -08:00
parent 1e8548297e
commit 3d133e9417
4 changed files with 253 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java
View File

@ -149,9 +149,9 @@ import java.util.*;
* no context simulation.
*/
public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
public static boolean debug = false;
public static boolean dfa_debug = false;
public static boolean retry_debug = false;
public static boolean debug = true;
public static boolean dfa_debug = true;
public static boolean retry_debug = true;
public static int ATN_failover = 0;
public static int predict_calls = 0;
@ -453,8 +453,13 @@ public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
}
else {
if ( debug ) System.out.println("RETRY with outerContext="+outerContext);
int old_k = input.index();
ATNConfigSet s0_closure = computeStartState(dfa.atnStartState, outerContext, greedy);
fullCtxSet = execATNWithFullContext(s0_closure, input, startIndex, greedy);
if ( old_k != input.index() ) {
parser.notifyListeners("used diff amount of k; old="+(old_k-startIndex+1)+
", new="+(input.index()-startIndex+1));
}
if ( fullCtxSet.conflictingAlts!=null ) {
reportAmbiguity(dfa, startIndex, input.index(), fullCtxSet.conflictingAlts, fullCtxSet);
predictedAlt = fullCtxSet.conflictingAlts.getMinElement();
@ -462,6 +467,11 @@ public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
}
else {
D.isCtxSensitive = true;
predictedAlt = fullCtxSet.uniqueAlt;
if ( D.ctxToPrediction==null ) {
D.ctxToPrediction = new LinkedHashMap<RuleContext, Integer>();
}
D.ctxToPrediction.put(outerContext, predictedAlt);
reportContextSensitivity(dfa, fullCtxSet, startIndex, input.index());
}
}

16
tool/playground/T.g
View File

@ -1,10 +1,8 @@
grammar T;
s : ( .* (tag {System.out.println($tag.text);} |header) )* EOF;
tag : '<' .+ '>' ;
header : 'x' 'y' ;
EQ : '=' ;
COMMA : ',' ;
ID : 'a'..'z'+ ;
STR : '"' (options {greedy=false;}:.)* '"' ;
INT : '0'..'9'+;
WS : (' '|'\n') {skip();} ;
s@after {dumpDFA();}
: '{' stat* '}' ;
stat: 'if' ID 'then' stat ('else' stat)?
| 'break'
| 'return'
;ID : 'a'..'z'+ ;
WS : (' '|'\t'|'\n')+ {skip();} ;

220
tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View File

@ -34,7 +34,10 @@ import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
@ -286,221 +289,6 @@ public class TestATNParserPrediction extends BaseTest {
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testFullLLContextParse() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
// AB predicted in both alts of e but in diff contexts.
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A;B;C;}\n" +
"a : e B ;\n" +
"b : e A B ;\n" +
"e : A | ;\n"); // TODO: try with three alts
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
semanticProcess(lg);
g.importVocab(lg);
semanticProcess(g);
ParserATNFactory f = new ParserATNFactory(g);
ATN atn = f.createATN();
RuleStartState aStart = atn.ruleToStartState[g.getRule("a").index];
RuleStartState bStart = atn.ruleToStartState[g.getRule("b").index];
RuleStartState eStart = atn.ruleToStartState[g.getRule("e").index];
ATNState a_e_invoke = aStart.transition(0).target; //
ATNState b_e_invoke = bStart.transition(0).target; //
ParserRuleContext a_ctx = new ParserRuleContext(null, -1, a_e_invoke.stateNumber);
ParserRuleContext b_ctx = new ParserRuleContext(null, -1, b_e_invoke.stateNumber);
ParserRuleContext a_e_ctx = new ParserRuleContext(a_ctx, a_e_invoke.stateNumber, bStart.stateNumber);
ParserRuleContext b_e_ctx = new ParserRuleContext(b_ctx, b_e_invoke.stateNumber, bStart.stateNumber);
List<Integer> types = getTokenTypesViaATN("ab", lexInterp);
System.out.println(types);
TokenStream input = new IntTokenStream(types);
// ParserATNSimulator interp = new ParserATNSimulator(atn);
ParserInterpreter interp = new ParserInterpreter(g, input);
// interp.setContextSensitive(true); the default
int alt = interp.adaptivePredict(input, 0, b_e_ctx);
assertEquals(alt, 2);
DFA dfa = interp.getATNSimulator().decisionToDFA[0];
String expecting =
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3@{[10]=2}\n";
assertEquals(expecting, dfa.toString(g.getTokenDisplayNames()));
alt = interp.adaptivePredict(input, 0, b_e_ctx); // cached
assertEquals(alt, 2);
expecting =
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3@{[10]=2}\n";
assertEquals(expecting, dfa.toString(g.getTokenDisplayNames()));
alt = interp.adaptivePredict(input, 0, a_e_ctx); // forces new context-sens ATN match
assertEquals(alt, 1);
expecting =
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3@{[10]=2, [6]=1}\n";
assertEquals(expecting, dfa.toString(g.getTokenDisplayNames()));
alt = interp.adaptivePredict(input, 0, b_e_ctx); // cached
assertEquals(alt, 2);
expecting =
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3@{[10]=2, [6]=1}\n";
assertEquals(expecting, dfa.toString(g.getTokenDisplayNames()));
alt = interp.adaptivePredict(input, 0, a_e_ctx); // cached
assertEquals(alt, 1);
expecting =
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3@{[10]=2, [6]=1}\n";
assertEquals(expecting, dfa.toString(g.getTokenDisplayNames()));
types = getTokenTypesViaATN("b", lexInterp);
System.out.println(types);
input = new IntTokenStream(types);
alt = interp.adaptivePredict(input, 0, null); // ctx irrelevant
assertEquals(alt, 2);
expecting =
"s0-'a'->s1\n" +
"s0-'b'->:s4=>2\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3@{[10]=2, [6]=1}\n";
assertEquals(expecting, dfa.toString(g.getTokenDisplayNames()));
types = getTokenTypesViaATN("aab", lexInterp);
System.out.println(types);
input = new IntTokenStream(types);
alt = interp.adaptivePredict(input, 0, null);
assertEquals(alt, 1);
expecting =
"s0-'a'->s1\n" +
"s0-'b'->:s4=>2\n" +
"s1-'a'->:s5=>1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3@{[10]=2, [6]=1}\n";
assertEquals(expecting, dfa.toString(g.getTokenDisplayNames()));
}
@Test public void testFullContextIF_THEN_ELSEParse() {
String grammar =
"grammar T;\n"+
"s" +
"@after {dumpDFA();}\n" +
" : '{' stat* '}'" +
" ;\n" +
"stat: 'if' ID 'then' stat ('else' stat)?\n" +
" | 'break'\n" +
" | 'return'\n" +
" ;" +
"ID : 'a'..'z'+ ;\n"+
"WS : (' '|'\\t'|'\\n')+ {skip();} ;\n";
String input = "{ if x then break }";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
String expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'}'->:s1=>2\n";
assertEquals(expecting, result);
assertEquals(null, this.stderrDuringParse);
input =
"{ if x then if y then break else break }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[21 6]=1}\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:28 reportAmbiguity {1..2}:[1|1|[], 1|2|[]], input=else\n",
this.stderrDuringParse);
input = "{ if x then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1}\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n",
this.stderrDuringParse);
input = "{ if x then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1}\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n",
this.stderrDuringParse);
input =
"{ if x then break else return\n" +
"if x then if y then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1, [21 6]=1}\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n" +
"line 2:26 reportAmbiguity {1..2}:[1|1|[], 1|2|[]], input=else\n",
this.stderrDuringParse);
input =
"{ if x then break else return\n" +
"if x then if y then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1, [21 6]=1}\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity: [15|1|[25], 29|1|[25], 31|1|[25], 15|2|[25]|up=1, 29|2|[25]|up=1, 31|2|[25]|up=1], input=else\n" +
"line 2:26 reportAmbiguity {1..2}:[1|1|[], 1|2|[]], input=else\n",
this.stderrDuringParse);
}
@Test public void testRecursiveLeftPrefix() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +

229
tool/test/org/antlr/v4/test/TestFullContextParsing.java Normal file
View File

@ -0,0 +1,229 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test;
import org.junit.Test;
/*
cover these cases:
dead end
single alt
single alt + preds
conflict
conflict + preds
*/
public class TestFullContextParsing extends BaseTest {
@Test public void testAmbigYieldsNonCtxSensitiveDFA() {
String grammar =
"grammar T;\n"+
"s" +
"@after {dumpDFA();}\n" +
" : ID | ID {;} ;\n" +
"ID : 'a'..'z'+ ;\n"+
"WS : (' '|'\\t'|'\\n')+ {skip();} ;\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s",
"abc", true);
String expecting =
"Decision 0:\n" +
"s0-ID->:s1=>1\n"; // not ctx sensitive
assertEquals(expecting, result);
assertEquals("line 1:0 reportAmbiguity d=0: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=abc\n",
this.stderrDuringParse);
}
@Test public void testCtxSensitiveDFA() {
String grammar =
"grammar T;\n"+
"s @after {dumpDFA();}\n" +
" : '$' a | '@' b ;\n" +
"a : e ID ;\n" +
"b : e INT ID ;\n" +
"e : INT | ;\n" +
"ID : 'a'..'z'+ ;\n"+
"INT : '0'..'9'+ ;\n"+
"WS : (' '|'\\t'|'\\n')+ {skip();} ;\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s",
"$ 34 abc", true);
String expecting =
"Decision 1:\n" +
"s0-INT->s1\n" +
"s1-ID->:s2@{[18 10]=1}\n";
assertEquals(expecting, result);
assertEquals("line 1:2 reportContextSensitivity d=1: [(20,1,[10])],uniqueAlt=1, input=34\n",
this.stderrDuringParse);
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
"@ 34 abc", true);
expecting =
"Decision 1:\n" +
"s0-INT->s1\n" +
"s1-ID->:s2@{[22 14]=2}\n";
assertEquals(expecting, result);
assertEquals("line 1:5 reportContextSensitivity d=1: [(1,2,[])],uniqueAlt=2, input=34abc\n",
this.stderrDuringParse);
}
@Test public void testCtxSensitiveDFATwoDiffInput() {
String grammar =
"grammar T;\n"+
"s @after {dumpDFA();}\n" +
" : ('$' a | '@' b)+ ;\n" +
"a : e ID ;\n" +
"b : e INT ID ;\n" +
"e : INT | ;\n" +
"ID : 'a'..'z'+ ;\n"+
"INT : '0'..'9'+ ;\n"+
"WS : (' '|'\\t'|'\\n')+ {skip();} ;\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s",
"$ 34 abc @ 34 abc", true);
String expecting =
"Decision 1:\n" +
"s0-EOF->:s3=>2\n" +
"s0-'@'->:s2=>1\n" +
"s0-'$'->:s1=>1\n" +
"\n" +
"Decision 2:\n" +
"s0-INT->s1\n" +
"s1-ID->:s2@{[20 10]=1, [24 14]=2}\n";
assertEquals(expecting, result);
assertEquals("line 1:2 reportContextSensitivity d=2: [(22,1,[10])],uniqueAlt=1, input=34\n" +
"line 1:14 reportContextSensitivity d=2: [(8,2,[]), (12,2,[]), (1,2,[])],uniqueAlt=2, input=34abc\n",
this.stderrDuringParse);
}
@Test public void testFullContextIF_THEN_ELSEParse() {
String grammar =
"grammar T;\n"+
"s" +
"@after {dumpDFA();}\n" +
" : '{' stat* '}'" +
" ;\n" +
"stat: 'if' ID 'then' stat ('else' stat)?\n" +
" | 'break'\n" +
" | 'return'\n" +
" ;" +
"ID : 'a'..'z'+ ;\n"+
"WS : (' '|'\\t'|'\\n')+ {skip();} ;\n";
String input = "{ if x then break }";
String result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
String expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'}'->:s1=>2\n";
assertEquals(expecting, result);
assertEquals(null, this.stderrDuringParse);
input =
"{ if x then if y then break else break }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1=>1\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:39 reportAmbiguity d=1: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=elsebreak}\n",
this.stderrDuringParse);
input = "{ if x then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1}\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n",
this.stderrDuringParse);
input = "{ if x then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1}\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n",
this.stderrDuringParse);
input =
"{ if x then break else return\n" +
"if x then if y then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1, [21 6]=1}\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n" +
"line 2:38 reportAmbiguity d=1: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=elsereturn}\n",
this.stderrDuringParse);
input =
"{ if x then break else return\n" +
"if x then if y then break else return }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1@{[6]=1, [21 6]=1}\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:18 reportContextSensitivity d=1: [(15,1,[25 6]), (29,1,[25 6]), (31,1,[25 6])],uniqueAlt=1, input=else\n" +
"line 2:38 reportAmbiguity d=1: {1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input=elsereturn}\n",
this.stderrDuringParse);
}
}