forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

got ambiguity reporting set right I think. fixed unit tests to force exact ambig detection.

This commit is contained in:
Terence Parr 2012-10-14 18:42:43 -07:00
parent e9c83c375f
commit 75a01636d0
8 changed files with 99 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -260,7 +260,7 @@ public class ParserATNSimulator extends ATNSimulator {
public static int predict_calls = 0;
public static int retry_with_context = 0;
public static int retry_with_context_indicates_no_conflict = 0;
public static int retry_with_context_predicts_same_as_alt = 0;
public static int retry_with_context_predicts_same_alt = 0;
public static int retry_with_context_from_dfa = 0;
@Nullable
@ -607,12 +607,11 @@ public class ParserATNSimulator extends ATNSimulator {
// }
// }
Collection<BitSet> altSubSets = PredictionMode.getConflictingAltSubsets(reach);
// System.out.println("SLL altsets: "+altSubSets);
int predictedAlt = PredictionMode.getUniqueAlt(altSubSets);
int predictedAlt = getUniqueAlt(reach);
if ( debug ) {
Collection<BitSet> altSubSets = PredictionMode.getConflictingAltSubsets(reach);
System.out.println("SLL altSubSets="+altSubSets+
", configs="+reach+
", predict="+predictedAlt+", allSubsetsConflict="+
@ -635,10 +634,9 @@ public class ParserATNSimulator extends ATNSimulator {
!D.configs.dipsIntoOuterContext ) // didn't fall out of rule
{
// SPECIAL CASE WHERE SLL KNOWS CONFLICT IS AMBIGUITY
if ( !D.configs.hasSemanticContext ) {
reportAmbiguity(dfa, D, startIndex, input.index(),
D.configs.conflictingAlts, D.configs);
}
// report even if preds
reportAmbiguity(dfa, D, startIndex, input.index(),
D.configs.conflictingAlts, D.configs);
}
// always stop at D
D.isAcceptState = true;
@ -737,10 +735,12 @@ public class ParserATNSimulator extends ATNSimulator {
System.out.println("execATNWithFullContext "+s0);
}
boolean fullCtx = true;
boolean foundExactAmbig = false;
ATNConfigSet reach = null;
ATNConfigSet previous = s0;
input.seek(startIndex);
int t = input.LA(1);
int predictedAlt;
while (true) { // while more work
// System.out.println("LL REACH "+getLookaheadName(input)+
// " from configs.size="+previous.size()+
@ -772,63 +772,81 @@ public class ParserATNSimulator extends ATNSimulator {
}
// System.out.println("altSubSets: "+altSubSets);
reach.uniqueAlt = PredictionMode.getUniqueAlt(altSubSets);
if ( reach.uniqueAlt!=ATN.INVALID_ALT_NUMBER ) break;
if ( mode == PredictionMode.LL_EXACT_AMBIG_DETECTION) {
if ( PredictionMode.allSubsetsConflict(altSubSets) &&
PredictionMode.allSubsetsEqual(altSubSets) )
{
reach.uniqueAlt = getUniqueAlt(reach);
// unique prediction?
if ( reach.uniqueAlt!=ATN.INVALID_ALT_NUMBER ) {
predictedAlt = reach.uniqueAlt;
break;
}
if ( mode != PredictionMode.LL_EXACT_AMBIG_DETECTION ) {
predictedAlt = PredictionMode.resolvesToJustOneViableAlt(altSubSets);
if ( predictedAlt != ATN.INVALID_ALT_NUMBER ) {
break;
}
}
else if ( PredictionMode.resolvesToJustOneViableAlt(altSubSets) ) {
break;
else {
// In exact ambiguity mode, we never try to terminate early.
// Just keeps scarfing until we know what the conflict is
if ( PredictionMode.allSubsetsConflict(altSubSets) &&
PredictionMode.allSubsetsEqual(altSubSets) )
{
foundExactAmbig = true;
predictedAlt = PredictionMode.getSingleViableAlt(altSubSets);
break;
}
// else there are multiple non-conflicting subsets or
// we're not sure what the ambiguity is yet.
// So, keep going.
}
previous = reach;
input.consume();
t = input.LA(1);
}
// If the configuration set uniquely predicts an alternative,
// without conflict, then we know that it's a full LL decision
// not SLL.
if ( reach.uniqueAlt != ATN.INVALID_ALT_NUMBER ) {
retry_with_context_indicates_no_conflict++;
reportContextSensitivity(dfa, reach, startIndex, input.index());
if ( reach.uniqueAlt == SLL_min_alt ) {
retry_with_context_predicts_same_as_alt++;
if ( predictedAlt == SLL_min_alt ) {
retry_with_context_predicts_same_alt++;
}
return reach.uniqueAlt;
return predictedAlt;
}
// We do not check predicates here because we have checked them
// on-the-fly when doing full context prediction.
// At this point, we know that we have conflicting configurations.
// But, that does not mean that there is no way forward without
// a conflict. It's possible to have nonconflicting alt subsets; e.g.,
//
// LL altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}]
//
// from
//
// [(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]),
// (13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])]
//
// In this case, (17,1,[5 $]) indicates there is some next sequence
// that would resolve this without conflict to alternative 1. Any
// other viable next sequence, however, is associated with a conflict.
// We stop looking for input because no amount of further lookahead
// will alter the fact that we should predict alternative 1.
// We just can't say for sure that there is an ambiguity without
// looking further.
/*
In non-exact ambiguity detection mode, we might actually be able to
detect an exact ambiguity, but I'm not going to spend the cycles
needed to check. We only emit ambiguity warnings in exact ambiguity
mode.
// if ( /* TODO: len(all subsets)>1 or input consistent with a subset with len=1 */ true ) {
// reportAmbiguity(dfa, D, startIndex, input.index(), getConflictingAlts(reach), reach);
// }
For example, we might know that we have conflicting configurations.
But, that does not mean that there is no way forward without a
conflict. It's possible to have nonconflicting alt subsets as in:
if ( mode == PredictionMode.LL_EXACT_AMBIG_DETECTION) {
LL altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}]
from
[(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]),
(13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])]
In this case, (17,1,[5 $]) indicates there is some next sequence that
would resolve this without conflict to alternative 1. Any other viable
next sequence, however, is associated with a conflict. We stop
looking for input because no amount of further lookahead will alter
the fact that we should predict alternative 1. We just can't say for
sure that there is an ambiguity without looking further.
*/
if ( foundExactAmbig ) {
reportAmbiguity(dfa, D, startIndex, input.index(), getConflictingAlts(reach), reach);
}
return getConflictingAlts(reach).nextSetBit(0);
return predictedAlt;
}
protected ATNConfigSet computeReachSet(ATNConfigSet closure, int t,

10
runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java
View File

@ -297,8 +297,8 @@ public enum PredictionMode {
going. We can only stop prediction when we need exact ambiguity
detection when the sets look like A={{1,2}} or {{1,2},{1,2}} etc...
*/
public static boolean resolvesToJustOneViableAlt(Collection<BitSet> altsets) {
return !hasMoreThanOneViableAlt(altsets);
public static int resolvesToJustOneViableAlt(Collection<BitSet> altsets) {
return getSingleViableAlt(altsets);
}
public static boolean allSubsetsConflict(Collection<BitSet> altsets) {
@ -393,16 +393,16 @@ public enum PredictionMode {
return false;
}
public static boolean hasMoreThanOneViableAlt(Collection<BitSet> altsets) {
public static int getSingleViableAlt(Collection<BitSet> altsets) {
BitSet viableAlts = new BitSet();
for (BitSet alts : altsets) {
int minAlt = alts.nextSetBit(0);
viableAlts.set(minAlt);
if ( viableAlts.cardinality()>1 ) { // more than 1 viable alt
return true;
return ATN.INVALID_ALT_NUMBER;
}
}
return false;
return viableAlts.nextSetBit(0);
}
}

2
tool/playground/T-input
View File

@ -1 +1 @@
a+b*c
abc

15
tool/playground/T.g
View File

@ -1,12 +1,5 @@
grammar T;
s : expr[0] ;
expr[int _p]
: ID
( {5 >= $_p}? '*' expr[6]
| {4 >= $_p}? '+' expr[5]
)*
;
ID : [a-zA-Z]+ ; // match identifiers
WS : [ \t\r\n]+ -> skip ; // toss out whitespace
s@after {dumpDFA();}
: ID | ID {;} ;
ID : 'a'..'z'+ ;
WS : (' '|'\t'|'\n')+ {skip();} ;

2
tool/playground/TestJavaLR.java
View File

@ -198,7 +198,7 @@ class TestJavaLR {
System.out.println(ParserATNSimulator.predict_calls +" parser predict calls");
System.out.println(ParserATNSimulator.retry_with_context +" retry_with_context after SLL conflict");
System.out.println(ParserATNSimulator.retry_with_context_indicates_no_conflict +" retry sees no conflict");
System.out.println(ParserATNSimulator.retry_with_context_predicts_same_as_alt +" retry predicts same alt as resolving conflict");
System.out.println(ParserATNSimulator.retry_with_context_predicts_same_alt +" retry predicts same alt as resolving conflict");
System.out.println(ParserATNSimulator.retry_with_context_from_dfa +" retry from DFA");
}

37
tool/test/org/antlr/v4/test/TestFullContextParsing.java
View File

@ -41,7 +41,7 @@ import org.junit.Test;
*/
public class TestFullContextParsing extends BaseTest {
@Test public void testAmbigYieldsNonCtxSensitiveDFA() {
@Test public void testAmbigYieldsCtxSensitiveDFA() {
String grammar =
"grammar T;\n"+
"s" +
@ -53,9 +53,9 @@ public class TestFullContextParsing extends BaseTest {
"abc", true);
String expecting =
"Decision 0:\n" +
"s0-ID->:s1=>1\n"; // not ctx sensitive
"s0-ID->s1^\n"; // ctx sensitive
assertEquals(expecting, result);
assertEquals("line 1:0 reportAmbiguity d=0: ambigAlts={1, 2}, input='abc'\n",
assertEquals("line 1:0 reportAttemptingFullContext d=0, input='abc'\n",
this.stderrDuringParse);
}
@ -122,6 +122,7 @@ public class TestFullContextParsing extends BaseTest {
String grammar =
"grammar T;\n"+
"s" +
"@init {_interp.setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);}\n" +
"@after {dumpDFA();}\n" +
" : '{' stat* '}'" +
" ;\n" +
@ -166,16 +167,15 @@ public class TestFullContextParsing extends BaseTest {
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:34 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n",
"line 1:38 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo}'\n",
this.stderrDuringParse);
// should not be ambiguous because the second 'else foo' clearly
// should not be ambiguous because the second 'else bar' clearly
// indicates that the first else should match to the innermost if.
// but, current ambig detection doesn't know that. It stops at 'else foo'
// instead of seeing the next else. See to-do in execATNWithFullContext()
// LL_EXACT_AMBIG_DETECTION makes us keep going to resolve
input =
"{ if x then if y then return else foo else foo }";
"{ if x then if y then return else foo else bar }";
result = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
@ -183,7 +183,7 @@ public class TestFullContextParsing extends BaseTest {
"s0-'else'->s1^\n";
assertEquals(expecting, result);
assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:34 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n" +
"line 1:38 reportContextSensitivity d=1, input='elsefooelse'\n" +
"line 1:38 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:38 reportContextSensitivity d=1, input='else'\n",
this.stderrDuringParse);
@ -201,7 +201,7 @@ public class TestFullContextParsing extends BaseTest {
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n" +
"line 2:27 reportAttemptingFullContext d=1, input='else'\n" +
"line 2:32 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n",
"line 2:36 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo}'\n",
this.stderrDuringParse);
input =
@ -217,7 +217,7 @@ public class TestFullContextParsing extends BaseTest {
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n" +
"line 2:27 reportAttemptingFullContext d=1, input='else'\n" +
"line 2:32 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n",
"line 2:36 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo}'\n",
this.stderrDuringParse);
}
@ -229,7 +229,9 @@ public class TestFullContextParsing extends BaseTest {
public void testLoopsSimulateTailRecursion() throws Exception {
String grammar =
"grammar T;\n" +
"prog: expr_or_assign*;\n" +
"prog\n" +
"@init {_interp.setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);}\n" +
" : expr_or_assign*;\n" +
"expr_or_assign\n" +
" : expr '++' {System.out.println(\"fail.\");}\n" +
" | expr {System.out.println(\"pass: \"+$expr.text);}\n" +
@ -247,7 +249,7 @@ public class TestFullContextParsing extends BaseTest {
assertEquals("pass: a(i)<-x\n", found);
String expecting =
"line 1:7 reportAttemptingFullContext d=3, input='a(i)<-x'\n" +
"line 1:3 reportAttemptingFullContext d=3, input='a(i)'\n" +
"line 1:7 reportAmbiguity d=3: ambigAlts={2, 3}, input='a(i)<-x'\n";
assertEquals(expecting, this.stderrDuringParse);
}
@ -257,7 +259,9 @@ public class TestFullContextParsing extends BaseTest {
// simpler version of testLoopsSimulateTailRecursion, no loops
String grammar =
"grammar T;\n" +
"prog: expr expr {System.out.println(\"alt 1\");}\n" +
"prog\n" +
"@init {_interp.setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);}\n" +
" : expr expr {System.out.println(\"alt 1\");}\n" +
" | expr\n" +
" ;\n" +
"expr: '@'\n" +
@ -271,6 +275,7 @@ public class TestFullContextParsing extends BaseTest {
assertEquals("alt 1\n", found);
String expecting =
"line 1:2 reportAttemptingFullContext d=0, input='a@'\n" +
"line 1:2 reportAmbiguity d=0: ambigAlts={1, 2}, input='a@'\n" +
"line 1:2 reportAttemptingFullContext d=1, input='a@'\n" +
"line 1:2 reportContextSensitivity d=1, input='a@'\n";
@ -282,7 +287,9 @@ public class TestFullContextParsing extends BaseTest {
// translated left-recursive expr rule to test ambig detection
String grammar =
"grammar T;\n" +
"s : expr[0] {System.out.println($expr.ctx.toStringTree(this));} ;\n" +
"s\n" +
"@init {_interp.setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);}\n" +
" : expr[0] {System.out.println($expr.ctx.toStringTree(this));} ;\n" +
"\n" +
"expr[int _p]\n" +
" : ID\n" +

3
tool/test/org/antlr/v4/test/TestLeftRecursion.java
View File

@ -330,8 +330,7 @@ public class TestLeftRecursion extends BaseTest {
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "a+b*2\n", true);
assertEquals("line 1:1 reportAttemptingFullContext d=3, input='+'\n" +
"line 1:1 reportContextSensitivity d=3, input='+'\n" +
"line 1:3 reportAttemptingFullContext d=3, input='*'\n" +
"line 1:3 reportAmbiguity d=3: ambigAlts={1, 2}, input='*'\n",
"line 1:3 reportAttemptingFullContext d=3, input='*'\n",
stderrDuringParse);
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "(1+2)*3\n", true);

15
tool/test/org/antlr/v4/test/TestSemPredEvalParser.java
View File

@ -147,16 +147,14 @@ public class TestSemPredEvalParser extends BaseTest {
}
@Test public void test2UnpredicatedAlts() throws Exception {
// We have n-2 predicates for n alternatives. We have no choice
// but to pick the first on predicated alternative if the n-2
// predicates fail.
// this should call reportInsufficientPredicates()
// We have n-2 predicates for n alternatives. pick first alt
String grammar =
"grammar T;\n" +
"@header {" +
"import java.util.*;" +
"}" +
"s : a ';' a;\n" + // do 2x: once in ATN, next in DFA
"s : {_interp.setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);}\n" +
" a ';' a;\n" + // do 2x: once in ATN, next in DFA
"a : ID {System.out.println(\"alt 1\");}\n" +
" | ID {System.out.println(\"alt 2\");}\n" +
" | {false}? ID {System.out.println(\"alt 3\");}\n" +
@ -179,16 +177,13 @@ public class TestSemPredEvalParser extends BaseTest {
}
@Test public void test2UnpredicatedAltsAndOneOrthogonalAlt() throws Exception {
// We have n-2 predicates for n alternatives. We have no choice
// but to pick the first on predicated alternative if the n-2
// predicates fail.
// this should call reportInsufficientPredicates()
String grammar =
"grammar T;\n" +
"@header {" +
"import java.util.*;" +
"}" +
"s : a ';' a ';' a;\n" +
"s : {_interp.setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);}\n" +
" a ';' a ';' a;\n" +
"a : INT {System.out.println(\"alt 1\");}\n" +
" | ID {System.out.println(\"alt 2\");}\n" + // must pick this one for ID since pred is false
" | ID {System.out.println(\"alt 3\");}\n" +