From f015942b5e0418eec59a9be4ca03b68b766a848e Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Wed, 24 Apr 2013 12:29:43 -0500 Subject: [PATCH] * Simplify creation of new DFA edges - only create the target state and let DFA code check for termination conditions * Fix handling of previously cached error edges (always need to check previous state for reaching a rule stop state) * Fix DFA created during SLL-only parsing (should be identical to the DFA created by full LL parsing) --- .../v4/runtime/atn/ParserATNSimulator.java | 154 ++++++------------ .../antlr/v4/runtime/dfa/DFASerializer.java | 14 +- .../v4/test/TestATNParserPrediction.java | 12 +- .../antlr/v4/test/TestFullContextParsing.java | 20 +-- 4 files changed, 74 insertions(+), 126 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 8d494810b..998a81909 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -433,128 +433,76 @@ public class ParserATNSimulator extends ATNSimulator { DFAState D = null; if (previousD.edges != null && t + 1 >= 0 && t + 1 < previousD.edges.length) { D = previousD.edges[t + 1]; - if ( D == ERROR ) { - throw noViableAlt(input, outerContext, previousD.configs, startIndex); - } } if (D == null) { // System.out.println("REACH "+getLookaheadName(input)); ATNConfigSet reach = computeReachSet(previousD.configs, t, false); if ( reach==null ) { - // if any configs in previous dipped into outer context, that - // means that input up to t actually finished entry rule - // at least for SLL decision. Full LL doesn't dip into outer - // so don't need special case. - // We will get an error no matter what so delay until after - // decision; better error message. Also, no reachable target - // ATN states in SLL implies LL will also get nowhere. - // If conflict in states that dip out, choose min since we - // will get error no matter what. - int alt = getAltThatFinishedDecisionEntryRule(previousD.configs); - if ( alt!=ATN.INVALID_ALT_NUMBER ) { - // return w/o altering DFA - return alt; + D = ERROR; + } + else { + // create new target state; we'll add to DFA after it's complete + D = new DFAState(reach); + + int predictedAlt = getUniqueAlt(reach); + + if ( debug ) { + Collection altSubSets = PredictionMode.getConflictingAltSubsets(reach); + System.out.println("SLL altSubSets="+altSubSets+ + ", configs="+reach+ + ", predict="+predictedAlt+", allSubsetsConflict="+ + PredictionMode.allSubsetsConflict(altSubSets)+", conflictingAlts="+ + getConflictingAlts(reach)); } - addDFAEdge(dfa, previousD, t, ERROR); - throw noViableAlt(input, outerContext, previousD.configs, startIndex); - } - - // create new target state; we'll add to DFA after it's complete - D = new DFAState(reach); - - int predictedAlt = getUniqueAlt(reach); - - if ( debug ) { - Collection altSubSets = PredictionMode.getConflictingAltSubsets(reach); - System.out.println("SLL altSubSets="+altSubSets+ - ", configs="+reach+ - ", predict="+predictedAlt+", allSubsetsConflict="+ - PredictionMode.allSubsetsConflict(altSubSets)+", conflictingAlts="+ - getConflictingAlts(reach)); - } - - if ( predictedAlt!=ATN.INVALID_ALT_NUMBER ) { - // NO CONFLICT, UNIQUELY PREDICTED ALT - D.isAcceptState = true; - D.configs.uniqueAlt = predictedAlt; - D.prediction = predictedAlt; - } - else if ( PredictionMode.hasSLLConflictTerminatingPrediction(mode, reach) ) { - // MORE THAN ONE VIABLE ALTERNATIVE - D.configs.conflictingAlts = getConflictingAlts(reach); - if ( mode == PredictionMode.SLL ) { - // stop w/o failover for sure - if ( outerContext == ParserRuleContext.EMPTY || // in grammar start rule - !D.configs.dipsIntoOuterContext ) // didn't fall out of rule - { - // SPECIAL CASE WHERE SLL KNOWS CONFLICT IS AMBIGUITY - // report even if preds - reportAmbiguity(dfa, D, startIndex, input.index(), - D.configs.conflictingAlts, D.configs); - } - // always stop at D + if ( predictedAlt!=ATN.INVALID_ALT_NUMBER ) { + // NO CONFLICT, UNIQUELY PREDICTED ALT + D.isAcceptState = true; + D.configs.uniqueAlt = predictedAlt; + D.prediction = predictedAlt; + } + else if ( PredictionMode.hasSLLConflictTerminatingPrediction(mode, reach) ) { + // MORE THAN ONE VIABLE ALTERNATIVE + D.configs.conflictingAlts = getConflictingAlts(reach); + D.requiresFullContext = true; + // in SLL-only mode, we will stop at this state and return the minimum alt D.isAcceptState = true; D.prediction = D.configs.conflictingAlts.nextSetBit(0); - if ( debug ) System.out.println("SLL RESOLVED TO "+D.prediction+" for "+D); - predictedAlt = D.prediction; - // Falls through to check predicates below } - else { - // IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) - if ( D.configs.hasSemanticContext ) { - predicateDFAState(D, atn.getDecisionState(dfa.decision)); - if (D.predicates != null) { - int conflictIndex = input.index(); - if (conflictIndex != startIndex) { - input.seek(startIndex); - } - BitSet alts = evalSemanticContext(D.predicates, outerContext, true); - if ( alts.cardinality()==1 ) { - if ( debug ) System.out.println("Full LL avoided"); - return alts.nextSetBit(0); - } - - if (conflictIndex != startIndex) { - // restore the index so reporting the fallback to full - // context occurs with the index at the correct spot - input.seek(conflictIndex); - } - } + if ( D.isAcceptState && D.configs.hasSemanticContext ) { + predicateDFAState(D, atn.getDecisionState(dfa.decision)); + if (D.predicates != null) { + D.prediction = ATN.INVALID_ALT_NUMBER; } - - // RETRY WITH FULL LL CONTEXT - if ( debug ) System.out.println("RETRY with outerContext="+outerContext); - ATNConfigSet s0_closure = - computeStartState(dfa.atnStartState, - outerContext, - true); - - // not accept state: isCtxSensitive - D.requiresFullContext = true; // always force DFA to ATN simulate - D.prediction = ATN.INVALID_ALT_NUMBER; - D = addDFAEdge(dfa, previousD, t, D); - - return execATNWithFullContext(dfa, D, s0_closure, - input, startIndex, - outerContext, - D.configs.conflictingAlts.nextSetBit(0)); - } - } - - if ( D.isAcceptState && D.configs.hasSemanticContext ) { - predicateDFAState(D, atn.getDecisionState(dfa.decision)); - if (D.predicates != null) { - D.prediction = ATN.INVALID_ALT_NUMBER; } } // all adds to dfa are done after we've created full D state D = addDFAEdge(dfa, previousD, t, D); } - else if ( D.requiresFullContext && mode != PredictionMode.SLL ) { + + if (D == ERROR) { + // if any configs in previous dipped into outer context, that + // means that input up to t actually finished entry rule + // at least for SLL decision. Full LL doesn't dip into outer + // so don't need special case. + // We will get an error no matter what so delay until after + // decision; better error message. Also, no reachable target + // ATN states in SLL implies LL will also get nowhere. + // If conflict in states that dip out, choose min since we + // will get error no matter what. + int alt = getAltThatFinishedDecisionEntryRule(previousD.configs); + if ( alt!=ATN.INVALID_ALT_NUMBER ) { + // return w/o altering DFA + return alt; + } + + throw noViableAlt(input, outerContext, previousD.configs, startIndex); + } + + if ( D.requiresFullContext && mode != PredictionMode.SLL ) { // IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) if ( D.predicates!=null ) { if ( debug ) System.out.println("DFA state has preds in DFA sim LL failover"); diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java index 4c2fe7c5b..cb69f43c5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java @@ -79,20 +79,20 @@ public class DFASerializer { return label; } - String getStateString(DFAState s) { + @NotNull + protected String getStateString(@NotNull DFAState s) { int n = s.stateNumber; - String stateStr = "s"+n; + final String baseStateStr = (s.isAcceptState ? ":" : "") + "s" + n + (s.requiresFullContext ? "^" : ""); if ( s.isAcceptState ) { if ( s.predicates!=null ) { - stateStr = ":s"+n+"=>"+Arrays.toString(s.predicates); + return baseStateStr + "=>" + Arrays.toString(s.predicates); } else { - stateStr = ":s"+n+"=>"+s.prediction; + return baseStateStr + "=>" + s.prediction; } } - else if ( s.requiresFullContext) { - stateStr = "s"+n+"^"; + else { + return baseStateStr; } - return stateStr; } } diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index e1286a31f..71485ca94 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -243,14 +243,14 @@ public class TestATNParserPrediction extends BaseTest { }; String[] dfa = { "s0-'a'->s1\n" + - "s1-EOF->s2^\n", + "s1-EOF->:s2^=>1\n", "s0-'a'->s1\n" + - "s1-EOF->s2^\n" + + "s1-EOF->:s2^=>1\n" + "s1-'b'->:s3=>3\n", "s0-'a'->s1\n" + - "s1-EOF->s2^\n" + + "s1-EOF->:s2^=>1\n" + "s1-'b'->:s3=>3\n", }; checkDFAConstruction(lg, g, decision, inputs, dfa); @@ -278,16 +278,16 @@ public class TestATNParserPrediction extends BaseTest { String[] dfa = { "s0-'a'->s1\n" + "s1-'b'->s2\n" + - "s2-EOF->s3^\n", + "s2-EOF->:s3^=>1\n", "s0-'a'->s1\n" + "s1-'b'->s2\n" + - "s2-EOF->s3^\n" + + "s2-EOF->:s3^=>1\n" + "s2-'c'->:s4=>3\n", "s0-'a'->s1\n" + "s1-'b'->s2\n" + - "s2-EOF->s3^\n" + + "s2-EOF->:s3^=>1\n" + "s2-'c'->:s4=>3\n", }; checkDFAConstruction(lg, g, decision, inputs, dfa); diff --git a/tool/test/org/antlr/v4/test/TestFullContextParsing.java b/tool/test/org/antlr/v4/test/TestFullContextParsing.java index f9d9a2c59..145c82e43 100644 --- a/tool/test/org/antlr/v4/test/TestFullContextParsing.java +++ b/tool/test/org/antlr/v4/test/TestFullContextParsing.java @@ -56,7 +56,7 @@ public class TestFullContextParsing extends BaseTest { "abc", true); String expecting = "Decision 0:\n" + - "s0-ID->s1^\n"; // ctx sensitive + "s0-ID->:s1^=>1\n"; // ctx sensitive assertEquals(expecting, result); assertEquals("line 1:0 reportAttemptingFullContext d=0, input='abc'\n", this.stderrDuringParse); @@ -78,7 +78,7 @@ public class TestFullContextParsing extends BaseTest { String expecting = "Decision 1:\n" + "s0-INT->s1\n" + - "s1-ID->s2^\n"; + "s1-ID->:s2^=>1\n"; assertEquals(expecting, result); assertEquals("line 1:5 reportAttemptingFullContext d=1, input='34abc'\n" + "line 1:2 reportContextSensitivity d=1, input='34'\n", @@ -89,7 +89,7 @@ public class TestFullContextParsing extends BaseTest { expecting = "Decision 1:\n" + "s0-INT->s1\n" + - "s1-ID->s2^\n"; + "s1-ID->:s2^=>1\n"; assertEquals(expecting, result); assertEquals("line 1:5 reportAttemptingFullContext d=1, input='34abc'\n" + "line 1:5 reportContextSensitivity d=1, input='34abc'\n", @@ -112,7 +112,7 @@ public class TestFullContextParsing extends BaseTest { String expecting = "Decision 2:\n" + "s0-INT->s1\n" + - "s1-ID->s2^\n"; + "s1-ID->:s2^=>1\n"; assertEquals(expecting, result); assertEquals("line 1:5 reportAttemptingFullContext d=2, input='34abc'\n" + "line 1:2 reportContextSensitivity d=2, input='34'\n" + @@ -138,7 +138,7 @@ public class TestFullContextParsing extends BaseTest { String expecting = "Decision 0:\n" + "s0-INT->s1\n" + - "s1-ID->s2^\n"; // Must point at accept state + "s1-ID->:s2^=>1\n"; // Must point at accept state assertEquals(expecting, result); assertEquals("line 1:3 reportAttemptingFullContext d=0, input='34abc'\n" + "line 1:0 reportContextSensitivity d=0, input='34'\n", @@ -172,7 +172,7 @@ public class TestFullContextParsing extends BaseTest { input, true); expecting = "Decision 1:\n" + - "s0-'else'->s1^\n"; + "s0-'else'->:s1^=>1\n"; assertEquals(expecting, result); // Technically, this input sequence is not ambiguous because else // uniquely predicts going into the optional subrule. else cannot @@ -190,7 +190,7 @@ public class TestFullContextParsing extends BaseTest { input, true); expecting = "Decision 1:\n" + - "s0-'else'->s1^\n" + + "s0-'else'->:s1^=>1\n" + "s0-'}'->:s2=>2\n"; assertEquals(expecting, result); assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" + @@ -207,7 +207,7 @@ public class TestFullContextParsing extends BaseTest { input, true); expecting = "Decision 1:\n" + - "s0-'else'->s1^\n"; + "s0-'else'->:s1^=>1\n"; assertEquals(expecting, result); assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" + "line 1:38 reportContextSensitivity d=1, input='elsefooelse'\n" + @@ -222,7 +222,7 @@ public class TestFullContextParsing extends BaseTest { input, true); expecting = "Decision 1:\n" + - "s0-'else'->s1^\n" + + "s0-'else'->:s1^=>1\n" + "s0-'}'->:s2=>2\n"; assertEquals(expecting, result); assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" + @@ -238,7 +238,7 @@ public class TestFullContextParsing extends BaseTest { input, true); expecting = "Decision 1:\n" + - "s0-'else'->s1^\n" + + "s0-'else'->:s1^=>1\n" + "s0-'}'->:s2=>2\n"; assertEquals(expecting, result); assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +