forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

almost there.

This commit is contained in:
Terence Parr 2012-10-11 18:47:47 -07:00
parent 058ed51349
commit badb48a987
10 changed files with 245 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java
View File

@ -31,32 +31,34 @@ package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.BitSet;
/** How to emit recognition errors */
public interface ANTLRErrorListener {
/** Upon syntax error, notify any interested parties. This is not how to
* recover from errors or compute error messages. The parser
* ANTLRErrorStrategy specifies how to recover from syntax errors
* and how to compute error messages. This listener's job is simply to
* emit a computed message, though it has enough information to
* create its own message in many cases.
/** Upon syntax error, notify any interested parties. This is not
* how to recover from errors or compute error messages. The
* parser ANTLRErrorStrategy specifies how to recover from syntax
* errors and how to compute error messages. This listener's job
* is simply to emit a computed message, though it has enough
* information to create its own message in many cases.
*
* The RecognitionException is non-null for all syntax errors except
* when we discover mismatched token errors that we can recover from
* in-line, without returning from the surrounding rule (via the
* single token insertion and deletion mechanism).
* The RecognitionException is non-null for all syntax errors
* except when we discover mismatched token errors that we can
* recover from in-line, without returning from the surrounding
* rule (via the single token insertion and deletion mechanism).
*
* @param recognizer
* What parser got the error. From this object, you
* can access the context as well as the input stream.
* What parser got the error. From this
* object, you can access the context as well
* as the input stream.
* @param offendingSymbol
* The offending token in the input token stream, unless recognizer
* is a lexer (then it's null)
* If no viable alternative error, e has token
* at which we started production for the decision.
* The offending token in the input token
* stream, unless recognizer is a lexer (then it's null) If
* no viable alternative error, e has token at which we
* started production for the decision.
* @param line
* At what line in input to the error occur? This always refers to
* stopTokenIndex
@ -77,15 +79,18 @@ public interface ANTLRErrorListener {
String msg,
@Nullable RecognitionException e);
/** Called when the parser detects a true ambiguity: an input sequence can be matched
* literally by two or more pass through the grammar. ANTLR resolves the ambiguity in
* favor of the alternative appearing first in the grammar. The start and stop index are
* zero-based absolute indices into the token stream. ambigAlts is a set of alternative numbers
* that can match the input sequence. This method is only called when we are parsing with
* full context.
/** Called when the parser detects a true ambiguity: an input
* sequence can be matched literally by two or more pass through
* the grammar. ANTLR resolves the ambiguity in favor of the
* alternative appearing first in the grammar. The start and stop
* index are zero-based absolute indices into the token
* stream. ambigAlts is a set of alternative numbers that can
* match the input sequence. This method is only called when we
* are parsing with full context.
*/
void reportAmbiguity(@NotNull Parser recognizer,
DFA dfa, int startIndex, int stopIndex, @NotNull IntervalSet ambigAlts,
DFA dfa, int startIndex, int stopIndex,
@NotNull BitSet ambigAlts,
@NotNull ATNConfigSet configs);
void reportAttemptingFullContext(@NotNull Parser recognizer,
@ -93,10 +98,11 @@ public interface ANTLRErrorListener {
int startIndex, int stopIndex,
@NotNull ATNConfigSet configs);
/** Called by the parser when it find a conflict that is resolved by retrying the parse
* with full context. This is not a warning; it simply notifies you that your grammar
* is more complicated than Strong LL can handle. The parser moved up to full context
* parsing for that input sequence.
/** Called by the parser when it find a conflict that is resolved
* by retrying the parse with full context. This is not a
* warning; it simply notifies you that your grammar is more
* complicated than Strong LL can handle. The parser moved up to
* full context parsing for that input sequence.
*/
void reportContextSensitivity(@NotNull Parser recognizer,
@NotNull DFA dfa,

6
runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java
View File

@ -30,10 +30,10 @@ package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntervalSet;
import java.util.BitSet;
/**
*
* @author Sam Harwell
*/
public class BaseErrorListener implements ANTLRErrorListener {
@ -52,7 +52,7 @@ public class BaseErrorListener implements ANTLRErrorListener {
DFA dfa,
int startIndex,
int stopIndex,
IntervalSet ambigAlts,
BitSet ambigAlts,
ATNConfigSet configs)
{
}

24
runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java
View File

@ -32,18 +32,20 @@ package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import java.util.BitSet;
public class DiagnosticErrorListener extends BaseErrorListener {
@Override
public void reportAmbiguity(@NotNull Parser recognizer,
DFA dfa, int startIndex, int stopIndex, @NotNull IntervalSet ambigAlts,
DFA dfa, int startIndex, int stopIndex,
@NotNull BitSet ambigAlts,
@NotNull ATNConfigSet configs)
{
recognizer.notifyErrorListeners("reportAmbiguity d=" + dfa.decision +
": ambigAlts=" + ambigAlts + ", input='" +
recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
": ambigAlts=" + ambigAlts + ", input='" +
recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
}
@Override
@ -53,16 +55,18 @@ public class DiagnosticErrorListener extends BaseErrorListener {
@NotNull ATNConfigSet configs)
{
recognizer.notifyErrorListeners("reportAttemptingFullContext d=" +
dfa.decision + ", input='" +
recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
dfa.decision + ", input='" +
recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
}
@Override
public void reportContextSensitivity(@NotNull Parser recognizer, @NotNull DFA dfa,
int startIndex, int stopIndex, @NotNull ATNConfigSet configs)
public void reportContextSensitivity(@NotNull Parser recognizer,
@NotNull DFA dfa,
int startIndex, int stopIndex,
@NotNull ATNConfigSet configs)
{
recognizer.notifyErrorListeners("reportContextSensitivity d=" +
dfa.decision + ", input='" +
recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
dfa.decision + ", input='" +
recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
}
}

5
runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java
View File

@ -30,12 +30,11 @@ package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntervalSet;
import java.util.BitSet;
import java.util.Collection;
/**
*
* @author Sam Harwell
*/
public class ProxyErrorListener implements ANTLRErrorListener {
@ -63,7 +62,7 @@ public class ProxyErrorListener implements ANTLRErrorListener {
DFA dfa,
int startIndex,
int stopIndex,
IntervalSet ambigAlts,
BitSet ambigAlts,
ATNConfigSet configs)
{
for (ANTLRErrorListener listener : delegates) {

5
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java
View File

@ -31,9 +31,9 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.Array2DHashSet;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import org.antlr.v4.runtime.misc.IntervalSet;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
@ -280,7 +280,8 @@ public class ATNConfigSet implements Set<ATNConfig> {
// TODO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation
// TODO: can we track conflicts as they are added to save scanning configs later?
public int uniqueAlt;
protected IntervalSet conflictingAlts;
protected BitSet conflictingAlts;
// Used in parser and lexer. In lexer, it indicates we hit a pred
// while computing a closure operation. Don't make a DFA state from this.
public boolean hasSemanticContext;

12
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -29,6 +29,7 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
@ -98,6 +99,9 @@ public abstract class ATNSimulator {
atn.grammarType = toInt(data[p++]);
atn.maxTokenType = toInt(data[p++]);
// Set up target of all EOF edges emanating from rule stop states
ATNState eofTarget = new ATNState();
//
// STATES
//
@ -223,6 +227,14 @@ public abstract class ATNSimulator {
}
}
// If no edges out of stop state, add EOF transition
for (RuleStopState ruleStopState : atn.ruleToStopState) {
if ( ruleStopState.getNumberOfTransitions()==0 ) {
Transition t = new AtomTransition(eofTarget, Token.EOF);
ruleStopState.addTransition(t);
}
}
for (ATNState state : atn.states) {
if (state instanceof PlusLoopbackState) {
PlusLoopbackState loopbackState = (PlusLoopbackState)state;

186
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -255,7 +255,7 @@ import java.util.Set;
*/
public class ParserATNSimulator extends ATNSimulator {
public static boolean debug = false;
public static boolean debug = true;
public static boolean debug_list_atn_decisions = false;
public static boolean dfa_debug = false;
public static boolean retry_debug = false;
@ -541,12 +541,12 @@ public class ParserATNSimulator extends ATNSimulator {
input.seek(startIndex);
// since we don't report ambiguities in execDFA, we never need to
// use complete predicate evaluation here
IntervalSet alts = evalSemanticContext(s.predicates, outerContext, false);
if (alts.isNil()) {
BitSet alts = evalSemanticContext(s.predicates, outerContext, false);
if (alts.isEmpty()) {
throw noViableAlt(input, outerContext, s.configs, startIndex);
}
return alts.getMinElement();
return alts.nextSetBit(0);
}
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
@ -665,7 +665,18 @@ public class ParserATNSimulator extends ATNSimulator {
// }
// }
int predictedAlt = getUniqueAlt(reach);
Collection<BitSet> altSubSets = getConflictingAltSubsets(reach);
// System.out.println("SLL altsets: "+altSubSets);
int predictedAlt = getUniqueAlt(altSubSets);
if ( debug ) {
System.out.println("SLL altSubSets="+altSubSets+
", predict="+predictedAlt+", allSubsetsConflict="+
allSubsetsConflict(altSubSets)+", conflictingAlts="+
getConflictingAlts(reach));
}
if ( predictedAlt!=ATN.INVALID_ALT_NUMBER ) {
// NO CONFLICT, UNIQUE PREDICTED ALT
D.isAcceptState = true;
@ -673,14 +684,12 @@ public class ParserATNSimulator extends ATNSimulator {
D.prediction = predictedAlt;
}
else {
boolean cont = needMoreLookaheadSLL(reach);
cont = needMoreLookaheadLL(reach);
D.configs.conflictingAlts = getConflictingAlts(reach);
if ( D.configs.conflictingAlts!=null ) {
// CONFLICT, GREEDY (TYPICAL SITUATION)
// MORE THAN ONE VIABLE ALTERNATIVE
if ( allSubsetsConflict(altSubSets) ) {
D.configs.conflictingAlts = getConflictingAlts(reach);
if ( outerContext == ParserRuleContext.EMPTY || // in grammar start rule
!D.configs.dipsIntoOuterContext || // didn't fall out of rule
SLL ) // forcing SLL only
!D.configs.dipsIntoOuterContext || // didn't fall out of rule
SLL ) // forcing SLL only
{
// SPECIAL CASE WHERE SLL KNOWS CONFLICT IS AMBIGUITY
if ( !D.configs.hasSemanticContext ) {
@ -688,7 +697,7 @@ public class ParserATNSimulator extends ATNSimulator {
D.configs.conflictingAlts, D.configs);
}
D.isAcceptState = true;
D.prediction = D.configs.conflictingAlts.getMinElement();
D.prediction = D.configs.conflictingAlts.nextSetBit(0);
if ( debug ) System.out.println("RESOLVED TO "+D.prediction+" for "+D);
predictedAlt = D.prediction;
// Falls through to check predicates below
@ -709,7 +718,7 @@ public class ParserATNSimulator extends ATNSimulator {
predictedAlt = execATNWithFullContext(dfa, D, s0_closure,
input, startIndex,
outerContext,
D.configs.conflictingAlts.getMinElement());
D.configs.conflictingAlts.nextSetBit(0));
// not accept state: isCtxSensitive
D.requiresFullContext = true; // always force DFA to ATN simulate
D.prediction = ATN.INVALID_ALT_NUMBER;
@ -725,7 +734,7 @@ public class ParserATNSimulator extends ATNSimulator {
int nalts = decState.getNumberOfTransitions();
// Update DFA so reach becomes accept state with (predicate,alt)
// pairs if preds found for conflicting alts
IntervalSet altsToCollectPredsFrom = getConflictingAltsOrUniqueAlt(D.configs);
BitSet altsToCollectPredsFrom = getConflictingAltsOrUniqueAlt(D.configs);
SemanticContext[] altToPred = getPredsForAmbigAlts(altsToCollectPredsFrom, D.configs, nalts);
if ( altToPred!=null ) {
D.predicates = getPredicatePredictions(altsToCollectPredsFrom, altToPred);
@ -735,13 +744,13 @@ public class ParserATNSimulator extends ATNSimulator {
// There are preds in configs but they might go away
// when OR'd together like {p}? || NONE == NONE. If neither
// alt has preds, resolve to min alt
D.prediction = altsToCollectPredsFrom.getMinElement();
D.prediction = altsToCollectPredsFrom.nextSetBit(0);
}
if ( D.predicates!=null ) {
int stopIndex = input.index();
input.seek(startIndex);
IntervalSet alts = evalSemanticContext(D.predicates, outerContext, true);
BitSet alts = evalSemanticContext(D.predicates, outerContext, true);
D.prediction = ATN.INVALID_ALT_NUMBER; // indicate we have preds
addDFAEdge(dfa, previousD, t, D);
switch (alts.size()) {
@ -749,13 +758,13 @@ public class ParserATNSimulator extends ATNSimulator {
throw noViableAlt(input, outerContext, D.configs, startIndex);
case 1:
return alts.getMinElement();
return alts.nextSetBit(0);
default:
// report ambiguity after predicate evaluation to make sure the correct
// set of ambig alts is reported.
reportAmbiguity(dfa, D, startIndex, stopIndex, alts, D.configs);
return alts.getMinElement();
return alts.nextSetBit(0);
}
}
}
@ -812,11 +821,23 @@ public class ParserATNSimulator extends ATNSimulator {
}
throw noViableAlt(input, outerContext, previous, startIndex);
}
reach.uniqueAlt = getUniqueAlt(reach);
Collection<BitSet> altSubSets = getConflictingAltSubsets(reach);
if ( debug ) {
System.out.println("LL altSubSets="+altSubSets+
", predict="+getUniqueAlt(altSubSets)+
", resolvesToJustOneViableAlt="+
resolvesToJustOneViableAlt(altSubSets)+
", conflictingAlts="+
getConflictingAlts(reach));
}
// System.out.println("altSubSets: "+altSubSets);
reach.uniqueAlt = getUniqueAlt(altSubSets);
if ( reach.uniqueAlt!=ATN.INVALID_ALT_NUMBER ) break;
// boolean cont = needMoreLookaheadLL(reach);
reach.conflictingAlts = getConflictingAlts(reach);
if ( reach.conflictingAlts!=null ) break;
if ( resolvesToJustOneViableAlt(altSubSets) ) break;
previous = reach;
input.consume();
t = input.LA(1);
@ -834,10 +855,30 @@ public class ParserATNSimulator extends ATNSimulator {
// We do not check predicates here because we have checked them
// on-the-fly when doing full context prediction.
// must have conflict
reportAmbiguity(dfa, D, startIndex, input.index(), reach.conflictingAlts, reach);
// At this point, we know that we have conflicting configurations.
// But, that does not mean that there is no way forward without
// a conflict. It's possible to have nonconflicting alt subsets; e.g.,
//
// LL altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}]
//
// from
//
// [(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]),
// (13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])]
//
// In this case, (17,1,[5 $]) indicates there is some next sequence
// that would resolve this without conflict to alternative 1. Any
// other viable next sequence, however, is associated with a conflict.
// We stop looking for input because no amount of further lookahead
// will alter the fact that we should predict alternative 1.
// We just can't say for sure that there is an ambiguity without
// looking further.
return reach.conflictingAlts.getMinElement();
if ( /* TODO: len(all subsets)>1 or input consistent with a subset with len=1 */ true ) {
reportAmbiguity(dfa, D, startIndex, input.index(), reach.conflictingAlts, reach);
}
return reach.conflictingAlts.nextSetBit(0);
}
protected ATNConfigSet computeReachSet(ATNConfigSet closure, int t,
@ -869,7 +910,7 @@ public class ParserATNSimulator extends ATNSimulator {
// among the configurations.
reach = new ATNConfigSet(intermediate);
}
else if ( ParserATNSimulator.getUniqueAlt(intermediate)==1 ) {
else if ( getUniqueAlt(intermediate)==1 ) {
// Also don't pursue the closure if there is unique alternative
// among the configurations.
reach = new ATNConfigSet(intermediate);
@ -948,7 +989,7 @@ public class ParserATNSimulator extends ATNSimulator {
}
}
public SemanticContext[] getPredsForAmbigAlts(@NotNull IntervalSet ambigAlts,
public SemanticContext[] getPredsForAmbigAlts(@NotNull BitSet ambigAlts,
@NotNull ATNConfigSet configs,
int nalts)
{
@ -966,7 +1007,7 @@ public class ParserATNSimulator extends ATNSimulator {
*/
SemanticContext[] altToPred = new SemanticContext[nalts + 1];
for (ATNConfig c : configs) {
if ( ambigAlts.contains(c.alt) ) {
if ( ambigAlts.get(c.alt) ) {
altToPred[c.alt] = SemanticContext.or(altToPred[c.alt], c.semanticContext);
}
}
@ -992,7 +1033,7 @@ public class ParserATNSimulator extends ATNSimulator {
return altToPred;
}
public List<DFAState.PredPrediction> getPredicatePredictions(IntervalSet ambigAlts,
public List<DFAState.PredPrediction> getPredicatePredictions(BitSet ambigAlts,
SemanticContext[] altToPred)
{
List<DFAState.PredPrediction> pairs = new ArrayList<DFAState.PredPrediction>();
@ -1003,7 +1044,7 @@ public class ParserATNSimulator extends ATNSimulator {
// unpredicated is indicated by SemanticContext.NONE
assert pred != null;
if (ambigAlts!=null && ambigAlts.contains(i)) {
if (ambigAlts!=null && ambigAlts.get(i)) {
pairs.add(new DFAState.PredPrediction(pred, i));
}
if ( pred!=SemanticContext.NONE ) containsPredicate = true;
@ -1032,14 +1073,14 @@ public class ParserATNSimulator extends ATNSimulator {
* then we stop at the first predicate that evaluates to true. This
* includes pairs with null predicates.
*/
public IntervalSet evalSemanticContext(List<DFAState.PredPrediction> predPredictions,
ParserRuleContext<?> outerContext,
boolean complete)
public BitSet evalSemanticContext(List<DFAState.PredPrediction> predPredictions,
ParserRuleContext<?> outerContext,
boolean complete)
{
IntervalSet predictions = new IntervalSet();
BitSet predictions = new BitSet();
for (DFAState.PredPrediction pair : predPredictions) {
if ( pair.pred==SemanticContext.NONE ) {
predictions.add(pair.alt);
predictions.set(pair.alt);
if (!complete) {
break;
}
@ -1053,7 +1094,7 @@ public class ParserATNSimulator extends ATNSimulator {
if ( predicateEvaluationResult ) {
if ( debug || dfa_debug ) System.out.println("PREDICT "+pair.alt);
predictions.add(pair.alt);
predictions.set(pair.alt);
if (!complete) {
break;
}
@ -1438,20 +1479,43 @@ public class ParserATNSimulator extends ATNSimulator {
return needMoreLookaheadLL(configs);
}
// combined SLL+LL mode parsing
System.out.println("configs: "+configs);
// System.out.println("SLL configs: "+configs);
// map (s,_,x,_) -> altset for all configs
Collection<BitSet> altsets = getConflictingAltSubsets(configs);
System.out.println("altsets: "+altsets);
// System.out.println("SLL altsets: "+altsets);
return hasNonConflictingAltSet(altsets);
}
public boolean allSubsetsConflict(Collection<BitSet> altsets) {
return !hasNonConflictingAltSet(altsets);
}
/** return (there exists len(m)==1 for some m in altsets) */
public boolean hasNonConflictingAltSet(Collection<BitSet> altsets) {
for (BitSet alts : altsets) {
if ( alts.cardinality()==1 ) { // more than 1 viable alt
System.out.println("go; found nonconflicting alt: "+alts);
// System.out.println("SLL go; found nonconflicting alt: "+alts);
return true; // use more lookahead
}
}
System.out.println("stop");
// System.out.println("SLL stop");
return false; // all sets conflict with len(viable_alts)>1, stop
}
public int getUniqueAlt(Collection<BitSet> altsets) {
BitSet all = getAlts(altsets);
if ( all.cardinality()==1 ) return all.nextSetBit(0);
return ATN.INVALID_ALT_NUMBER;
}
public BitSet getAlts(Collection<BitSet> altsets) {
BitSet all = new BitSet();
for (BitSet alts : altsets) {
all.or(alts);
}
return all;
}
/**
Full LL prediction termination.
@ -1577,20 +1641,20 @@ public class ParserATNSimulator extends ATNSimulator {
return false; # len(viable_alts)==1, stop
*/
public boolean needMoreLookaheadLL(@NotNull ATNConfigSet configs) {
System.out.println("configs: "+configs);
// System.out.println("configs: "+configs);
// map (s,_,x,_) -> altset for all configs
Collection<BitSet> altsets = getConflictingAltSubsets(configs);
System.out.println("altsets: "+altsets);
// System.out.println("altsets: "+altsets);
BitSet viableAlts = new BitSet();
for (BitSet alts : altsets) {
int minAlt = alts.nextSetBit(0);
viableAlts.set(minAlt);
if ( viableAlts.cardinality()>1 ) { // more than 1 viable alt
System.out.println("go; viableAlts="+viableAlts);
// System.out.println("go; viableAlts="+viableAlts);
return true; // try using more lookahead
}
}
System.out.println("stop");
// System.out.println("stop");
return false; // len(viable_alts)==1, stop
}
@ -1611,6 +1675,27 @@ public class ParserATNSimulator extends ATNSimulator {
return configToAlts.values();
}
public boolean resolvesToJustOneViableAlt(Collection<BitSet> altsets) {
return !hasMoreThanOneViableAlt(altsets);
}
public boolean hasMoreThanOneViableAlt(Collection<BitSet> altsets) {
BitSet viableAlts = new BitSet();
for (BitSet alts : altsets) {
int minAlt = alts.nextSetBit(0);
viableAlts.set(minAlt);
if ( viableAlts.cardinality()>1 ) { // more than 1 viable alt
return true;
}
}
return false;
}
public BitSet getConflictingAlts(ATNConfigSet configs) {
Collection<BitSet> altsets = getConflictingAltSubsets(configs);
return getAlts(altsets);
}
/**
* From grammar:
@ -1681,7 +1766,7 @@ public class ParserATNSimulator extends ATNSimulator {
TODO: now we know contexts are merged, can we optimize? Use big int -> config array?
*/
@Nullable
public IntervalSet getConflictingAlts(@NotNull ATNConfigSet configs) {
public IntervalSet getConflictingAlts_old(@NotNull ATNConfigSet configs) {
if ( debug ) System.out.println("### check ambiguous "+configs);
// System.out.println("getConflictingAlts; set size="+configs.size());
// First get a list of configurations for each state.
@ -1777,10 +1862,11 @@ public class ParserATNSimulator extends ATNSimulator {
return ambigAlts;
}
protected IntervalSet getConflictingAltsOrUniqueAlt(ATNConfigSet configs) {
IntervalSet conflictingAlts;
protected BitSet getConflictingAltsOrUniqueAlt(ATNConfigSet configs) {
BitSet conflictingAlts;
if ( configs.uniqueAlt!= ATN.INVALID_ALT_NUMBER ) {
conflictingAlts = IntervalSet.of(configs.uniqueAlt);
conflictingAlts = new BitSet();
conflictingAlts.set(configs.uniqueAlt);
}
else {
conflictingAlts = configs.conflictingAlts;
@ -1861,7 +1947,7 @@ public class ParserATNSimulator extends ATNSimulator {
configs, outerContext);
}
public static int getUniqueAlt(@NotNull Collection<ATNConfig> configs) {
public static int getUniqueAlt(@NotNull ATNConfigSet configs) {
int alt = ATN.INVALID_ALT_NUMBER;
for (ATNConfig c : configs) {
if ( alt == ATN.INVALID_ALT_NUMBER ) {
@ -1940,7 +2026,7 @@ public class ParserATNSimulator extends ATNSimulator {
/** If context sensitive parsing, we know it's ambiguity not conflict */
public void reportAmbiguity(@NotNull DFA dfa, DFAState D, int startIndex, int stopIndex,
@NotNull IntervalSet ambigAlts,
@NotNull BitSet ambigAlts,
@NotNull ATNConfigSet configs)
{
if ( debug || retry_debug ) {

13
tool/playground/T.g
View File

@ -1,5 +1,10 @@
grammar T;
options {tokenVocab=A;}
s : ID ;
ID : 'a'..'z'+ ;
WS : (' '|'\n') {skip();} ;
s : expr expr
| expr
;
expr: '@'
| ID '@'
| ID
;
ID : [a-z]+ ;
WS : [ \r\n\t]+ -> skip ;

31
tool/playground/TestT.java
View File

@ -1,13 +1,9 @@
import org.antlr.v4.runtime.BailErrorStrategy;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonTokenFactory;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ConsoleErrorListener;
import org.antlr.v4.runtime.DefaultErrorStrategy;
import org.antlr.v4.runtime.DiagnosticErrorListener;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.UnbufferedCharStream;
import java.io.FileInputStream;
import java.io.InputStream;
@ -20,7 +16,7 @@ public class TestT {
if ( inputFile!=null ) {
is = new FileInputStream(inputFile);
}
CharStream input = new UnbufferedCharStream(is);
CharStream input = new ANTLRInputStream(is);
TLexer lex = new TLexer(input);
lex.setTokenFactory(new CommonTokenFactory(true));
@ -30,31 +26,10 @@ public class TestT {
parser.addErrorListener(new DiagnosticErrorListener());
ParserRuleContext tree = null;
parser.getInterpreter().setSLL(true); // try with just SLL(*)
// no errors messages or recovery wanted during first try
parser.removeErrorListeners();
parser.setErrorHandler(new BailErrorStrategy());
try {
tree = parser.s();
}
catch (RuntimeException ex) {
if (ex.getClass() == RuntimeException.class &&
ex.getCause() instanceof RecognitionException)
{
System.out.println("trying with LL(*)");
tokens.reset(); // rewind
// back to standard listeners/handlers
parser.addErrorListener(ConsoleErrorListener.INSTANCE);
parser.setErrorHandler(new DefaultErrorStrategy());
parser.getInterpreter().setSLL(false); // try full LL(*)
tree = parser.s();
}
}
// parser.getInterpreter().setSLL(true);
// parser.setTrace(true);
ParserRuleContext tree = parser.s();
System.out.println(tree.toStringTree(parser));
// tree.save(parser, "/tmp/t.ps");
}

50
tool/test/org/antlr/v4/test/TestFullContextParsing.java
View File

@ -55,7 +55,7 @@ public class TestFullContextParsing extends BaseTest {
"Decision 0:\n" +
"s0-ID->:s1=>1\n"; // not ctx sensitive
assertEquals(expecting, result);
assertEquals("line 1:0 reportAmbiguity d=0: ambigAlts={1..2}, input='abc'\n",
assertEquals("line 1:0 reportAmbiguity d=0: ambigAlts={1, 2}, input='abc'\n",
this.stderrDuringParse);
}
@ -125,7 +125,7 @@ public class TestFullContextParsing extends BaseTest {
"@after {dumpDFA();}\n" +
" : '{' stat* '}'" +
" ;\n" +
"stat: 'if' ID 'then' stat ('else' 'foo')?\n" +
"stat: 'if' ID 'then' stat ('else' ID)?\n" +
" | 'return'\n" +
" ;" +
"ID : 'a'..'z'+ ;\n"+
@ -139,19 +139,6 @@ public class TestFullContextParsing extends BaseTest {
assertEquals(expecting, result);
assertEquals(null, this.stderrDuringParse);
input =
"{ if x then if y then return else foo }";
result = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 1:\n" +
"s0-'else'->s1^\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:38 reportAmbiguity d=1: ambigAlts={1..2}, input='elsefoo}'\n",
this.stderrDuringParse);
input = "{ if x then return else foo }";
result = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
@ -169,15 +156,36 @@ public class TestFullContextParsing extends BaseTest {
"line 1:19 reportContextSensitivity d=1, input='else'\n",
this.stderrDuringParse);
input = "{ if x then return else foo }";
input =
"{ if x then if y then return else foo }";
result = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 1:\n" +
"s0-'else'->s1^\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:34 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n",
this.stderrDuringParse);
// should not be ambiguous because the second 'else foo' clearly
// indicates that the first else should match to the innermost if.
// but, current ambig detection doesn't know that. It stops at 'else foo'
// instead of seeing the next else. See to-do in execATNWithFullContext()
input =
"{ if x then if y then return else foo else foo }";
result = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 1:\n" +
"s0-'else'->s1^\n";
assertEquals(expecting, result);
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n",
assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:34 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n" +
"line 1:38 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:38 reportContextSensitivity d=1, input='else'\n",
this.stderrDuringParse);
input =
@ -193,7 +201,7 @@ public class TestFullContextParsing extends BaseTest {
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n" +
"line 2:27 reportAttemptingFullContext d=1, input='else'\n" +
"line 2:36 reportAmbiguity d=1: ambigAlts={1..2}, input='elsefoo}'\n",
"line 2:32 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n",
this.stderrDuringParse);
input =
@ -209,7 +217,7 @@ public class TestFullContextParsing extends BaseTest {
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n" +
"line 2:27 reportAttemptingFullContext d=1, input='else'\n" +
"line 2:36 reportAmbiguity d=1: ambigAlts={1..2}, input='elsefoo}'\n",
"line 2:32 reportAmbiguity d=1: ambigAlts={1, 2}, input='elsefoo'\n",
this.stderrDuringParse);
}
@ -236,7 +244,7 @@ public class TestFullContextParsing extends BaseTest {
"";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "prog", "a(i)<-x", true);
assertEquals("pass.\n", found);
assertEquals("pass: a(i)<-x\n", found);
String expecting =
"line 1:3 reportAttemptingFullContext d=3, input='a(i)'\n" +