jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

squash all commits for this branch

This commit is contained in:
Terence Parr 2015-02-14 13:51:31 -08:00
parent 66e275d164
commit d4a43684be
23 changed files with 685 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java
View File

@ -34,7 +34,6 @@ import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.NotNull;
import java.util.BitSet;

4
runtime/Java/src/org/antlr/v4/runtime/IntStream.java
View File

@ -30,8 +30,6 @@
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.NotNull;
/**
* A simple stream of symbols whose values are represented as integers. This
* interface provides <em>marked ranges</em> with support for a minimum level
@ -197,7 +195,7 @@ public interface IntStream {
* Set the input cursor to the position indicated by {@code index}. If the
* specified index lies past the end of the stream, the operation behaves as
* though {@code index} was the index of the EOF symbol. After this method
* returns without throwing an exception, the at least one of the following
* returns without throwing an exception, then at least one of the following
* will be true.
*
* <ul>

17
runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java
View File

@ -45,7 +45,9 @@ public class InterpreterRuleContext extends ParserRuleContext {
/**
* This is the backing field for {@link #getRuleIndex}.
*/
private final int ruleIndex;
private int ruleIndex = -1;
public InterpreterRuleContext() { }
/**
* Constructs a new {@link InterpreterRuleContext} with the specified
@ -67,4 +69,17 @@ public class InterpreterRuleContext extends ParserRuleContext {
public int getRuleIndex() {
return ruleIndex;
}
/** Copy a {@link ParserRuleContext} or {@link InterpreterRuleContext}
* stack to a {@link InterpreterRuleContext} tree.
* Return {@link null} if {@code ctx} is null.
*/
public static InterpreterRuleContext fromParserRuleContext(ParserRuleContext ctx) {
if ( ctx==null ) return null;
InterpreterRuleContext dup = new InterpreterRuleContext();
dup.copyFrom(ctx);
dup.ruleIndex = ctx.getRuleIndex();
dup.parent = fromParserRuleContext(ctx.getParent());
return dup;
}
}

109
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -32,8 +32,10 @@ package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializationOptions;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNSimulator;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.AmbiguityInfo;
import org.antlr.v4.runtime.atn.ParseInfo;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.atn.PredictionMode;
@ -46,10 +48,12 @@ import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.antlr.v4.runtime.tree.Trees;
import org.antlr.v4.runtime.tree.pattern.ParseTreePattern;
import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@ -679,8 +683,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
}
}
/**
* Like {@link #enterRule} but for recursive rules.
/** Like {@link #enterRule} but for recursive rules.
* Make the current context the child of the incoming localctx.
*/
public void pushNewRecursionContext(ParserRuleContext localctx, int state, int ruleIndex) {
ParserRuleContext previous = _ctx;
@ -751,6 +755,105 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
return false;
}
/** Given an AmbiguityInfo object that contains information about an
* ambiguous decision event, return the list of ambiguous parse trees.
* An ambiguity occurs when a specific token sequence can be recognized
* in more than one way by the grammar. These ambiguities are detected only
* at decision points.
*
* The list of trees includes the actual interpretation (that for
* the minimum alternative number) and all ambiguous alternatives.
*
* This method reuses the same physical input token stream used to
* detect the ambiguity by the original parser in the first place.
* This method resets/seeks within but does not alter originalParser.
* The input position is restored upon exit from this method.
* Parsers using a {@link UnbufferedTokenStream} may not be able to
* perform the necessary save index() / seek(saved_index) operation.
*
* The trees are rooted at the node whose start..stop token indices
* include the start and stop indices of this ambiguity event. That is,
* the trees returns will always include the complete ambiguous subphrase
* identified by the ambiguity event.
*
* Be aware that this method does NOT notify error or parse listeners as
* it would trigger duplicate or otherwise unwanted events.
*
* This uses a temporary ParserATNSimulator and a ParserInterpreter
* so we don't mess up any statistics, event lists, etc...
* The parse tree constructed while identifying/making ambiguityInfo is
* not affected by this method as it creates a new parser interp to
* get the ambiguous interpretations.
*
* Nodes in the returned ambig trees are independent of the original parse
* tree (constructed while identifying/creating ambiguityInfo).
*
* @since 4.5.1
*
* @param originalParser The parser used to create ambiguityInfo; it
* is not modified by this routine and can be either
* a generated or interpreted parser. It's token
* stream *is* reset/seek()'d.
* @param ambiguityInfo The information about an ambiguous decision event
* for which you want ambiguous parse trees.
*
* @throws RecognitionException Throws upon syntax error while matching
* ambig input.
*/
public static List<ParserRuleContext> getAmbiguousParseTrees(Parser originalParser,
AmbiguityInfo ambiguityInfo,
int startRuleIndex)
throws RecognitionException
{
List<ParserRuleContext> trees = new ArrayList<ParserRuleContext>();
int saveTokenInputPosition = originalParser.getTokenStream().index();
try {
// Create a new parser interpreter to parse the ambiguous subphrase
ParserInterpreter parser;
if ( originalParser instanceof ParserInterpreter ) {
parser = new ParserInterpreter((ParserInterpreter) originalParser);
}
else {
char[] serializedAtn = ATNSerializer.getSerializedAsChars(originalParser.getATN());
ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
parser = new ParserInterpreter(originalParser.getGrammarFileName(),
originalParser.getVocabulary(),
Arrays.asList(originalParser.getRuleNames()),
deserialized,
originalParser.getTokenStream());
}
// Make sure that we don't get any error messages from using this temporary parser
parser.removeErrorListeners();
parser.removeParseListeners();
parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
// get ambig trees
int alt = ambiguityInfo.ambigAlts.nextSetBit(0);
while ( alt>=0 ) {
// re-parse input for all ambiguous alternatives
// (don't have to do first as it's been parsed, but do again for simplicity
// using this temp parser.)
parser.reset();
parser.getTokenStream().seek(ambiguityInfo.startIndex);
parser.overrideDecision = ambiguityInfo.decision;
parser.overrideDecisionInputIndex = ambiguityInfo.startIndex;
parser.overrideDecisionAlt = alt;
ParserRuleContext t = parser.parse(startRuleIndex);
ParserRuleContext ambigSubTree =
Trees.getRootOfSubtreeEnclosingRegion(t, ambiguityInfo.startIndex,
ambiguityInfo.stopIndex);
trees.add(ambigSubTree);
alt = ambiguityInfo.ambigAlts.nextSetBit(alt+1);
}
}
finally {
originalParser.getTokenStream().seek(saveTokenInputPosition);
}
return trees;
}
/**
* Checks whether or not {@code symbol} can follow the current state in the
* ATN. The behavior of this method is equivalent to the following, but is
@ -846,7 +949,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
return stack;
}
/** For debugging and other purposes. */
/** For debugging and other purposes. */
public List<String> getDFAStrings() {
synchronized (_interp.decisionToDFA) {
List<String> s = new ArrayList<String>();

201
runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java
View File

@ -45,7 +45,6 @@ import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.StarLoopEntryState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Pair;
import java.util.ArrayDeque;
@ -69,7 +68,10 @@ import java.util.Deque;
public class ParserInterpreter extends Parser {
protected final String grammarFileName;
protected final ATN atn;
protected final BitSet pushRecursionContextStates;
/** This identifies StarLoopEntryState's that begin the (...)*
* precedence loops of left recursive rules.
*/
protected final BitSet statesNeedingLeftRecursionContext;
protected final DFA[] decisionToDFA; // not shared like it is for generated parsers
protected final PredictionContextCache sharedContextCache =
@ -81,7 +83,37 @@ public class ParserInterpreter extends Parser {
private final Vocabulary vocabulary;
protected final Deque<Pair<ParserRuleContext, Integer>> _parentContextStack = new ArrayDeque<Pair<ParserRuleContext, Integer>>();
/** Tracks LR rules for adjusting the contexts */
protected final Deque<Pair<ParserRuleContext, Integer>> _parentContextStack =
new ArrayDeque<Pair<ParserRuleContext, Integer>>();
/** We need a map from (decision,inputIndex)->forced alt for computing ambiguous
* parse trees. For now, we allow exactly one override.
*/
protected int overrideDecision = -1;
protected int overrideDecisionInputIndex = -1;
protected int overrideDecisionAlt = -1;
/** A copy constructor that creates a new parser interpreter by reusing
* the fields of a previous interpreter.
*
* @since 4.5.1
*
* @param old The interpreter to copy
*/
public ParserInterpreter(ParserInterpreter old) {
super(old.getTokenStream());
this.atn = old.atn;
this.grammarFileName = old.grammarFileName;
this.statesNeedingLeftRecursionContext = old.statesNeedingLeftRecursionContext;
this.decisionToDFA = old.decisionToDFA;
this.tokenNames = old.tokenNames;
this.ruleNames = old.ruleNames;
this.vocabulary = old.vocabulary;
setInterpreter(new ParserATNSimulator(this, atn,
decisionToDFA,
sharedContextCache));
}
/**
* @deprecated Use {@link #ParserInterpreter(String, Vocabulary, Collection, ATN, TokenStream)} instead.
@ -110,15 +142,15 @@ public class ParserInterpreter extends Parser {
decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
}
// identify the ATN states where pushNewRecursionContext must be called
this.pushRecursionContextStates = new BitSet(atn.states.size());
// identify the ATN states where pushNewRecursionContext() must be called
this.statesNeedingLeftRecursionContext = new BitSet(atn.states.size());
for (ATNState state : atn.states) {
if (!(state instanceof StarLoopEntryState)) {
continue;
}
if (((StarLoopEntryState)state).precedenceRuleDecision) {
this.pushRecursionContextStates.set(state.stateNumber);
this.statesNeedingLeftRecursionContext.set(state.stateNumber);
}
}
@ -205,7 +237,8 @@ public class ParserInterpreter extends Parser {
@Override
public void enterRecursionRule(ParserRuleContext localctx, int state, int ruleIndex, int precedence) {
_parentContextStack.push(new Pair<ParserRuleContext, Integer>(_ctx, localctx.invokingState));
Pair<ParserRuleContext, Integer> pair = new Pair<ParserRuleContext, Integer>(_ctx, localctx.invokingState);
_parentContextStack.push(pair);
super.enterRecursionRule(localctx, state, ruleIndex, precedence);
}
@ -217,7 +250,13 @@ public class ParserInterpreter extends Parser {
int edge;
if (p.getNumberOfTransitions() > 1) {
getErrorHandler().sync(this);
edge = getInterpreter().adaptivePredict(_input, ((DecisionState)p).decision, _ctx);
int decision = ((DecisionState) p).decision;
if ( decision == overrideDecision && _input.index() == overrideDecisionInputIndex ) {
edge = overrideDecisionAlt;
}
else {
edge = getInterpreter().adaptivePredict(_input, decision, _ctx);
}
}
else {
edge = 1;
@ -225,63 +264,67 @@ public class ParserInterpreter extends Parser {
Transition transition = p.transition(edge - 1);
switch (transition.getSerializationType()) {
case Transition.EPSILON:
if (pushRecursionContextStates.get(p.stateNumber) && !(transition.target instanceof LoopEndState)) {
InterpreterRuleContext ctx = new InterpreterRuleContext(_parentContextStack.peek().a, _parentContextStack.peek().b, _ctx.getRuleIndex());
pushNewRecursionContext(ctx, atn.ruleToStartState[p.ruleIndex].stateNumber, _ctx.getRuleIndex());
}
break;
case Transition.EPSILON:
if ( statesNeedingLeftRecursionContext.get(p.stateNumber) &&
!(transition.target instanceof LoopEndState))
{
// We are at the start of a left recursive rule's (...)* loop
// but it's not the exit branch of loop.
InterpreterRuleContext ctx = new InterpreterRuleContext(_parentContextStack.peek().a, _parentContextStack.peek().b, _ctx.getRuleIndex());
pushNewRecursionContext(ctx, atn.ruleToStartState[p.ruleIndex].stateNumber, _ctx.getRuleIndex());
}
break;
case Transition.ATOM:
match(((AtomTransition)transition).label);
break;
case Transition.ATOM:
match(((AtomTransition)transition).label);
break;
case Transition.RANGE:
case Transition.SET:
case Transition.NOT_SET:
if (!transition.matches(_input.LA(1), Token.MIN_USER_TOKEN_TYPE, 65535)) {
_errHandler.recoverInline(this);
}
matchWildcard();
break;
case Transition.RANGE:
case Transition.SET:
case Transition.NOT_SET:
if (!transition.matches(_input.LA(1), Token.MIN_USER_TOKEN_TYPE, 65535)) {
_errHandler.recoverInline(this);
}
matchWildcard();
break;
case Transition.WILDCARD:
matchWildcard();
break;
case Transition.WILDCARD:
matchWildcard();
break;
case Transition.RULE:
RuleStartState ruleStartState = (RuleStartState)transition.target;
int ruleIndex = ruleStartState.ruleIndex;
InterpreterRuleContext ctx = new InterpreterRuleContext(_ctx, p.stateNumber, ruleIndex);
if (ruleStartState.isPrecedenceRule) {
enterRecursionRule(ctx, ruleStartState.stateNumber, ruleIndex, ((RuleTransition)transition).precedence);
}
else {
enterRule(ctx, transition.target.stateNumber, ruleIndex);
}
break;
case Transition.RULE:
RuleStartState ruleStartState = (RuleStartState)transition.target;
int ruleIndex = ruleStartState.ruleIndex;
InterpreterRuleContext ctx = new InterpreterRuleContext(_ctx, p.stateNumber, ruleIndex);
if (ruleStartState.isPrecedenceRule) {
enterRecursionRule(ctx, ruleStartState.stateNumber, ruleIndex, ((RuleTransition)transition).precedence);
}
else {
enterRule(ctx, transition.target.stateNumber, ruleIndex);
}
break;
case Transition.PREDICATE:
PredicateTransition predicateTransition = (PredicateTransition)transition;
if (!sempred(_ctx, predicateTransition.ruleIndex, predicateTransition.predIndex)) {
throw new FailedPredicateException(this);
}
case Transition.PREDICATE:
PredicateTransition predicateTransition = (PredicateTransition)transition;
if (!sempred(_ctx, predicateTransition.ruleIndex, predicateTransition.predIndex)) {
throw new FailedPredicateException(this);
}
break;
break;
case Transition.ACTION:
ActionTransition actionTransition = (ActionTransition)transition;
action(_ctx, actionTransition.ruleIndex, actionTransition.actionIndex);
break;
case Transition.ACTION:
ActionTransition actionTransition = (ActionTransition)transition;
action(_ctx, actionTransition.ruleIndex, actionTransition.actionIndex);
break;
case Transition.PRECEDENCE:
if (!precpred(_ctx, ((PrecedencePredicateTransition)transition).precedence)) {
throw new FailedPredicateException(this, String.format("precpred(_ctx, %d)", ((PrecedencePredicateTransition)transition).precedence));
}
break;
case Transition.PRECEDENCE:
if (!precpred(_ctx, ((PrecedencePredicateTransition)transition).precedence)) {
throw new FailedPredicateException(this, String.format("precpred(_ctx, %d)", ((PrecedencePredicateTransition)transition).precedence));
}
break;
default:
throw new UnsupportedOperationException("Unrecognized ATN transition type.");
default:
throw new UnsupportedOperationException("Unrecognized ATN transition type.");
}
setState(transition.target.stateNumber);
@ -301,4 +344,50 @@ public class ParserInterpreter extends Parser {
RuleTransition ruleTransition = (RuleTransition)atn.states.get(getState()).transition(0);
setState(ruleTransition.followState.stateNumber);
}
/** Override this parser interpreters normal decision-making process
* at a particular decision and input token index. Instead of
* allowing the adaptive prediction mechanism to choose the
* first alternative within a block that leads to a successful parse,
* force it to take the alternative, 1..n for n alternatives.
*
* As an implementation limitation right now, you can only specify one
* override. This is sufficient to allow construction of different
* parse trees for ambiguous input. It means re-parsing the entire input
* in general because you're never sure where an ambiguous sequence would
* live in the various parse trees. For example, in one interpretation,
* an ambiguous input sequence would be matched completely in expression
* but in another it could match all the way back to the root.
*
* s : e '!'? ;
* e : ID
* | ID '!'
* ;
*
* Here, x! can be matched as (s (e ID) !) or (s (e ID !)). In the first
* case, the ambiguous sequence is fully contained only by the root.
* In the second case, the ambiguous sequences fully contained within just
* e, as in: (e ID !).
*
* Rather than trying to optimize this and make
* some intelligent decisions for optimization purposes, I settled on
* just re-parsing the whole input and then using
* {link Trees#getRootOfSubtreeEnclosingRegion} to find the minimal
* subtree that contains the ambiguous sequence. I originally tried to
* record the call stack at the point the parser detected and ambiguity but
* left recursive rules create a parse tree stack that does not reflect
* the actual call stack. That impedance mismatch was enough to make
* it it challenging to restart the parser at a deeply nested rule
* invocation.
*
* Only parser interpreters can override decisions so as to avoid inserting
* override checking code in the critical ALL(*) prediction execution path.
*
* @since 4.5.1
*/
public void addDecisionOverride(int decision, int tokenIndex, int forcedAlt) {
overrideDecision = decision;
overrideDecisionInputIndex = tokenIndex;
overrideDecisionAlt = forcedAlt;
}
}

10
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
View File

@ -45,9 +45,8 @@ import java.util.List;
*
* Contains all of the information about the current rule not stored in the
* RuleContext. It handles parse tree children list, Any ATN state
* tracing, and the default values available for rule indications:
* start, stop, rule index, current alt number, current
* ATN state.
* tracing, and the default values available for rule invocations:
* start, stop, rule index, current alt number.
*
* Subclasses made for each rule and grammar track the parameters,
* return values, locals, and labels specific to that rule. These
@ -103,9 +102,10 @@ public class ParserRuleContext extends RuleContext {
public ParserRuleContext() { }
/** COPY a ctx (I'm deliberately not using copy constructor) */
/** COPY a ctx (I'm deliberately not using copy constructor) to avoid
* confusion with creating node with parent. Does not copy children.
*/
public void copyFrom(ParserRuleContext ctx) {
// from RuleContext
this.parent = ctx.parent;
this.invokingState = ctx.invokingState;

55
runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
View File

@ -43,16 +43,46 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Future;
/** A rule context is a record of a single rule invocation. It knows
* which context invoked it, if any. If there is no parent context, then
* naturally the invoking state is not valid. The parent link
* provides a chain upwards from the current rule invocation to the root
* of the invocation tree, forming a stack. We actually carry no
* information about the rule associated with this context (except
* when parsing). We keep only the state number of the invoking state from
* the ATN submachine that invoked this. Contrast this with the s
* pointer inside ParserRuleContext that tracks the current state
* being "executed" for the current rule.
/** /** A rule context is a record of a single rule invocation.
*
* We form a stack of these context objects using the parent
* pointer. A parent pointer of null indicates that the current
* context is the bottom of the stack. The ParserRuleContext subclass
* as a children list so that we can turn this data structure into a
* tree.
*
* The root node always has a null pointer and invokingState of -1.
*
* Upon entry to parsing, the first invoked rule function creates a
* context object (asubclass specialized for that rule such as
* SContext) and makes it the root of a parse tree, recorded by field
* Parser._ctx.
*
* public final SContext s() throws RecognitionException {
* SContext _localctx = new SContext(_ctx, getState()); <-- create new node
* enterRule(_localctx, 0, RULE_s); <-- push it
* ...
* exitRule(); <-- pop back to _localctx
* return _localctx;
* }
*
* A subsequent rule invocation of r from the start rule s pushes a
* new context object for r whose parent points at s and use invoking
* state is the state with r emanating as edge label.
*
* The invokingState fields from a context object to the root
* together form a stack of rule indication states where the root
* (bottom of the stack) has a -1 sentinel value. If we invoke start
* symbol s then call r1, which calls r2, the would look like
* this:
*
* SContext[-1] <- root node (bottom of the stack)
* R1Context[p] <- p in rule s called r1
* R2Context[q] <- q in rule r1 called r2
*
* So the top of the stack, _ctx, represents a call to the current
* rule and it holds the return address from another rule that invoke
* to this rule. To invoke a rule, we must always have a current context.
*
* The parent contexts are useful for computing lookahead sets and
* getting error information.
@ -71,7 +101,8 @@ public class RuleContext implements RuleNode {
/** What state invoked the rule associated with this context?
* The "return address" is the followState of invokingState
* If parent is null, this should be -1.
* If parent is null, this should be -1 this context object represents
* the start rule.
*/
public int invokingState = -1;
@ -93,7 +124,7 @@ public class RuleContext implements RuleNode {
return n;
}
/** A context is empty if there is no invoking state; meaning nobody call
/** A context is empty if there is no invoking state; meaning nobody called
* current context.
*/
public boolean isEmpty() {

6
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java
View File

@ -33,7 +33,6 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.AbstractEqualityComparator;
import org.antlr.v4.runtime.misc.Array2DHashSet;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import org.antlr.v4.runtime.misc.NotNull;
import java.util.ArrayList;
import java.util.BitSet;
@ -107,6 +106,11 @@ public class ATNConfigSet implements Set<ATNConfig> {
// TODO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation
// TODO: can we track conflicts as they are added to save scanning configs later?
public int uniqueAlt;
/** Currently this is only used when we detect SLL conflict; this does
* not necessarily represent the ambiguous alternatives. In fact,
* I should also point out that this seems to include predicated alternatives
* that have predicates that evaluate to false. Computed in computeTargetState().
*/
protected BitSet conflictingAlts;
// Used in parser and lexer. In lexer, it indicates we hit a pred

14
runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java
View File

@ -33,6 +33,8 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.ANTLRErrorListener;
import org.antlr.v4.runtime.TokenStream;
import java.util.BitSet;
/**
* This class represents profiling event information for an ambiguity.
* Ambiguities are decisions where a particular input resulted in an SLL
@ -63,6 +65,9 @@ import org.antlr.v4.runtime.TokenStream;
* @since 4.3
*/
public class AmbiguityInfo extends DecisionEventInfo {
/** The set of alternative numbers for this decision event that lead to a valid parse. */
public BitSet ambigAlts;
/**
* Constructs a new instance of the {@link AmbiguityInfo} class with the
* specified detailed ambiguity information.
@ -70,6 +75,7 @@ public class AmbiguityInfo extends DecisionEventInfo {
* @param decision The decision number
* @param configs The final configuration set identifying the ambiguous
* alternatives for the current input
* @param ambigAlts The set of alternatives in the decision that lead to a valid parse.
* @param input The input token stream
* @param startIndex The start index for the current prediction
* @param stopIndex The index at which the ambiguity was identified during
@ -78,9 +84,13 @@ public class AmbiguityInfo extends DecisionEventInfo {
* prediction; otherwise, {@code false} if the ambiguity was identified
* during SLL prediction
*/
public AmbiguityInfo(int decision, ATNConfigSet configs, TokenStream input,
int startIndex, int stopIndex, boolean fullCtx)
public AmbiguityInfo(int decision,
ATNConfigSet configs,
BitSet ambigAlts,
TokenStream input, int startIndex, int stopIndex,
boolean fullCtx)
{
super(decision, configs, input, startIndex, stopIndex, fullCtx);
this.ambigAlts = ambigAlts;
}
}

2
runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java
View File

@ -35,10 +35,8 @@ package org.antlr.v4.runtime.atn;
* @author Sam Harwell
*/
public final class BasicBlockStartState extends BlockStartState {
@Override
public int getStateType() {
return BLOCK_START;
}
}

5
runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java
View File

@ -64,9 +64,10 @@ public class ContextSensitivityInfo extends DecisionEventInfo {
* @param stopIndex The index at which the context sensitivity was
* identified during full-context prediction
*/
public ContextSensitivityInfo(int decision, ATNConfigSet configs,
public ContextSensitivityInfo(int decision,
ATNConfigSet configs,
TokenStream input, int startIndex, int stopIndex)
{
super(decision, configs, input, startIndex, stopIndex, true);
super( decision, configs, input, startIndex, stopIndex, true);
}
}

18
runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java
View File

@ -31,13 +31,20 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
/**
* This is the base class for gathering detailed information about prediction
* events which occur during parsing.
*
* Note that we could record the parser call stack at the time this event
* occurred but in the presence of left recursive rules, the stack is kind of
* meaningless. It's better to look at the individual configurations for their
* individual stacks. Of course that is a {@link PredictionContext} object
* not a parse tree node and so it does not have information about the extent
* (start...stop) of the various subtrees. Examining the stack tops of all
* configurations provide the return states for the rule invocations.
* From there you can get the enclosing rule.
*
* @since 4.3
*/
public class DecisionEventInfo {
@ -77,9 +84,10 @@ public class DecisionEventInfo {
*/
public final boolean fullCtx;
public DecisionEventInfo(int decision, ATNConfigSet configs,
TokenStream input, int startIndex,
int stopIndex, boolean fullCtx)
public DecisionEventInfo(int decision,
ATNConfigSet configs,
TokenStream input, int startIndex, int stopIndex,
boolean fullCtx)
{
this.decision = decision;
this.fullCtx = fullCtx;

6
runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java
View File

@ -62,8 +62,10 @@ public class ErrorInfo extends DecisionEventInfo {
* prediction; otherwise, {@code false} if the syntax error was identified
* during SLL prediction
*/
public ErrorInfo(int decision, ATNConfigSet configs, TokenStream input,
int startIndex, int stopIndex, boolean fullCtx)
public ErrorInfo(int decision,
ATNConfigSet configs,
TokenStream input, int startIndex, int stopIndex,
boolean fullCtx)
{
super(decision, configs, input, startIndex, stopIndex, fullCtx);
}

3
runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java
View File

@ -54,7 +54,8 @@ public class LookaheadEventInfo extends DecisionEventInfo {
* prediction; otherwise, {@code false} if the current lookahead is part of
* an SLL prediction
*/
public LookaheadEventInfo(int decision, ATNConfigSet configs,
public LookaheadEventInfo(int decision,
ATNConfigSet configs,
TokenStream input, int startIndex, int stopIndex,
boolean fullCtx)
{

14
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -46,8 +46,6 @@ import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.Pair;
import java.util.ArrayList;
@ -662,7 +660,7 @@ public class ParserATNSimulator extends ATNSimulator {
// comes back with reach.uniqueAlt set to a valid alt
protected int execATNWithFullContext(DFA dfa,
DFAState D, // how far we got before failing over
DFAState D, // how far we got in SLL DFA before failing over
ATNConfigSet s0,
TokenStream input, int startIndex,
ParserRuleContext outerContext)
@ -710,6 +708,7 @@ public class ParserATNSimulator extends ATNSimulator {
}
// System.out.println("altSubSets: "+altSubSets);
// System.err.println("reach="+reach+", "+reach.conflictingAlts);
reach.uniqueAlt = getUniqueAlt(reach);
// unique prediction?
if ( reach.uniqueAlt!=ATN.INVALID_ALT_NUMBER ) {
@ -779,7 +778,8 @@ public class ParserATNSimulator extends ATNSimulator {
the fact that we should predict alternative 1. We just can't say for
sure that there is an ambiguity without looking further.
*/
reportAmbiguity(dfa, D, startIndex, input.index(), foundExactAmbig, null, reach);
reportAmbiguity(dfa, D, startIndex, input.index(), foundExactAmbig,
reach.getAlts(), reach);
return predictedAlt;
}
@ -2006,10 +2006,12 @@ public class ParserATNSimulator extends ATNSimulator {
}
/** If context sensitive parsing, we know it's ambiguity not conflict */
protected void reportAmbiguity(DFA dfa, DFAState D, int startIndex, int stopIndex,
protected void reportAmbiguity(DFA dfa,
DFAState D, // the DFA state from execATN() that had SLL conflicts
int startIndex, int stopIndex,
boolean exact,
BitSet ambigAlts,
ATNConfigSet configs)
ATNConfigSet configs) // configs that LL not SLL considered conflicting
{
if ( debug || retry_debug ) {
Interval interval = Interval.of(startIndex, stopIndex);

10
runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java
View File

@ -33,7 +33,6 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.AbstractEqualityComparator;
import org.antlr.v4.runtime.misc.FlexibleHashMap;
import org.antlr.v4.runtime.misc.MurmurHash;
import org.antlr.v4.runtime.misc.NotNull;
import java.util.BitSet;
import java.util.Collection;
@ -546,6 +545,15 @@ public enum PredictionMode {
return all;
}
/** Get union of all alts from configs. @since 4.5.1 */
public static BitSet getAlts(ATNConfigSet configs) {
BitSet alts = new BitSet();
for (ATNConfig config : configs) {
alts.set(config.alt);
}
return alts;
}
/**
* This function gets the conflicting alt subsets from a configuration set.
* For each configuration {@code c} in {@code configs}:

3
runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java
View File

@ -233,7 +233,8 @@ public class ProfilingATNSimulator extends ParserATNSimulator {
);
}
decisions[currentDecision].ambiguities.add(
new AmbiguityInfo(currentDecision, configs, _input, startIndex, stopIndex, configs.fullCtx)
new AmbiguityInfo(currentDecision, configs, ambigAlts,
_input, startIndex, stopIndex, configs.fullCtx)
);
super.reportAmbiguity(dfa, D, startIndex, stopIndex, exact, ambigAlts, configs);
}

2
runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java
View File

@ -32,7 +32,7 @@ package org.antlr.v4.runtime.atn;
public final class RuleStartState extends ATNState {
public RuleStopState stopState;
public boolean isPrecedenceRule;
public boolean isPrecedenceRule; //Synonymous with rule being left recursive; consider renaming.
@Override
public int getStateType() {

11
runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java
View File

@ -40,6 +40,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@ -201,4 +202,14 @@ public class Utils {
}
return cdata;
}
public static IntervalSet toSet(BitSet bits) {
IntervalSet s = new IntervalSet();
int i = bits.nextSetBit(0);
while ( i >= 0 ) {
s.add(i);
i = bits.nextSetBit(i+1);
}
return s;
}
}

27
runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java
View File

@ -33,7 +33,6 @@ package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.runtime.tree.gui.TreePostScriptGenerator;
@ -221,6 +220,32 @@ public class Trees {
return nodes;
}
/** Find smallest subtree of t enclosing range startTokenIndex..stopTokenIndex
* inclusively using postorder traversal. Recursive depth-first-search.
*
* @since 4.5.1
*/
public static ParserRuleContext getRootOfSubtreeEnclosingRegion(ParseTree t,
int startTokenIndex, // inclusive
int stopTokenIndex) // inclusive
{
int n = t.getChildCount();
for (int i = 0; i<n; i++) {
ParseTree child = t.getChild(i);
ParserRuleContext r = getRootOfSubtreeEnclosingRegion(child, startTokenIndex, stopTokenIndex);
if ( r!=null ) return r;
}
if ( t instanceof ParserRuleContext ) {
ParserRuleContext r = (ParserRuleContext) t;
if ( startTokenIndex>=r.getStart().getTokenIndex() && // is range fully contained in t?
stopTokenIndex<=r.getStop().getTokenIndex() )
{
return r;
}
}
return null;
}
private Trees() {
}
}

2
tool/test/org/antlr/v4/test/rt/java/TestParserErrors.java
View File

@ -333,4 +333,4 @@ public class TestParserErrors extends BaseTest {
}
}
}

257
tool/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java Normal file
View File

@ -0,0 +1,257 @@
package org.antlr.v4.test.tool;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.LexerInterpreter;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserInterpreter;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.AmbiguityInfo;
import org.antlr.v4.runtime.atn.BasicBlockStartState;
import org.antlr.v4.runtime.atn.DecisionInfo;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.atn.RuleStartState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.junit.Test;
import java.util.List;
import static org.junit.Assert.assertEquals;
public class TestAmbigParseTrees {
@Test public void testParseDecisionWithinAmbiguousStartRule() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
"s : A x C" +
" | A B C" +
" ;" +
"x : B ; \n",
lg);
testInterpAtSpecificAlt(lg, g, "s", 1, "abc", "(s a (x b) c)");
testInterpAtSpecificAlt(lg, g, "s", 2, "abc", "(s a b c)");
}
@Test public void testAmbigAltsAtRoot() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
"s : A x C" +
" | A B C" +
" ;" +
"x : B ; \n",
lg);
String startRule = "s";
String input = "abc";
String expectedAmbigAlts = "{1, 2}";
int decision = 0;
String expectedOverallTree = "(s a (x b) c)";
String[] expectedParseTrees = {"(s a (x b) c)","(s a b c)"};
testInterp(lg, g, startRule, input, decision,
expectedAmbigAlts,
expectedOverallTree, expectedParseTrees);
}
@Test public void testAmbigAltsNotAtRoot() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
"s : a ;" +
"a : b ;" +
"b : A x C" +
" | A B C" +
" ;" +
"x : B ; \n",
lg);
String startRule = "s";
String input = "abc";
String expectedAmbigAlts = "{1, 2}";
int decision = 0;
String expectedOverallTree = "(s (a (b a (x b) c)))";
String[] expectedParseTrees = {"(b a (x b) c)","(b a b c)"};
testInterp(lg, g, startRule, input, decision,
expectedAmbigAlts,
expectedOverallTree, expectedParseTrees);
}
@Test public void testAmbigAltDipsIntoOuterContextToRoot() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"SELF : 'self' ;\n" +
"ID : [a-z]+ ;\n" +
"DOT : '.' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
// "s : e ;\n"+
"e : p (DOT ID)* ;\n"+
"p : SELF" +
" | SELF DOT ID" +
" ;",
lg);
String startRule = "e";
String input = "self.x";
String expectedAmbigAlts = "{1, 2}";
int decision = 1; // decision in s
String expectedOverallTree = "(e (p self) . x)";
String[] expectedParseTrees = {"(e (p self) . x)","(p self . x)"};
testInterp(lg, g, startRule, input, decision,
expectedAmbigAlts,
expectedOverallTree, expectedParseTrees);
}
@Test public void testAmbigAltDipsIntoOuterContextBelowRoot() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"SELF : 'self' ;\n" +
"ID : [a-z]+ ;\n" +
"DOT : '.' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
"s : e ;\n"+
"e : p (DOT ID)* ;\n"+
"p : SELF" +
" | SELF DOT ID" +
" ;",
lg);
String startRule = "s";
String input = "self.x";
String expectedAmbigAlts = "{1, 2}";
int decision = 1; // decision in s
String expectedOverallTree = "(s (e (p self) . x))";
String[] expectedParseTrees = {"(e (p self) . x)","(p self . x)"};
testInterp(lg, g, startRule, input, decision,
expectedAmbigAlts,
expectedOverallTree, expectedParseTrees);
}
@Test public void testAmbigAltInLeftRecursiveBelowStartRule() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"SELF : 'self' ;\n" +
"ID : [a-z]+ ;\n" +
"DOT : '.' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
"s : e ;\n" +
"e : p | e DOT ID ;\n"+
"p : SELF" +
" | SELF DOT ID" +
" ;",
lg);
String startRule = "s";
String input = "self.x";
String expectedAmbigAlts = "{1, 2}";
int decision = 1; // decision in s
String expectedOverallTree = "(s (e (e (p self)) . x))";
String[] expectedParseTrees = {"(e (e (p self)) . x)","(p self . x)"};
testInterp(lg, g, startRule, input, decision,
expectedAmbigAlts,
expectedOverallTree, expectedParseTrees);
}
@Test public void testAmbigAltInLeftRecursiveStartRule() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"SELF : 'self' ;\n" +
"ID : [a-z]+ ;\n" +
"DOT : '.' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
"e : p | e DOT ID ;\n"+
"p : SELF" +
" | SELF DOT ID" +
" ;",
lg);
String startRule = "e";
String input = "self.x";
String expectedAmbigAlts = "{1, 2}";
int decision = 1; // decision in s
String expectedOverallTree = "(e (e (p self)) . x)";
String[] expectedParseTrees = {"(e (e (p self)) . x)","(p self . x)"};
testInterp(lg, g, startRule, input, decision,
expectedAmbigAlts,
expectedOverallTree, expectedParseTrees);
}
public void testInterp(LexerGrammar lg, Grammar g,
String startRule, String input, int decision,
String expectedAmbigAlts,
String overallTree,
String[] expectedParseTrees)
{
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
final ParserInterpreter parser = g.createParserInterpreter(tokens);
parser.setProfile(true);
parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
// PARSE
int ruleIndex = g.rules.get(startRule).index;
ParserRuleContext parseTree = parser.parse(ruleIndex);
assertEquals(overallTree, parseTree.toStringTree(parser));
System.out.println();
DecisionInfo[] decisionInfo = parser.getParseInfo().getDecisionInfo();
List<AmbiguityInfo> ambiguities = decisionInfo[decision].ambiguities;
assertEquals(1, ambiguities.size());
AmbiguityInfo ambiguityInfo = ambiguities.get(0);
List<ParserRuleContext> ambiguousParseTrees = Parser.getAmbiguousParseTrees(parser, ambiguityInfo, ruleIndex);
assertEquals(expectedAmbigAlts, ambiguityInfo.ambigAlts.toString());
assertEquals(ambiguityInfo.ambigAlts.cardinality(), ambiguousParseTrees.size());
for (int i = 0; i<ambiguousParseTrees.size(); i++) {
ParserRuleContext t = ambiguousParseTrees.get(i);
assertEquals(expectedParseTrees[i], t.toStringTree(parser));
}
}
void testInterpAtSpecificAlt(LexerGrammar lg, Grammar g,
String startRule, int startAlt,
String input,
String expectedParseTree)
{
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
ParserInterpreter parser = g.createParserInterpreter(tokens);
RuleStartState ruleStartState = g.atn.ruleToStartState[g.getRule(startRule).index];
Transition tr = ruleStartState.transition(0);
ATNState t2 = tr.target;
if ( !(t2 instanceof BasicBlockStartState) ) {
throw new IllegalArgumentException("rule has no decision: "+startRule);
}
parser.addDecisionOverride(((DecisionState)t2).decision, 0, startAlt);
ParseTree t = parser.parse(g.rules.get(startRule).index);
assertEquals(expectedParseTree, t.toStringTree(parser));
}
}

17
tool/test/org/antlr/v4/test/tool/TestParserInterpreter.java
View File

@ -43,6 +43,19 @@ import static org.junit.Assert.assertEquals;
public class TestParserInterpreter extends BaseTest {
@Test public void testEmptyStartRule() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n" +
"s : ;",
lg);
testInterp(lg, g, "s", "", "s");
testInterp(lg, g, "s", "a", "s");
}
@Test public void testA() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
@ -223,13 +236,13 @@ public class TestParserInterpreter extends BaseTest {
void testInterp(LexerGrammar lg, Grammar g,
String startRule, String input,
String parseTree)
String expectedParseTree)
{
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
ParserInterpreter parser = g.createParserInterpreter(tokens);
ParseTree t = parser.parse(g.rules.get(startRule).index);
System.out.println("parse tree: "+t.toStringTree(parser));
assertEquals(parseTree, t.toStringTree(parser));
assertEquals(expectedParseTree, t.toStringTree(parser));
}
}