jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Reorganized to pass a proposed NFA configuration to closure instead of all those parameters. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6781]
This commit is contained in:
parrt 2010-03-31 15:49:19 -08:00
parent e9fd3d8b8c
commit a6f0d43a36
9 changed files with 119 additions and 813 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
tool/src/org/antlr/v4/analysis/LexerNFAToDFAConverter.java
View File

@ -68,9 +68,10 @@ public class LexerNFAToDFAConverter {
for (int ruleIndex=1; ruleIndex<=dfa.nAlts; ruleIndex++) {
Transition t = dfa.decisionNFAStartState.transition(ruleIndex-1);
NFAState altStart = t.target;
d.addNFAConfig(altStart, ruleIndex,
NFAContext.EMPTY,
SemanticContext.EMPTY_SEMANTIC_CONTEXT);
d.addNFAConfig(
new NFAConfig(altStart, ruleIndex,
NFAContext.EMPTY(),
SemanticContext.EMPTY_SEMANTIC_CONTEXT));
}
closure(d);
@ -130,8 +131,8 @@ public class LexerNFAToDFAConverter {
// found a transition with label; does it collide with label?
if ( !t.isEpsilon() && !t.label().and(label).isNil() ) {
// add NFA target to (potentially) new DFA state
labelTarget.addNFAConfig(t.target, c.alt, c.context,
SemanticContext.EMPTY_SEMANTIC_CONTEXT);
labelTarget.addNFAConfig(
new NFAConfig(c, t.target, SemanticContext.EMPTY_SEMANTIC_CONTEXT));
}
}
}
@ -162,6 +163,7 @@ public class LexerNFAToDFAConverter {
//System.out.println("after closure d="+d);
}
// TODO: make pass NFAConfig like other DFA
public void closure(LexerState d, NFAState s, int ruleIndex, NFAContext context) {
NFAConfig proposedNFAConfig =
new NFAConfig(s, ruleIndex, context, SemanticContext.EMPTY_SEMANTIC_CONTEXT);
@ -174,7 +176,7 @@ public class LexerNFAToDFAConverter {
if ( s instanceof RuleStopState ) {
// TODO: chase FOLLOW links if recursive
if ( context!=NFAContext.EMPTY ) {
if ( !context.isEmpty() ) {
closure(d, context.returnState, ruleIndex, context.parent);
// do nothing if context not empty and already added to nfaStates
}

7
tool/src/org/antlr/v4/analysis/LinearApproximator.java
View File

@ -163,7 +163,7 @@ public class LinearApproximator {
configs[i] = new OrderedHashSet<NFAConfig>();
}
_LOOK(s, alt, k, NFAContext.EMPTY);
_LOOK(s, alt, k, NFAContext.EMPTY());
return look;
}
@ -173,7 +173,7 @@ public class LinearApproximator {
if ( lookBusy.contains(ac) ) return;
lookBusy.add(ac);
if ( s instanceof RuleStopState && context!=NFAContext.EMPTY ) {
if ( s instanceof RuleStopState && !context.isEmpty() ) {
_LOOK(context.returnState, alt, k, context.parent);
return;
}
@ -192,7 +192,8 @@ public class LinearApproximator {
else {
//System.out.println("adding "+ t.label().toString(g) +" @ i="+(MAX_K-k+1));
look[max_k-k+1].addAll( t.label() );
NFAConfig c = new NFAConfig(t.target, alt, context, SemanticContext.EMPTY_SEMANTIC_CONTEXT);
NFAConfig c = new NFAConfig(t.target, alt, context,
SemanticContext.EMPTY_SEMANTIC_CONTEXT);
configs[max_k-k+1].add(c);
if ( k>1 ) _LOOK(t.target, alt, k-1, context);
}

33
tool/src/org/antlr/v4/analysis/NFAConfig.java
View File

@ -40,7 +40,7 @@ public class NFAConfig {
* nondeterministic configurations (as it does for "resolved" field)
* that have enough predicates to resolve the conflit.
*/
protected boolean resolvedWithPredicate;
boolean resolvedWithPredicate;
public NFAConfig(NFAState state,
int alt,
@ -53,6 +53,35 @@ public class NFAConfig {
this.semanticContext = semanticContext;
}
public NFAConfig(NFAConfig c) {
this.state = c.state;
this.alt = c.alt;
this.context = c.context;
this.semanticContext = c.semanticContext;
}
public NFAConfig(NFAConfig c, NFAState state) {
this(c);
this.state = state;
}
public NFAConfig(NFAConfig c, NFAState state, NFAContext context) {
this(c);
this.state = state;
this.context = context;
}
public NFAConfig(NFAConfig c, NFAContext context) {
this(c);
this.context = context;
}
public NFAConfig(NFAConfig c, NFAState state, SemanticContext semanticContext) {
this(c);
this.state = state;
this.semanticContext = semanticContext;
}
/** An NFA configuration is equal to another if both have
* the same state, they predict the same alternative, and
* syntactic/semantic contexts are the same. I don't think
@ -89,7 +118,7 @@ public class NFAConfig {
buf.append("|");
buf.append(alt);
}
if ( context!=null && context!= NFAContext.EMPTY) {
if ( context!=null && !context.isEmpty() ) {
buf.append("|");
buf.append(context);
}

10
tool/src/org/antlr/v4/analysis/NFAContext.java
View File

@ -46,8 +46,6 @@ import org.antlr.v4.automata.NFAState;
* on the path from this node thru the parent pointers to the root.
*/
public class NFAContext {
public static final NFAContext EMPTY = new NFAContext(null, null);
public NFAContext parent;
/** The NFA state following state that invoked another rule's start state
@ -55,6 +53,9 @@ public class NFAContext {
*/
public NFAState returnState;
/** Indicates this config led to recursive closure request. Everything
* derived from here is approximation.
*/
public boolean recursed;
/** Computing the hashCode is very expensive and closureBusy()
@ -79,6 +80,9 @@ public class NFAContext {
}
}
public static NFAContext EMPTY() { return new NFAContext(null, null); }
/** Is s anywhere in the context? */
public boolean contains(NFAState s) {
NFAContext sp = this;
@ -214,7 +218,7 @@ public class NFAContext {
public int depth() {
int n = 0;
NFAContext sp = this;
while ( sp != EMPTY) {
while ( !sp.isEmpty() ) {
n++;
sp = sp.parent;
}

141
tool/src/org/antlr/v4/analysis/PredictionDFAFactory.java
View File

@ -146,14 +146,14 @@ public class PredictionDFAFactory {
try {
closure(t); // add any NFA states reachable via epsilon
}
catch (RecursionOverflowSignal ros) {
recursionOverflowState = d;
ErrorManager.recursionOverflow(g.fileName, d, ros.state, ros.altNum, ros.depth);
}
catch (MultipleRecursiveAltsSignal mras) {
abortedDueToMultipleRecursiveAltsAt = d;
ErrorManager.multipleRecursiveAlts(g.fileName, d, mras.recursiveAltSet);
}
// catch (RecursionOverflowSignal ros) {
// recursionOverflowState = d;
// ErrorManager.recursionOverflow(g.fileName, d, ros.state, ros.altNum, ros.depth);
// }
// catch (MultipleRecursiveAltsSignal mras) {
// abortedDueToMultipleRecursiveAltsAt = d;
// ErrorManager.multipleRecursiveAlts(g.fileName, d, mras.recursiveAltSet);
// }
catch (AnalysisTimeoutSignal at) {// TODO: nobody throws yet
ErrorManager.analysisTimeout();
}
@ -177,7 +177,7 @@ public class PredictionDFAFactory {
// by using semantic predicates if present.
resolver.resolveAmbiguities(t);
// If deterministic, don't add this state; it's an accept state
// If deterministic, don't add this state to work list; it's an accept state
// Just return as a valid DFA state
int alt = t.getUniquelyPredictedAlt();
if ( alt > 0 ) { // uniquely predicts an alt?
@ -222,7 +222,7 @@ public class PredictionDFAFactory {
// found a transition with label; does it collide with label?
if ( !t.isEpsilon() && !t.label().and(label).isNil() ) {
// add NFA target to (potentially) new DFA state
labelTarget.addNFAConfig(t.target, c.alt, c.context, c.semanticContext);
labelTarget.addNFAConfig(new NFAConfig(c, t.target));
}
}
}
@ -249,9 +249,10 @@ public class PredictionDFAFactory {
for (int altNum=1; altNum<=dfa.nAlts; altNum++) {
Transition t = nfaStartState.transition(altNum-1);
NFAState altStart = t.target;
d.addNFAConfig(altStart, altNum,
NFAContext.EMPTY,
SemanticContext.EMPTY_SEMANTIC_CONTEXT);
d.addNFAConfig(
new NFAConfig(altStart, altNum,
NFAContext.EMPTY(),
SemanticContext.EMPTY_SEMANTIC_CONTEXT));
}
closure(d);
@ -275,7 +276,7 @@ public class PredictionDFAFactory {
// OH: concurrent modification. dup initialconfigs? works for lexers, try here to save configs param
List<NFAConfig> configs = new ArrayList<NFAConfig>();
for (NFAConfig c : d.nfaConfigs) {
closure(c.state, c.alt, c.context, c.semanticContext, collectPredicates, configs);
closure(c, collectPredicates, configs);
}
d.nfaConfigs.addAll(configs); // Add new NFA configs to DFA state d
@ -303,76 +304,70 @@ public class PredictionDFAFactory {
* I check for left-recursive stuff and terminate before analysis to
* avoid need to do this more expensive computation.
*
* This
*
* TODO: remove altNum if we don't reorder for loopback nodes
* TODO: pass in a config?
*/
public void closure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext,
boolean collectPredicates,
List<NFAConfig> configs)
{
NFAConfig proposedNFAConfig = new NFAConfig(s, altNum, context, semanticContext);
public void closure(NFAConfig c, boolean collectPredicates, List<NFAConfig> configs) {
//NFAConfig proposedNFAConfig = new NFAConfig(s, altNum, context, semanticContext);
if ( closureBusy.contains(proposedNFAConfig) ) return;
closureBusy.add(proposedNFAConfig);
if ( closureBusy.contains(c) ) return;
closureBusy.add(c);
// p itself is always in closure
configs.add(proposedNFAConfig);
configs.add(c);
// if ( s instanceof RuleStartState ) {
// visited.add(s.rule.index);
// }
if ( s instanceof RuleStopState ) {
ruleStopStateClosure(s, altNum, context, semanticContext, collectPredicates, configs);
if ( c.state instanceof RuleStopState ) {
ruleStopStateClosure(c, collectPredicates, configs);
}
else {
commonClosure(s, altNum, context, semanticContext, collectPredicates, configs);
commonClosure(c, collectPredicates, configs);
}
}
// if we have context info and we're at rule stop state, do
// local follow for invokingRule and global follow for other links
void ruleStopStateClosure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext,
boolean collectPredicates,
List<NFAConfig> configs)
{
if ( !context.recursed ) {
System.out.println("dynamic FOLLOW of "+s+" context="+context);
if ( context != NFAContext.EMPTY) {
NFAContext newContext = context.parent; // "pop" invoking state
closure(context.returnState, altNum, newContext, semanticContext, collectPredicates, configs);
void ruleStopStateClosure(NFAConfig c, boolean collectPredicates, List<NFAConfig> configs) {
if ( !c.context.recursed ) {
System.out.println("dynamic FOLLOW of "+c.state+" context="+c.context);
if ( !c.context.isEmpty() ) {
NFAContext newContext = c.context.parent; // "pop" invoking state
closure(new NFAConfig(c, c.context.returnState, newContext),
collectPredicates, configs);
}
else {
commonClosure(s, altNum, context, semanticContext, collectPredicates, configs); // do global FOLLOW
commonClosure(c, collectPredicates, configs); // do global FOLLOW
}
return;
}
Rule invokingRule = null;
if ( context != NFAContext.EMPTY) {
if ( !c.context.isEmpty() ) {
// if stack not empty, get invoking rule from top of stack
invokingRule = context.returnState.rule;
invokingRule = c.context.returnState.rule;
}
System.out.println("FOLLOW of "+s+" context="+context);
System.out.println("FOLLOW of "+c.state+" context="+c.context);
// follow all static FOLLOW links
int n = s.getNumberOfTransitions();
int n = c.state.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
Transition t = c.state.transition(i);
if ( !(t instanceof EpsilonTransition) ) continue; // ignore EOF transitions
// Chase global FOLLOW links if they don't point at invoking rule
// else follow link to context state only
if ( t.target.rule != invokingRule ) {
//System.out.println("OFF TO "+t.target);
closure(t.target, altNum, context, semanticContext, collectPredicates, configs);
closure(new NFAConfig(c, t.target), collectPredicates, configs);
}
else { // t.target is in invoking rule; only follow context's link
if ( t.target == context.returnState ) {
if ( t.target == c.context.returnState ) {
//System.out.println("OFF TO CALL SITE "+t.target);
// go only to specific call site; pop context
NFAContext newContext = context.parent; // "pop" invoking state
closure(t.target, altNum, newContext, semanticContext, collectPredicates, configs);
NFAContext newContext = c.context.parent; // "pop" invoking state
closure(new NFAConfig(c, t.target, newContext),
collectPredicates, configs);
}
}
}
@ -380,46 +375,44 @@ public class PredictionDFAFactory {
}
void commonClosure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext, boolean collectPredicates,
List<NFAConfig> configs)
{
int n = s.getNumberOfTransitions();
void commonClosure(NFAConfig c, boolean collectPredicates, List<NFAConfig> configs) {
int n = c.state.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
Transition t = c.state.transition(i);
if ( t instanceof RuleTransition) {
NFAState retState = ((RuleTransition)t).followState;
NFAContext newContext = context;
NFAContext newContext = c.context;
//if ( !visited.member(t.target.rule.index) ) { // !recursive?
if ( s.rule != t.target.rule &&
!context.contains(((RuleTransition)t).followState) ) { // !recursive?
if ( c.state.rule != t.target.rule &&
!c.context.contains(((RuleTransition)t).followState) ) { // !recursive?
// first create a new context and push onto call tree,
// recording the fact that we are invoking a rule and
// from which state.
System.out.println("nonrecursive invoke of "+t.target+" ret to "+retState+" ctx="+context);
newContext = new NFAContext(context, retState);
System.out.println("nonrecursive invoke of "+t.target+" ret to "+retState+" ctx="+c.context);
newContext = new NFAContext(c.context, retState);
}
else {
System.out.println("# recursive invoke of "+t.target+" ret to "+retState+" ctx="+context);
System.out.println("# recursive invoke of "+t.target+" ret to "+retState+" ctx="+c.context);
// don't record recursion, but record we did so we know
// what to do at end of rule.
context.recursed = true;
c.context.recursed = true;
}
// traverse epsilon edge to new rule
closure(t.target, altNum, newContext, semanticContext, collectPredicates, configs);
closure(new NFAConfig(c, t.target, newContext),
collectPredicates, configs);
}
else if ( t instanceof ActionTransition ) {
collectPredicates = false; // can't see past actions
closure(t.target, altNum, context, semanticContext, collectPredicates, configs);
closure(new NFAConfig(c, t.target), collectPredicates, configs);
}
else if ( t instanceof PredicateTransition ) {
SemanticContext labelContext = ((PredicateTransition)t).semanticContext;
SemanticContext newSemanticContext = semanticContext;
SemanticContext newSemanticContext = c.semanticContext;
if ( collectPredicates ) {
// AND the previous semantic context with new pred
// int walkAlt =
// dfa.decisionNFAStartState.translateDisplayAltToWalkAlt(alt);
NFAState altLeftEdge = dfa.decisionNFAStartState.transition(altNum-1).target;
NFAState altLeftEdge = dfa.decisionNFAStartState.transition(c.alt-1).target;
/*
System.out.println("state "+p.stateNumber+" alt "+alt+" walkAlt "+walkAlt+" trans to "+transition0.target);
System.out.println("DFA start state "+dfa.decisionNFAStartState.stateNumber);
@ -429,21 +422,22 @@ public class PredictionDFAFactory {
*/
// do not hoist syn preds from other rules; only get if in
// starting state's rule (i.e., context is empty)
if ( !labelContext.isSyntacticPredicate() || s==altLeftEdge ) {
System.out.println("&"+labelContext+" enclosingRule="+s.rule);
if ( !labelContext.isSyntacticPredicate() || c.state==altLeftEdge ) {
System.out.println("&"+labelContext+" enclosingRule="+c.state.rule);
newSemanticContext =
SemanticContext.and(semanticContext, labelContext);
SemanticContext.and(c.semanticContext, labelContext);
}
}
else {
// if we're not collecting, means we saw an action previously. that blocks this pred
hasPredicateBlockedByAction = true;
}
closure(t.target, altNum, context, newSemanticContext, collectPredicates, configs);
closure(new NFAConfig(c, t.target, newSemanticContext),
collectPredicates, configs);
}
else if ( t.isEpsilon() ) {
closure(t.target, altNum, context, semanticContext, collectPredicates, configs);
closure(new NFAConfig(c, t.target), collectPredicates, configs);
}
}
}
@ -479,10 +473,7 @@ public class PredictionDFAFactory {
for (NFAConfig c : predConfigsSortedByAlt) {
DFAState predDFATarget = dfa.newState();
// new DFA state is a target of the predicate from d
predDFATarget.addNFAConfig(c.state,
c.alt,
c.context,
c.semanticContext);
predDFATarget.addNFAConfig(c);
dfa.addAcceptState(c.alt, predDFATarget);
// add a transition to pred target from d
d.addEdge(new PredicateEdge(c.semanticContext, predDFATarget));

6
tool/src/org/antlr/v4/analysis/Resolver.java
View File

@ -116,11 +116,11 @@ public class Resolver {
}
public void resolveAmbiguities(DFAState d) {
if ( unused_StackLimitedNFAToDFAConverter.debug ) {
if ( PredictionDFAFactory.debug ) {
System.out.println("resolveNonDeterminisms "+d.toString());
}
Set<Integer> ambiguousAlts = getAmbiguousAlts(d);
if ( unused_StackLimitedNFAToDFAConverter.debug && ambiguousAlts!=null ) {
if ( PredictionDFAFactory.debug && ambiguousAlts!=null ) {
System.out.println("ambig alts="+ambiguousAlts);
}
@ -133,7 +133,7 @@ public class Resolver {
boolean resolved =
semResolver.tryToResolveWithSemanticPredicates(d, ambiguousAlts);
if ( resolved ) {
if ( unused_StackLimitedNFAToDFAConverter.debug ) {
if ( PredictionDFAFactory.debug ) {
System.out.println("resolved DFA state "+d.stateNumber+" with pred");
}
d.resolvedWithPredicates = true;

239
tool/src/org/antlr/v4/analysis/unused_RecursionLimitedNFAToDFAConverter.java
View File

@ -1,239 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.DecisionState;
import org.antlr.v4.automata.NFAState;
import org.antlr.v4.automata.RuleTransition;
import org.antlr.v4.automata.Transition;
import org.antlr.v4.misc.BitSet;
import org.antlr.v4.misc.IntSet;
import org.antlr.v4.tool.Grammar;
import java.util.List;
/**
* closure: Add new NFA states + context to DFA state d. Also add semantic
* predicates to semantic context if collectPredicates is set. We only
* collect predicates at hoisting depth 0, meaning before any token/char
* have been recognized. This corresponds, during analysis, to the
* initial DFA start state construction closure() invocation.
*
* When is a closure operation in a cycle condition? While it is
* very possible to have the same NFA state mentioned twice
* within the same DFA state, there are two situations that
* would lead to nontermination of closure operation:
*
* o Whenever closure reaches a configuration where the same state
* with same or a suffix context already exists. This catches
* the IF-THEN-ELSE tail recursion cycle and things like
*
* a : A a | B ;
*
* the context will be $ (empty stack).
*
* We have to check
* larger context stacks because of (...)+ loops. For
* example, the context of a (...)+ can be nonempty if the
* surrounding rule is invoked by another rule:
*
* a : b A | X ;
* b : (B|)+ ; // nondeterministic by the way
*
* The context of the (B|)+ loop is "invoked from item
* a : . b A ;" and then the empty alt of the loop can reach back
* to itself. The context stack will have one "return
* address" element and so we must check for same state, same
* context for arbitrary context stacks.
*
* Idea: If we've seen this configuration before during closure, stop.
* We also need to avoid reaching same state with conflicting context.
* Ultimately analysis would stop and we'd find the conflict, but we
* should stop the computation. Previously I only checked for
* exact config. Need to check for same state, suffix context
* not just exact context.
*
* o Whenever closure reaches a configuration where state p
* is present in its own context stack. This means that
* p is a rule invocation state and the target rule has
* been called before. NFAContext.MAX_RECURSIVE_INVOCATIONS
* (See the comment there also) determines how many times
* it's possible to recurse; clearly we cannot recurse forever.
* Some grammars such as the following actually require at
* least one recursive call to correctly compute the lookahead:
*
* a : L ID R
* | b
* ;
* b : ID
* | L a R
* ;
*
* Input L ID R is ambiguous but to figure this out, ANTLR
* needs to go a->b->a->b to find the L ID sequence.
*
* Do not allow closure to add a configuration that would
* allow too much recursion.
*
* This case also catches infinite left recursion.
*/
public class unused_RecursionLimitedNFAToDFAConverter extends unused_StackLimitedNFAToDFAConverter {
/** This is similar to Bermudez's m constant in his LAR(m) where
* you bound the stack so your states don't explode. The main difference
* is that I bound only recursion on the stack, not the simple stack size.
* This looser constraint will let the conversion roam further to find
* lookahead to resolve a decision.
*
* Bermudez's m operates differently as it is his LR stack depth
* I'm pretty sure it therefore includes all stack symbols. Here I
* restrict the size of an NFA configuration to be finite because a
* stack component may mention the same NFA invocation state at
* most m times. Hence, the number of DFA states will not grow forever.
* With recursive rules like
*
* e : '(' e ')' | INT ;
*
* you could chase your tail forever if somebody said "s : e '.' | e ';' ;"
* This constant prevents new states from being created after a stack gets
* "too big". Actually (12/14/2007) I realize that this example is
* trapped by the non-LL(*) detector for recursion in > 1 alt. Here is
* an example that trips stack overflow:
*
* s : a Y | A A A A A X ; // force recursion past m=4
* a : A a | Q;
*
* If that were:
*
* s : a Y | A+ X ;
*
* it could loop forever.
*
* Imagine doing a depth-first search on the e DFA...as you chase an input
* sequence you can recurse to same rule such as e above. You'd have a
* chain of ((((. When you get do some point, you have to give up. The
* states in the chain will have longer and longer NFA config stacks.
* Must limit size.
*
* max=0 implies you cannot ever jump to another rule during closure.
* max=1 implies you can make as many calls as you want--you just
* can't ever visit a state that is on your rule invocation stack.
* I.e., you cannot ever recurse.
* max=2 implies you are able to recurse once (i.e., call a rule twice
* from the same place).
*
* This tracks recursion to a rule specific to an invocation site!
* It does not detect multiple calls to a rule from different rule
* invocation states. We are guaranteed to terminate because the
* stack can only grow as big as the number of NFA states * max.
*
* I noticed that the Java grammar didn't work with max=1, but did with
* max=4. Let's set to 4. Recursion is sometimes needed to resolve some
* fixed lookahead decisions.
*/
public static int DEFAULT_MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK = 4;
/** Max recursion depth.
* approx is setting stack size like bermudez: m=1 in this case.
* full alg limits recursion not overall stack size. that's more
* like LL(k) analysis which can have any stack size, but will recurse
* a max of k times since it can only see k tokens. each recurse pumps
* another token. limiting stack size to m={0,1} lets us convert
* recursion to loops. use r constant not m for recursion depth?
*/
public int r = DEFAULT_MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK;
/** Track whether an alt discovers recursion for each alt during
* NFA to DFA conversion; >1 alt with recursion implies nonregular.
*/
public IntSet recursiveAltSet = new BitSet();
public unused_RecursionLimitedNFAToDFAConverter(Grammar g, DecisionState nfaStartState) {
super(g, nfaStartState);
}
/**
* 1. Reach an NFA state associated with the end of a rule, r, in the
* grammar from which it was built. We must add an implicit (i.e.,
* don't actually add an epsilon transition) epsilon transition
* from r's end state to the NFA state following the NFA state
* that transitioned to rule r's start state. Because there are
* many states that could reach r, the context for a rule invocation
* is part of a call tree not a simple stack. When we fall off end
* of rule, "pop" a state off the call tree and add that state's
* "following" node to d's NFA configuration list. The context
* for this new addition will be the new "stack top" in the call tree.
*
* 2. Like case 1, we reach an NFA state associated with the end of a
* rule, r, in the grammar from which NFA was built. In this case,
* however, we realize that during this NFA->DFA conversion, no state
* invoked the current rule's NFA. There is no choice but to add
* all NFA states that follow references to r's start state. This is
* analogous to computing the FOLLOW(r) in the LL(k) world. By
* construction, even rule stop state has a chain of nodes emanating
* from it that points to every possible following node. This case
* is conveniently handled then by the common closure case.
*/
void ruleStopStateClosure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext,
boolean collectPredicates,
List<NFAConfig> configs)
{
if ( context != NFAContext.EMPTY) {
NFAContext newContext = context.parent; // "pop" invoking state
closure(context.returnState, altNum, newContext, semanticContext, collectPredicates, configs);
}
else {
commonClosure(s, altNum, context, semanticContext, collectPredicates, configs); // do global FOLLOW
}
}
/** 1. Traverse an edge that takes us to the start state of another
* rule, r. We must push this state so that if the DFA
* conversion hits the end of rule r, then it knows to continue
* the conversion at state following state that "invoked" r. By
* construction, there is a single transition emanating from a rule
* ref node.
*
* 2. Normal case. If s can see another NFA state q via epsilon, then add
* q to d's configuration list, copying p's context for q's context.
* If there is a semantic predicate on the transition, then AND it
* with any existing semantic context.
*
* 3. Preds?
*/
void commonClosure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext,
boolean collectPredicates,
List<NFAConfig> configs)
{
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( t instanceof RuleTransition) {
NFAState retState = ((RuleTransition)t).followState;
int depth = context.recursionDepthEmanatingFromState(retState.stateNumber);
if ( depth==1 ) { // recursion
recursiveAltSet.add(altNum); // indicate that this alt is recursive
if ( recursiveAltSet.size()>1 ) {
throw new MultipleRecursiveAltsSignal(recursiveAltSet);
}
}
// Detect an attempt to recurse too high
// if this context has hit the max recursions for p.stateNumber,
// don't allow it to enter p.stateNumber again
if ( depth >= r ) {
throw new RecursionOverflowSignal(altNum, depth, s);
}
// first create a new context and push onto call tree,
// recording the fact that we are invoking a rule and
// from which state (case 2 below will get the following state
// via the RuleTransition emanating from the invoking state
// pushed on the stack).
NFAContext newContext = new NFAContext(context, retState);
// traverse epsilon edge to new rule
closure(t.target, altNum, newContext, semanticContext, collectPredicates, configs);
}
else if ( t.isEpsilon() ) {
closure(t.target, altNum, context, semanticContext, collectPredicates, configs);
}
}
}
}

470
tool/src/org/antlr/v4/analysis/unused_StackLimitedNFAToDFAConverter.java
View File

@ -1,470 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.*;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.util.*;
/** Code that embodies the NFA conversion to DFA. A new object is needed
* per DFA (also required for thread safety if multiple conversions
* launched).
*/
public class unused_StackLimitedNFAToDFAConverter {
Grammar g;
DecisionState nfaStartState;
/** DFA we are creating */
DFA dfa;
/** Stack depth max; same as Bermudez's m */
int m = 1;
/** A list of DFA states we still need to process during NFA conversion */
List<DFAState> work = new LinkedList<DFAState>();
/** Each alt in an NFA derived from a grammar must have a DFA state that
* predicts it lest the parser not know what to do. Nondeterminisms can
* lead to this situation (assuming no semantic predicates can resolve
* the problem) and when for some reason, I cannot compute the lookahead
* (which might arise from an error in the algorithm or from
* left-recursion etc...).
*/
public Set<Integer> unreachableAlts;
/** Track all DFA states with ambiguous configurations.
* By reaching the same DFA state, a path through the NFA for some input
* is able to reach the same NFA state by starting at more than one
* alternative's left edge. If the context is the same or conflicts,
* then we have ambiguity. If the context is different, it's simply
* nondeterministic and we should keep looking for edges that will
* render it deterministic. If we run out of things to add to the DFA,
* we'll get a dangling state; it's non-LL(*). Later we may find that predicates
* resolve the issue, but track ambiguous states anyway.
*/
public Set<DFAState> ambiguousStates = new HashSet<DFAState>();
/** The set of states w/o emanating edges (and w/o resolving sem preds). */
public Set<DFAState> danglingStates = new HashSet<DFAState>();
/** Was a syntactic ambiguity resolved with predicates? Any DFA
* state that predicts more than one alternative, must be resolved
* with predicates or it should be reported to the user.
*/
public Set<DFAState> resolvedWithSemanticPredicates = new HashSet<DFAState>();
/** Tracks alts insufficiently covered.
* For example, p1||true gets reduced to true and so leaves
* whole alt uncovered. This maps alt num to the set of (Token)
* locations in grammar of uncovered elements.
*/
public Map<DFAState, List<Integer>> statesWithIncompletelyCoveredAlts = new HashMap<DFAState, List<Integer>>();
public boolean hasPredicateBlockedByAction = false;
/** Recursion is limited to a particular depth. Which state tripped it? */
public DFAState recursionOverflowState;
/** Which state found multiple recursive alts? */
public DFAState abortedDueToMultipleRecursiveAltsAt;
/** Are there any loops in this DFA? */
// public boolean cyclic = false;
/** Used to prevent the closure operation from looping to itself and
* hence looping forever. Sensitive to the NFA state, the alt, and
* the stack context.
*/
Set<NFAConfig> closureBusy;
Resolver resolver;
public static boolean debug = false;
public unused_StackLimitedNFAToDFAConverter(Grammar g, DecisionState nfaStartState) {
this.g = g;
this.nfaStartState = nfaStartState;
dfa = new DFA(g, nfaStartState);
// dfa.converter = this;
//resolver = new Resolver(this);
}
public DFA createDFA() {
closureBusy = new HashSet<NFAConfig>();
computeStartState();
dfa.addState(dfa.startState); // make sure dfa knows about this state
work.add(dfa.startState);
// while more DFA states to check, process them
while ( work.size()>0 ) {
DFAState d = work.get(0);
reach(d);
resolver.resolveDeadState(d);
work.remove(0); // we're done with this DFA state
}
unreachableAlts = getUnreachableAlts();
closureBusy = null; // wack all that memory used during closure
return dfa;
}
/** From this node, add a d--a-->t transition for all
* labels 'a' where t is a DFA node created
* from the set of NFA states reachable from any NFA
* configuration in DFA state d.
*/
void reach(DFAState d) {
OrderedHashSet<IntervalSet> labels = DFA.getReachableLabels(d);
for (IntervalSet label : labels) {
DFAState t = reach(d, label);
if ( debug ) {
System.out.println("DFA state after reach -" +
label.toString(g)+"->"+t);
}
// nothing was reached by label; we must have resolved
// all NFA configs in d, when added to work, that point at label
if ( t==null ) continue;
// if ( t.getUniqueAlt()==NFA.INVALID_ALT_NUMBER ) {
// // Only compute closure if a unique alt number is not known.
// // If a unique alternative is mentioned among all NFA
// // configurations then there is no possibility of needing to look
// // beyond this state; also no possibility of a nondeterminism.
// // This optimization May 22, 2006 just dropped -Xint time
// // for analysis of Java grammar from 11.5s to 2s! Wow.
// closure(t); // add any NFA states reachable via epsilon
// }
try {
closure(t); // add any NFA states reachable via epsilon
}
catch (RecursionOverflowSignal ros) {
recursionOverflowState = d;
ErrorManager.recursionOverflow(g.fileName, d, ros.state, ros.altNum, ros.depth);
}
catch (MultipleRecursiveAltsSignal mras) {
abortedDueToMultipleRecursiveAltsAt = d;
ErrorManager.multipleRecursiveAlts(g.fileName, d, mras.recursiveAltSet);
}
catch (AnalysisTimeoutSignal at) {// TODO: nobody throws yet
ErrorManager.analysisTimeout();
}
addTransition(d, label, t); // make d-label->t transition
}
// Add semantic predicate transitions if we resolved when added to work list
if ( d.resolvedWithPredicates ) addPredicateTransitions(d);
}
/** Add t if not in DFA yet, resolving nondet's and then make d-label->t */
void addTransition(DFAState d, IntervalSet label, DFAState t) {
DFAState existing = dfa.stateSet.get(t);
if ( existing != null ) { // seen before; point at old one
d.addEdge(new Edge(existing, label));
return;
}
// resolve any syntactic conflicts by choosing a single alt or
// by using semantic predicates if present.
resolver.resolveAmbiguities(t);
// If deterministic, don't add this state; it's an accept state
// Just return as a valid DFA state
int alt = t.getUniquelyPredictedAlt();
if ( alt > 0 ) { // uniquely predicts an alt?
System.out.println(t+" predicts "+alt);
// Define new stop state
dfa.addAcceptState(alt, t);
}
else {
System.out.println("ADD "+t);
work.add(t); // unresolved, add to work list to continue NFA conversion
dfa.addState(t); // add state we've never seen before
}
d.addEdge(new Edge(t, label));
}
/** Given the set of NFA states in DFA state d, find all NFA states
* reachable traversing label arcs. By definition, there can be
* only one DFA state reachable by a single label from DFA state d so we must
* find and merge all NFA states reachable via label. Return a new
* DFAState that has all of those NFA states with their context.
*
* Because we cannot jump to another rule nor fall off the end of a rule
* via a non-epsilon transition, NFA states reachable from d have the
* same configuration as the NFA state in d. So if NFA state 7 in d's
* configurations can reach NFA state 13 then 13 will be added to the
* new DFAState (labelDFATarget) with the same configuration as state
* 7 had.
*/
public DFAState reach(DFAState d, IntervalSet label) {
//System.out.println("reach "+label.toString(g)+" from "+d.stateNumber);
DFAState labelTarget = dfa.newState();
for (NFAConfig c : d.nfaConfigs) {
int n = c.state.getNumberOfTransitions();
for (int i=0; i<n; i++) { // for each transition
Transition t = c.state.transition(i);
// when we added this state as target of some other state,
// we tried to resolve any conflicts. Ignore anything we
// were able to fix previously
if ( c.resolved || c.resolvedWithPredicate) continue;
// found a transition with label; does it collide with label?
if ( !t.isEpsilon() && !t.label().and(label).isNil() ) {
// add NFA target to (potentially) new DFA state
labelTarget.addNFAConfig(t.target, c.alt, c.context, c.semanticContext);
}
}
}
// if we couldn't find any non-resolved edges to add, return nothing
if ( labelTarget.nfaConfigs.size()==0 ) return null;
return labelTarget;
}
/** From this first NFA state of a decision, create a DFA.
* Walk each alt in decision and compute closure from the start of that
* rule, making sure that the closure does not include other alts within
* that same decision. The idea is to associate a specific alt number
* with the starting closure so we can trace the alt number for all states
* derived from this. At a stop state in the DFA, we can return this alt
* number, indicating which alt is predicted.
*/
public void computeStartState() {
DFAState d = dfa.newState();
dfa.startState = d;
// add config for each alt start, then add closure for those states
for (int altNum=1; altNum<=dfa.nAlts; altNum++) {
Transition t = nfaStartState.transition(altNum-1);
NFAState altStart = t.target;
d.addNFAConfig(altStart, altNum,
NFAContext.EMPTY,
SemanticContext.EMPTY_SEMANTIC_CONTEXT);
}
closure(d);
}
/** For all NFA states (configurations) merged in d,
* compute the epsilon closure; that is, find all NFA states reachable
* from the NFA states in d via purely epsilon transitions.
*/
public void closure(DFAState d) {
if ( debug ) {
System.out.println("closure("+d+")");
}
// Only the start state initiates pred collection; gets turned
// off maybe by actions later hence we need a parameter to carry
// it forward
boolean collectPredicates = (d == dfa.startState);
// TODO: can we avoid this separate list by directly filling d.nfaConfigs?
// OH: concurrent modification. dup initialconfigs? works for lexers, try here to save configs param
List<NFAConfig> configs = new ArrayList<NFAConfig>();
for (NFAConfig c : d.nfaConfigs) {
closure(c.state, c.alt, c.context, c.semanticContext, collectPredicates, configs);
}
d.nfaConfigs.addAll(configs); // Add new NFA configs to DFA state d
closureBusy.clear();
if ( debug ) {
System.out.println("after closure("+d+")");
}
//System.out.println("after closure d="+d);
}
/** Where can we get from NFA state s traversing only epsilon transitions?
*
* A closure operation should abort if that computation has already
* been done or a computation with a conflicting context has already
* been done. If proposed NFA config's state and alt are the same
* there is potentially a problem. If the stack context is identical
* then clearly the exact same computation is proposed. If a context
* is a suffix of the other, then again the computation is in an
* identical context. beta $ and beta alpha $ are considered the same stack.
* We could walk configurations linearly doing the comparison instead
* of a set for exact matches but it's much slower because you can't
* do a Set lookup. I use exact match as ANTLR
* always detect the conflict later when checking for context suffixes...
* I check for left-recursive stuff and terminate before analysis to
* avoid need to do this more expensive computation.
*
* TODO: remove altNum if we don't reorder for loopback nodes
*/
public void closure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext,
boolean collectPredicates,
List<NFAConfig> configs)
{
NFAConfig proposedNFAConfig = new NFAConfig(s, altNum, context, semanticContext);
if ( closureBusy.contains(proposedNFAConfig) ) return;
closureBusy.add(proposedNFAConfig);
// p itself is always in closure
configs.add(proposedNFAConfig);
if ( s instanceof RuleStopState ) {
ruleStopStateClosure(s, altNum, context, semanticContext, collectPredicates, configs);
}
else {
commonClosure(s, altNum, context, semanticContext, collectPredicates, configs);
}
}
// if we have context info and we're at rule stop state, do
// local follow for invokingRule and global follow for other links
void ruleStopStateClosure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext,
boolean collectPredicates,
List<NFAConfig> configs)
{
Rule invokingRule = null;
if ( context!=NFAContext.EMPTY) {
// if stack not empty, get invoking rule from top of stack
invokingRule = context.returnState.rule;
}
//System.out.println("FOLLOW of "+s+" context="+context);
// follow all static FOLLOW links
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( !(t instanceof EpsilonTransition) ) continue; // ignore EOF transitions
// Chase global FOLLOW links if they don't point at invoking rule
// else follow link to context state only
if ( t.target.rule != invokingRule ) {
//System.out.println("OFF TO "+t.target);
closure(t.target, altNum, context, semanticContext, collectPredicates, configs);
}
else { // t.target is in invoking rule; only follow context's link
if ( t.target == context.returnState ) {
//System.out.println("OFF TO CALL SITE "+t.target);
// go only to specific call site; pop context
NFAContext newContext = context.parent; // "pop" invoking state
closure(t.target, altNum, newContext, semanticContext, collectPredicates, configs);
}
}
}
return;
}
void commonClosure(NFAState s, int altNum, NFAContext context,
SemanticContext semanticContext, boolean collectPredicates,
List<NFAConfig> configs)
{
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( t instanceof RuleTransition) {
NFAContext newContext = context; // assume old context
NFAState retState = ((RuleTransition)t).followState;
if ( context.depth() < m ) { // track first call return state only
newContext = new NFAContext(context, retState);
}
closure(t.target, altNum, newContext, semanticContext, collectPredicates, configs);
}
else if ( t instanceof ActionTransition ) {
collectPredicates = false; // can't see past actions
closure(t.target, altNum, context, semanticContext, collectPredicates, configs);
}
else if ( t instanceof PredicateTransition ) {
SemanticContext labelContext = ((PredicateTransition)t).semanticContext;
SemanticContext newSemanticContext = semanticContext;
if ( collectPredicates ) {
// AND the previous semantic context with new pred
// int walkAlt =
// dfa.decisionNFAStartState.translateDisplayAltToWalkAlt(alt);
NFAState altLeftEdge = dfa.decisionNFAStartState.transition(altNum-1).target;
/*
System.out.println("state "+p.stateNumber+" alt "+alt+" walkAlt "+walkAlt+" trans to "+transition0.target);
System.out.println("DFA start state "+dfa.decisionNFAStartState.stateNumber);
System.out.println("alt left edge "+altLeftEdge.stateNumber+
", epsilon target "+
altLeftEdge.transition(0).target.stateNumber);
*/
// do not hoist syn preds from other rules; only get if in
// starting state's rule (i.e., context is empty)
if ( !labelContext.isSyntacticPredicate() || s==altLeftEdge ) {
System.out.println("&"+labelContext+" enclosingRule="+s.rule);
newSemanticContext =
SemanticContext.and(semanticContext, labelContext);
}
}
else {
// if we're not collecting, means we saw an action previously. that blocks this pred
hasPredicateBlockedByAction = true;
}
closure(t.target, altNum, context, newSemanticContext, collectPredicates, configs);
}
else if ( t.isEpsilon() ) {
closure(t.target, altNum, context, semanticContext, collectPredicates, configs);
}
}
}
/** for each NFA config in d, look for "predicate required" sign we set
* during nondeterminism resolution.
*
* Add the predicate edges sorted by the alternative number; I'm fairly
* sure that I could walk the configs backwards so they are added to
* the predDFATarget in the right order, but it's best to make sure.
* Predicates succeed in the order they are specifed. Alt i wins
* over alt i+1 if both predicates are true.
*/
protected void addPredicateTransitions(DFAState d) {
List<NFAConfig> configsWithPreds = new ArrayList<NFAConfig>();
// get a list of all configs with predicates
for (NFAConfig c : d.nfaConfigs) {
if ( c.resolvedWithPredicate) {
configsWithPreds.add(c);
}
}
// Sort ascending according to alt; alt i has higher precedence than i+1
Collections.sort(configsWithPreds,
new Comparator<NFAConfig>() {
public int compare(NFAConfig a, NFAConfig b) {
if ( a.alt < b.alt ) return -1;
else if ( a.alt > b.alt ) return 1;
return 0;
}
});
List<NFAConfig> predConfigsSortedByAlt = configsWithPreds;
// Now, we can add edges emanating from d for these preds in right order
for (NFAConfig c : predConfigsSortedByAlt) {
DFAState predDFATarget = dfa.newState();
// new DFA state is a target of the predicate from d
predDFATarget.addNFAConfig(c.state,
c.alt,
c.context,
c.semanticContext);
dfa.addAcceptState(c.alt, predDFATarget);
// add a transition to pred target from d
d.addEdge(new PredicateEdge(c.semanticContext, predDFATarget));
}
}
public Set<Integer> getUnreachableAlts() {
Set<Integer> unreachable = new HashSet<Integer>();
for (int alt=1; alt<=dfa.nAlts; alt++) {
if ( dfa.altToAcceptStates[alt]==null ) unreachable.add(alt);
}
return unreachable;
}
void issueAmbiguityWarnings() { resolver.issueAmbiguityWarnings(); }
}

12
tool/src/org/antlr/v4/automata/DFAState.java
View File

@ -1,9 +1,7 @@
package org.antlr.v4.automata;
import org.antlr.v4.analysis.NFAConfig;
import org.antlr.v4.analysis.NFAContext;
import org.antlr.v4.analysis.Resolver;
import org.antlr.v4.analysis.SemanticContext;
import org.antlr.v4.misc.IntSet;
import org.antlr.v4.misc.OrderedHashSet;
@ -82,16 +80,6 @@ public class DFAState {
nfaConfigs.add(c);
}
public NFAConfig addNFAConfig(NFAState state,
int alt,
NFAContext context,
SemanticContext semanticContext)
{
NFAConfig c = new NFAConfig(state, alt, context, semanticContext);
addNFAConfig(c);
return c;
}
/** Walk each configuration and if they are all the same alt,
* even the resolved configs.
*/