jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

got predicate issue resolved. executes preds if not dependent on context and no dipping into outer context 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8919]
This commit is contained in:
parrt 2011-07-28 12:43:28 -08:00
parent 0cb91817d9
commit 3d15daccad
10 changed files with 107 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java
View File

@ -50,42 +50,35 @@ public class ATNConfig {
*/
public RuleContext context;
/** The stack acquired from parser invoking the ATN interpreter.
* Initially it's the rule stack upon entry into interp.adaptivePredict().
*
* We only pop from outerContext if we hit rule stop state
* of rule that initiates the adaptivePredict(). outerContext
* tracks as we go back up the entry context. If we do a call,
* we again push onto the regular context, rather than pushing onto
* outerContext.
*
* At an accept state, we ignore outerContext unless we had to use it
* during prediction. If outerContext != originalContext
* (stored in the ATN interpreter), then this config needed context
* to find another symbol.
*
* Lexer matches with ATN so there is no external context; this is null.
*/
//public RuleContext outerContext;
/**
Indicates that we have reached this ATN configuration after
traversing a predicate transition. This is important because we
cannot cache DFA states derived from such configurations
otherwise predicates would not get executed again (DFAs don't
have predicated edges in v4).
* Indicates that we have reached this ATN configuration after
* traversing a predicate transition. This is important because we
* cannot cache DFA states derived from such configurations
* otherwise predicates would not get executed again (DFAs don't
* have predicated edges in v4).
*/
public boolean traversedPredicate;
/**
Indicates that we have reached this ATN configuration after
traversing a non-forced action transition. We do not execute
predicates after such actions because the predicates could be
functions of the side effects. Force actions must be either side
effect free or automatically undone as the parse continues.
* Indicates that we have reached this ATN configuration after
* traversing a non-forced action transition. We do not execute
* predicates after such actions because the predicates could be
* functions of the side effects. Force actions must be either side
* effect free or automatically undone as the parse continues.
*/
public boolean traversedAction;
/**
* We cannot execute predicates dependent upon local context unless
* we know for sure we are in the correct context. Because there is
* no way to do this efficiently, we simply cannot evaluate
* dependent predicates if we are pursuing a global FOLLOW
* operation. closure() tracks the depth of how far we dip into the
* outer context. We then avoid executing predicates that are
* dependent on local context if this depth is > 0.
*/
public int reachesIntoOuterContext;
public ATNConfig(ATNState state,
int alt,
RuleContext context)
@ -99,9 +92,9 @@ public class ATNConfig {
this.state = c.state;
this.alt = c.alt;
this.context = c.context;
// this.outerContext = c.outerContext;
this.traversedPredicate = c.traversedPredicate;
this.traversedAction = c.traversedAction;
this.reachesIntoOuterContext = c.reachesIntoOuterContext;
}
public ATNConfig(ATNConfig c, ATNState state) {

5
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -139,11 +139,14 @@ public abstract class ATNSimulator {
case Transition.RANGE : return new RangeTransition(arg1, arg2, target);
case Transition.RULE : return new RuleTransition(arg2, atn.states.get(arg1), target);
case Transition.PREDICATE : return new PredicateTransition(target, arg1, arg2);
case Transition.DEPENDENT_PREDICATE :
PredicateTransition p = new PredicateTransition(target, arg1, arg2);
p.isCtxDependent = true;
return p;
case Transition.ATOM : return new AtomTransition(arg1, target);
case Transition.ACTION : return new ActionTransition(target, arg1, arg2);
case Transition.FORCED_ACTION : return new ActionTransition(target, arg1, arg2);
case Transition.SET : return new SetTransition(sets.get(arg1), target);
// case Transition.NOT_ATOM : return new NotAtomTransition(arg1, target);
case Transition.NOT_SET : return new NotSetTransition(sets.get(arg1), null, target);
case Transition.WILDCARD : return new WildcardTransition(target);
}

46
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -37,7 +37,7 @@ import org.stringtemplate.v4.misc.MultiMap;
import java.util.*;
public class ParserATNSimulator extends ATNSimulator {
public static boolean debug = false;
public static boolean debug = true;
public static boolean dfa_debug = false;
public static int ATN_failover = 0;
@ -107,6 +107,7 @@ public class ParserATNSimulator extends ATNSimulator {
boolean useContext)
{
if ( originalContext==null ) originalContext = RuleContext.EMPTY;
this.originalContext = originalContext;
RuleContext ctx = RuleContext.EMPTY;
if ( useContext ) ctx = originalContext;
OrderedHashSet<ATNConfig> s0_closure =
@ -141,6 +142,8 @@ public class ParserATNSimulator extends ATNSimulator {
public int execDFA(TokenStream input, DFA dfa, DFAState s0, RuleContext originalContext) {
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+" exec LA(1)=="+input.LT(1));
// dump(dfa);
if ( originalContext==null ) originalContext = RuleContext.EMPTY;
this.originalContext = originalContext;
DFAState prevAcceptState = null;
DFAState s = s0;
int t = input.LA(1);
@ -462,7 +465,13 @@ public class ParserATNSimulator extends ATNSimulator {
closure(c, configs, closureBusy);
return;
}
// else if we have no context info, just chase follow links
else {
// else if we have no context info, just chase follow links
// but track how far we dip into outer context. Might
// come in handy and we avoid evaluating context dependent
// preds if this is > 0.
config.reachesIntoOuterContext++;
}
}
ATNState p = config.state;
@ -471,13 +480,13 @@ public class ParserATNSimulator extends ATNSimulator {
for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i);
boolean evalPreds = !config.traversedAction;
ATNConfig c = getEpsilonTarget(config, t, evalPreds);
boolean ignorePreds = config.traversedAction;
ATNConfig c = getEpsilonTarget(config, t, ignorePreds);
if ( c!=null ) closure(c, configs, closureBusy);
}
}
public ATNConfig getEpsilonTarget(ATNConfig config, Transition t, boolean evalPreds) {
public ATNConfig getEpsilonTarget(ATNConfig config, Transition t, boolean ignorePreds) {
ATNConfig c = null;
if ( t instanceof RuleTransition ) {
ATNState p = config.state;
@ -495,13 +504,24 @@ public class ParserATNSimulator extends ATNSimulator {
}
else if ( t instanceof PredicateTransition ) {
PredicateTransition pt = (PredicateTransition)t;
if ( debug ) System.out.println("PRED (eval="+evalPreds+") "+pt.ruleIndex+":"+pt.predIndex);
if ( parser != null ) System.out.println("rule surrounding pred is "+
parser.getRuleNames()[pt.ruleIndex]);
// preds are epsilon if we're not doing preds.
if ( debug ) {
System.out.println("PRED (ignore="+ignorePreds+") "+pt.ruleIndex+":"+pt.predIndex+
", ctx dependent="+pt.isCtxDependent+
", reachesIntoOuterContext="+config.reachesIntoOuterContext);
if ( parser != null ) System.out.println("rule surrounding pred is "+
parser.getRuleNames()[pt.ruleIndex]);
System.out.println();
}
// preds are epsilon if we're not doing preds (we saw an action).
// if we are doing preds, pred must eval to true
if ( !evalPreds ||
(evalPreds && parser.sempred(originalContext, pt.ruleIndex, pt.predIndex)) ) {
// Cannot exec preds out of context if they are context dependent
RuleContext ctx = config.context;
if ( ctx == RuleContext.EMPTY ) ctx = originalContext;
boolean ctxIssue = pt.isCtxDependent && config.reachesIntoOuterContext>0;
boolean seeThroughPred =
ignorePreds || ctxIssue ||
(!ctxIssue && parser.sempred(ctx, pt.ruleIndex, pt.predIndex));
if ( seeThroughPred ) {
c = new ATNConfig(config, t.target);
c.traversedPredicate = true;
}
@ -512,7 +532,9 @@ public class ParserATNSimulator extends ATNSimulator {
if ( debug ) System.out.println("ACTION edge "+at.ruleIndex+":"+at.actionIndex);
if ( at.actionIndex>=0 ) {
if ( debug ) System.out.println("DO ACTION "+at.ruleIndex+":"+at.actionIndex);
parser.action(originalContext, at.ruleIndex, at.actionIndex);
RuleContext ctx = config.context;
if ( ctx == RuleContext.EMPTY ) ctx = originalContext;
parser.action(ctx, at.ruleIndex, at.actionIndex);
}
else {
// non-forced action traversed to get to t.target

1
runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java
View File

@ -38,6 +38,7 @@ package org.antlr.v4.runtime.atn;
public class PredicateTransition extends Transition {
public int ruleIndex;
public int predIndex;
public boolean isCtxDependent;
public PredicateTransition(ATNState target) {
super(target);

21
runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java
View File

@ -50,14 +50,14 @@ public abstract class Transition {
public static final int EPSILON = 1;
public static final int RANGE = 2;
public static final int RULE = 3;
public static final int PREDICATE = 4;
public static final int ATOM = 5;
public static final int ACTION = 6;
public static final int FORCED_ACTION = 7;
public static final int SET = 8; // ~(A|B) or ~atom, wildcard, which convert to next 2
// public static final int NOT_ATOM = 9;
public static final int NOT_SET = 9;
public static final int WILDCARD = 10;
public static final int PREDICATE = 4; // e.g., {isType(input.LT(1))}?
public static final int DEPENDENT_PREDICATE = 5; // e.g., {$p>3}?
public static final int ATOM = 6;
public static final int ACTION = 7;
public static final int FORCED_ACTION = 8;
public static final int SET = 9; // ~(A|B) or ~atom, wildcard, which convert to next 2
public static final int NOT_SET = 10;
public static final int WILDCARD = 11;
public static String[] serializationNames = {
@ -66,11 +66,11 @@ public abstract class Transition {
"RANGE",
"RULE",
"PREDICATE",
"DEPENDENT_PREDICATE",
"ATOM",
"ACTION",
"FORCED_ACTION",
"SET",
// "NOT_ATOM",
"NOT_SET",
"WILDCARD",
};
@ -82,9 +82,8 @@ public abstract class Transition {
put(RuleTransition.class, RULE);
put(PredicateTransition.class, PREDICATE);
put(AtomTransition.class, ATOM);
put(ActionTransition.class, ACTION); // TODO: FORCED?
put(ActionTransition.class, ACTION);
put(SetTransition.class, SET);
// put(NotAtomTransition.class, NOT_ATOM);
put(NotSetTransition.class, NOT_SET);
put(WildcardTransition.class, WILDCARD);
}};

4
tool/playground/T.g
View File

@ -3,7 +3,7 @@ grammar T;
s : e[9] {true}? ';' ;
e[int i]
: {$i>=0}? ID
: {$i>=0}? {{System.out.println("i=="+$i);}} ID
| ID '!'
;
@ -11,4 +11,4 @@ foo[int j] : e[8] {$j==2}? '$' ; // not called but in FOLLOW(e)
ID : 'a'..'z'+;
WS : ' '+ {skip();} ;
WS : (' '|'\n')+ {skip();} ;

2
tool/resources/org/antlr/v4/tool/templates/LeftRecursiveRules.stg
View File

@ -62,7 +62,7 @@ recRule(ruleName, precArgDef, argName, alts, setResultAction, buildAST,
}
<endif>
(
(<alts; separator="\n | ">)
<alts; separator="\n | ">
)*
;
>>

1
tool/src/org/antlr/v4/automata/ATNSerializer.java
View File

@ -143,6 +143,7 @@ public class ATNSerializer {
PredicateTransition pt = (PredicateTransition)t;
arg1 = pt.ruleIndex;
arg2 = pt.predIndex;
if ( pt.isCtxDependent ) edgeType = Transition.DEPENDENT_PREDICATE;
break;
case Transition.RANGE :
arg1 = ((RangeTransition)t).from;

2
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -36,6 +36,7 @@ import org.antlr.v4.misc.CharSupport;
import org.antlr.v4.parse.*;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.semantics.UseDefAnalyzer;
import org.antlr.v4.tool.*;
import java.lang.reflect.Constructor;
@ -222,6 +223,7 @@ public class ParserATNFactory implements ATNFactory {
PredicateTransition p = new PredicateTransition(right);
p.ruleIndex = currentRule.index;
p.predIndex = g.sempreds.get(pred);
p.isCtxDependent = UseDefAnalyzer.actionIsContextDependent(pred);
left.transition = p;
pred.atnState = left;
return new Handle(left, right);

32
tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java
View File

@ -29,8 +29,8 @@
package org.antlr.v4.semantics;
import org.antlr.runtime.Token;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.runtime.*;
import org.antlr.v4.parse.*;
import org.antlr.v4.tool.*;
import java.util.*;
@ -75,6 +75,34 @@ public class UseDefAnalyzer {
}
}
public static boolean actionIsContextDependent(PredAST predAST) {
ANTLRStringStream in = new ANTLRStringStream(predAST.token.getText());
in.setLine(predAST.token.getLine());
in.setCharPositionInLine(predAST.token.getCharPositionInLine());
final boolean[] dependent = new boolean[] {false}; // can't be simple bool with anon class
ActionSplitterListener listener = new BlankActionSplitterListener() {
@Override
public void nonLocalAttr(String expr, Token x, Token y) { dependent[0] = true; }
@Override
public void qualifiedAttr(String expr, Token x, Token y) { dependent[0] = true; }
@Override
public void setAttr(String expr, Token x, Token rhs) { dependent[0] = true; }
@Override
public void setExprAttribute(String expr) { dependent[0] = true; }
@Override
public void setNonLocalAttr(String expr, Token x, Token y, Token rhs) { dependent[0] = true; }
@Override
public void setQualifiedAttr(String expr, Token x, Token y, Token rhs) { dependent[0] = true; }
@Override
public void attr(String expr, Token x) { dependent[0] = true; }
};
ActionSplitter splitter = new ActionSplitter(in, listener);
// forces eval, triggers listener methods
splitter.getActionTokens();
System.out.println("action "+predAST.getText()+" ctx depends="+dependent[0]);
return dependent[0];
}
/** Given -> (ALT ...), return list of element refs at
* top level
*/