forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #531 from sharwell/eof-transitions 

Treat EOF transitions in the ATN as epsilon transitions...
This commit is contained in:
Terence Parr 2014-04-01 20:52:34 -07:00
parent 6de95c15e7 bbbf5e34db
commit 2a0857b7d5
2 changed files with 53 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -351,7 +351,8 @@ public class LexerATNSimulator extends ATNSimulator {
lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index() - startIndex); lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index() - startIndex);
} }
if (closure(input, new LexerATNConfig((LexerATNConfig)c, target, lexerActionExecutor), reach, currentAltReachedAcceptState, true)) { boolean treatEofAsEpsilon = t == CharStream.EOF;
if (closure(input, new LexerATNConfig((LexerATNConfig)c, target, lexerActionExecutor), reach, currentAltReachedAcceptState, true, treatEofAsEpsilon)) {
// any remaining configs for this alt have a lower priority than // any remaining configs for this alt have a lower priority than
// the one that just reached an accept state. // the one that just reached an accept state.
skipAlt = c.alt; skipAlt = c.alt;
@ -400,7 +401,7 @@ public class LexerATNSimulator extends ATNSimulator {
for (int i=0; i<p.getNumberOfTransitions(); i++) { for (int i=0; i<p.getNumberOfTransitions(); i++) {
ATNState target = p.transition(i).target; ATNState target = p.transition(i).target;
LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext); LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
closure(input, c, configs, false, false); closure(input, c, configs, false, false, false);
} }
return configs; return configs;
} }
@ -415,7 +416,7 @@ public class LexerATNSimulator extends ATNSimulator {
* @return {@code true} if an accept state is reached, otherwise * @return {@code true} if an accept state is reached, otherwise
* {@code false}. * {@code false}.
*/ */
protected boolean closure(@NotNull CharStream input, @NotNull LexerATNConfig config, @NotNull ATNConfigSet configs, boolean currentAltReachedAcceptState, boolean speculative) { protected boolean closure(@NotNull CharStream input, @NotNull LexerATNConfig config, @NotNull ATNConfigSet configs, boolean currentAltReachedAcceptState, boolean speculative, boolean treatEofAsEpsilon) {
if ( debug ) { if ( debug ) {
System.out.println("closure("+config.toString(recog, true)+")"); System.out.println("closure("+config.toString(recog, true)+")");
} }
@ -447,7 +448,7 @@ public class LexerATNSimulator extends ATNSimulator {
PredictionContext newContext = config.context.getParent(i); // "pop" return state PredictionContext newContext = config.context.getParent(i); // "pop" return state
ATNState returnState = atn.states.get(config.context.getReturnState(i)); ATNState returnState = atn.states.get(config.context.getReturnState(i));
LexerATNConfig c = new LexerATNConfig(config, returnState, newContext); LexerATNConfig c = new LexerATNConfig(config, returnState, newContext);
currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative); currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon);
} }
} }
} }
@ -465,9 +466,9 @@ public class LexerATNSimulator extends ATNSimulator {
ATNState p = config.state; ATNState p = config.state;
for (int i=0; i<p.getNumberOfTransitions(); i++) { for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i); Transition t = p.transition(i);
LexerATNConfig c = getEpsilonTarget(input, config, t, configs, speculative); LexerATNConfig c = getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon);
if ( c!=null ) { if ( c!=null ) {
currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative); currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon);
} }
} }
@ -480,7 +481,8 @@ public class LexerATNSimulator extends ATNSimulator {
@NotNull LexerATNConfig config, @NotNull LexerATNConfig config,
@NotNull Transition t, @NotNull Transition t,
@NotNull ATNConfigSet configs, @NotNull ATNConfigSet configs,
boolean speculative) boolean speculative,
boolean treatEofAsEpsilon)
{ {
LexerATNConfig c = null; LexerATNConfig c = null;
switch (t.getSerializationType()) { switch (t.getSerializationType()) {
@ -550,6 +552,18 @@ public class LexerATNSimulator extends ATNSimulator {
case Transition.EPSILON: case Transition.EPSILON:
c = new LexerATNConfig(config, t.target); c = new LexerATNConfig(config, t.target);
break; break;
case Transition.ATOM:
case Transition.RANGE:
case Transition.SET:
if (treatEofAsEpsilon) {
if (t.matches(CharStream.EOF, Character.MIN_VALUE, Character.MAX_VALUE)) {
c = new LexerATNConfig(config, t.target);
break;
}
}
break;
} }
return c; return c;

44
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -868,8 +868,9 @@ public class ParserATNSimulator extends ATNSimulator {
if (reach == null) { if (reach == null) {
reach = new ATNConfigSet(fullCtx); reach = new ATNConfigSet(fullCtx);
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>(); Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
boolean treatEofAsEpsilon = t == Token.EOF;
for (ATNConfig c : intermediate) { for (ATNConfig c : intermediate) {
closure(c, reach, closureBusy, false, fullCtx); closure(c, reach, closureBusy, false, fullCtx, treatEofAsEpsilon);
} }
} }
@ -971,7 +972,7 @@ public class ParserATNSimulator extends ATNSimulator {
ATNState target = p.transition(i).target; ATNState target = p.transition(i).target;
ATNConfig c = new ATNConfig(target, i+1, initialContext); ATNConfig c = new ATNConfig(target, i+1, initialContext);
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>(); Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
closure(c, configs, closureBusy, true, fullCtx); closure(c, configs, closureBusy, true, fullCtx, false);
} }
return configs; return configs;
@ -1218,12 +1219,13 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull ATNConfigSet configs, @NotNull ATNConfigSet configs,
@NotNull Set<ATNConfig> closureBusy, @NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates, boolean collectPredicates,
boolean fullCtx) boolean fullCtx,
boolean treatEofAsEpsilon)
{ {
final int initialDepth = 0; final int initialDepth = 0;
closureCheckingStopState(config, configs, closureBusy, collectPredicates, closureCheckingStopState(config, configs, closureBusy, collectPredicates,
fullCtx, fullCtx,
initialDepth); initialDepth, treatEofAsEpsilon);
assert !fullCtx || !configs.dipsIntoOuterContext; assert !fullCtx || !configs.dipsIntoOuterContext;
} }
@ -1232,7 +1234,8 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull Set<ATNConfig> closureBusy, @NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates, boolean collectPredicates,
boolean fullCtx, boolean fullCtx,
int depth) int depth,
boolean treatEofAsEpsilon)
{ {
if ( debug ) System.out.println("closure("+config.toString(parser,true)+")"); if ( debug ) System.out.println("closure("+config.toString(parser,true)+")");
@ -1251,7 +1254,7 @@ public class ParserATNSimulator extends ATNSimulator {
if ( debug ) System.out.println("FALLING off rule "+ if ( debug ) System.out.println("FALLING off rule "+
getRuleName(config.state.ruleIndex)); getRuleName(config.state.ruleIndex));
closure_(config, configs, closureBusy, collectPredicates, closure_(config, configs, closureBusy, collectPredicates,
fullCtx, depth); fullCtx, depth, treatEofAsEpsilon);
} }
continue; continue;
} }
@ -1265,7 +1268,7 @@ public class ParserATNSimulator extends ATNSimulator {
c.reachesIntoOuterContext = config.reachesIntoOuterContext; c.reachesIntoOuterContext = config.reachesIntoOuterContext;
assert depth > Integer.MIN_VALUE; assert depth > Integer.MIN_VALUE;
closureCheckingStopState(c, configs, closureBusy, collectPredicates, closureCheckingStopState(c, configs, closureBusy, collectPredicates,
fullCtx, depth - 1); fullCtx, depth - 1, treatEofAsEpsilon);
} }
return; return;
} }
@ -1282,7 +1285,7 @@ public class ParserATNSimulator extends ATNSimulator {
} }
closure_(config, configs, closureBusy, collectPredicates, closure_(config, configs, closureBusy, collectPredicates,
fullCtx, depth); fullCtx, depth, treatEofAsEpsilon);
} }
/** Do the actual work of walking epsilon edges */ /** Do the actual work of walking epsilon edges */
@ -1291,12 +1294,15 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull Set<ATNConfig> closureBusy, @NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates, boolean collectPredicates,
boolean fullCtx, boolean fullCtx,
int depth) int depth,
boolean treatEofAsEpsilon)
{ {
ATNState p = config.state; ATNState p = config.state;
// optimization // optimization
if ( !p.onlyHasEpsilonTransitions() ) { if ( !p.onlyHasEpsilonTransitions() ) {
configs.add(config, mergeCache); configs.add(config, mergeCache);
// make sure to not return here, because EOF transitions can act as
// both epsilon transitions and non-epsilon transitions.
// if ( debug ) System.out.println("added config "+configs); // if ( debug ) System.out.println("added config "+configs);
} }
@ -1305,7 +1311,7 @@ public class ParserATNSimulator extends ATNSimulator {
boolean continueCollecting = boolean continueCollecting =
!(t instanceof ActionTransition) && collectPredicates; !(t instanceof ActionTransition) && collectPredicates;
ATNConfig c = getEpsilonTarget(config, t, continueCollecting, ATNConfig c = getEpsilonTarget(config, t, continueCollecting,
depth == 0, fullCtx); depth == 0, fullCtx, treatEofAsEpsilon);
if ( c!=null ) { if ( c!=null ) {
int newDepth = depth; int newDepth = depth;
if ( config.state instanceof RuleStopState) { if ( config.state instanceof RuleStopState) {
@ -1335,7 +1341,7 @@ public class ParserATNSimulator extends ATNSimulator {
} }
closureCheckingStopState(c, configs, closureBusy, continueCollecting, closureCheckingStopState(c, configs, closureBusy, continueCollecting,
fullCtx, newDepth); fullCtx, newDepth, treatEofAsEpsilon);
} }
} }
} }
@ -1351,7 +1357,8 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull Transition t, @NotNull Transition t,
boolean collectPredicates, boolean collectPredicates,
boolean inContext, boolean inContext,
boolean fullCtx) boolean fullCtx,
boolean treatEofAsEpsilon)
{ {
switch (t.getSerializationType()) { switch (t.getSerializationType()) {
case Transition.RULE: case Transition.RULE:
@ -1372,6 +1379,19 @@ public class ParserATNSimulator extends ATNSimulator {
case Transition.EPSILON: case Transition.EPSILON:
return new ATNConfig(config, t.target); return new ATNConfig(config, t.target);
case Transition.ATOM:
case Transition.RANGE:
case Transition.SET:
// EOF transitions act like epsilon transitions after the first EOF
// transition is traversed
if (treatEofAsEpsilon) {
if (t.matches(Token.EOF, 0, 1)) {
return new ATNConfig(config, t.target);
}
}
return null;
default: default:
return null; return null;
} }