jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge pull request #531 from sharwell/eof-transitions 

Treat EOF transitions in the ATN as epsilon transitions...
This commit is contained in:
Terence Parr 2014-04-01 20:52:34 -07:00
parent 6de95c15e7 bbbf5e34db
commit 2a0857b7d5
2 changed files with 53 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -351,7 +351,8 @@ public class LexerATNSimulator extends ATNSimulator {
lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index() - startIndex);
}
if (closure(input, new LexerATNConfig((LexerATNConfig)c, target, lexerActionExecutor), reach, currentAltReachedAcceptState, true)) {
boolean treatEofAsEpsilon = t == CharStream.EOF;
if (closure(input, new LexerATNConfig((LexerATNConfig)c, target, lexerActionExecutor), reach, currentAltReachedAcceptState, true, treatEofAsEpsilon)) {
// any remaining configs for this alt have a lower priority than
// the one that just reached an accept state.
skipAlt = c.alt;
@ -400,7 +401,7 @@ public class LexerATNSimulator extends ATNSimulator {
for (int i=0; i<p.getNumberOfTransitions(); i++) {
ATNState target = p.transition(i).target;
LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
closure(input, c, configs, false, false);
closure(input, c, configs, false, false, false);
}
return configs;
}
@ -415,7 +416,7 @@ public class LexerATNSimulator extends ATNSimulator {
* @return {@code true} if an accept state is reached, otherwise
* {@code false}.
*/
protected boolean closure(@NotNull CharStream input, @NotNull LexerATNConfig config, @NotNull ATNConfigSet configs, boolean currentAltReachedAcceptState, boolean speculative) {
protected boolean closure(@NotNull CharStream input, @NotNull LexerATNConfig config, @NotNull ATNConfigSet configs, boolean currentAltReachedAcceptState, boolean speculative, boolean treatEofAsEpsilon) {
if ( debug ) {
System.out.println("closure("+config.toString(recog, true)+")");
}
@ -447,7 +448,7 @@ public class LexerATNSimulator extends ATNSimulator {
PredictionContext newContext = config.context.getParent(i); // "pop" return state
ATNState returnState = atn.states.get(config.context.getReturnState(i));
LexerATNConfig c = new LexerATNConfig(config, returnState, newContext);
currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative);
currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon);
}
}
}
@ -465,9 +466,9 @@ public class LexerATNSimulator extends ATNSimulator {
ATNState p = config.state;
for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i);
LexerATNConfig c = getEpsilonTarget(input, config, t, configs, speculative);
LexerATNConfig c = getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon);
if ( c!=null ) {
currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative);
currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon);
}
}
@ -480,7 +481,8 @@ public class LexerATNSimulator extends ATNSimulator {
@NotNull LexerATNConfig config,
@NotNull Transition t,
@NotNull ATNConfigSet configs,
boolean speculative)
boolean speculative,
boolean treatEofAsEpsilon)
{
LexerATNConfig c = null;
switch (t.getSerializationType()) {
@ -550,6 +552,18 @@ public class LexerATNSimulator extends ATNSimulator {
case Transition.EPSILON:
c = new LexerATNConfig(config, t.target);
break;
case Transition.ATOM:
case Transition.RANGE:
case Transition.SET:
if (treatEofAsEpsilon) {
if (t.matches(CharStream.EOF, Character.MIN_VALUE, Character.MAX_VALUE)) {
c = new LexerATNConfig(config, t.target);
break;
}
}
break;
}
return c;

44
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -868,8 +868,9 @@ public class ParserATNSimulator extends ATNSimulator {
if (reach == null) {
reach = new ATNConfigSet(fullCtx);
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
boolean treatEofAsEpsilon = t == Token.EOF;
for (ATNConfig c : intermediate) {
closure(c, reach, closureBusy, false, fullCtx);
closure(c, reach, closureBusy, false, fullCtx, treatEofAsEpsilon);
}
}
@ -971,7 +972,7 @@ public class ParserATNSimulator extends ATNSimulator {
ATNState target = p.transition(i).target;
ATNConfig c = new ATNConfig(target, i+1, initialContext);
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
closure(c, configs, closureBusy, true, fullCtx);
closure(c, configs, closureBusy, true, fullCtx, false);
}
return configs;
@ -1218,12 +1219,13 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull ATNConfigSet configs,
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean fullCtx)
boolean fullCtx,
boolean treatEofAsEpsilon)
{
final int initialDepth = 0;
closureCheckingStopState(config, configs, closureBusy, collectPredicates,
fullCtx,
initialDepth);
initialDepth, treatEofAsEpsilon);
assert !fullCtx || !configs.dipsIntoOuterContext;
}
@ -1232,7 +1234,8 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean fullCtx,
int depth)
int depth,
boolean treatEofAsEpsilon)
{
if ( debug ) System.out.println("closure("+config.toString(parser,true)+")");
@ -1251,7 +1254,7 @@ public class ParserATNSimulator extends ATNSimulator {
if ( debug ) System.out.println("FALLING off rule "+
getRuleName(config.state.ruleIndex));
closure_(config, configs, closureBusy, collectPredicates,
fullCtx, depth);
fullCtx, depth, treatEofAsEpsilon);
}
continue;
}
@ -1265,7 +1268,7 @@ public class ParserATNSimulator extends ATNSimulator {
c.reachesIntoOuterContext = config.reachesIntoOuterContext;
assert depth > Integer.MIN_VALUE;
closureCheckingStopState(c, configs, closureBusy, collectPredicates,
fullCtx, depth - 1);
fullCtx, depth - 1, treatEofAsEpsilon);
}
return;
}
@ -1282,7 +1285,7 @@ public class ParserATNSimulator extends ATNSimulator {
}
closure_(config, configs, closureBusy, collectPredicates,
fullCtx, depth);
fullCtx, depth, treatEofAsEpsilon);
}
/** Do the actual work of walking epsilon edges */
@ -1291,12 +1294,15 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean fullCtx,
int depth)
int depth,
boolean treatEofAsEpsilon)
{
ATNState p = config.state;
// optimization
if ( !p.onlyHasEpsilonTransitions() ) {
configs.add(config, mergeCache);
// make sure to not return here, because EOF transitions can act as
// both epsilon transitions and non-epsilon transitions.
// if ( debug ) System.out.println("added config "+configs);
}
@ -1305,7 +1311,7 @@ public class ParserATNSimulator extends ATNSimulator {
boolean continueCollecting =
!(t instanceof ActionTransition) && collectPredicates;
ATNConfig c = getEpsilonTarget(config, t, continueCollecting,
depth == 0, fullCtx);
depth == 0, fullCtx, treatEofAsEpsilon);
if ( c!=null ) {
int newDepth = depth;
if ( config.state instanceof RuleStopState) {
@ -1335,7 +1341,7 @@ public class ParserATNSimulator extends ATNSimulator {
}
closureCheckingStopState(c, configs, closureBusy, continueCollecting,
fullCtx, newDepth);
fullCtx, newDepth, treatEofAsEpsilon);
}
}
}
@ -1351,7 +1357,8 @@ public class ParserATNSimulator extends ATNSimulator {
@NotNull Transition t,
boolean collectPredicates,
boolean inContext,
boolean fullCtx)
boolean fullCtx,
boolean treatEofAsEpsilon)
{
switch (t.getSerializationType()) {
case Transition.RULE:
@ -1372,6 +1379,19 @@ public class ParserATNSimulator extends ATNSimulator {
case Transition.EPSILON:
return new ATNConfig(config, t.target);
case Transition.ATOM:
case Transition.RANGE:
case Transition.SET:
// EOF transitions act like epsilon transitions after the first EOF
// transition is traversed
if (treatEofAsEpsilon) {
if (t.matches(Token.EOF, 0, 1)) {
return new ATNConfig(config, t.target);
}
}
return null;
default:
return null;
}