jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge pull request #503 from sharwell/fix-469 

Fix 469
This commit is contained in:
Terence Parr 2014-03-23 11:25:59 -07:00
parent 0577b4dcaa bc646df824
commit e66f663618
3 changed files with 67 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java
View File

@ -36,8 +36,11 @@ import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.ObjectEqualityComparator;
public class LexerATNConfig extends ATNConfig {
/** Capture lexer actions we traverse. */
public LexerActionExecutor lexerActionExecutor;
/**
* This is the backing field for {@link #getLexerActionExecutor}.
*/
@Nullable
private final LexerActionExecutor lexerActionExecutor;
private final boolean passedThroughNonGreedyDecision;
@ -47,12 +50,13 @@ public class LexerATNConfig extends ATNConfig {
{
super(state, alt, context, SemanticContext.NONE);
this.passedThroughNonGreedyDecision = false;
this.lexerActionExecutor = null;
}
public LexerATNConfig(@NotNull ATNState state,
int alt,
@Nullable PredictionContext context,
LexerActionExecutor lexerActionExecutor)
@Nullable LexerActionExecutor lexerActionExecutor)
{
super(state, alt, context, SemanticContext.NONE);
this.lexerActionExecutor = lexerActionExecutor;
@ -66,7 +70,7 @@ public class LexerATNConfig extends ATNConfig {
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
LexerActionExecutor lexerActionExecutor)
@Nullable LexerActionExecutor lexerActionExecutor)
{
super(c, state, c.context, c.semanticContext);
this.lexerActionExecutor = lexerActionExecutor;
@ -80,6 +84,15 @@ public class LexerATNConfig extends ATNConfig {
this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
}
/**
* Gets the {@link LexerActionExecutor} capable of executing the embedded
* action(s) for the current configuration.
*/
@Nullable
public final LexerActionExecutor getLexerActionExecutor() {
return lexerActionExecutor;
}
public final boolean hasPassedThroughNonGreedyDecision() {
return passedThroughNonGreedyDecision;
}

8
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -346,7 +346,7 @@ public class LexerATNSimulator extends ATNSimulator {
Transition trans = c.state.transition(ti);
ATNState target = getReachableTarget(trans, t);
if ( target!=null ) {
LexerActionExecutor lexerActionExecutor = ((LexerATNConfig)c).lexerActionExecutor;
LexerActionExecutor lexerActionExecutor = ((LexerATNConfig)c).getLexerActionExecutor();
if (lexerActionExecutor != null) {
lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index() - startIndex);
}
@ -446,7 +446,7 @@ public class LexerATNSimulator extends ATNSimulator {
if (config.context.getReturnState(i) != PredictionContext.EMPTY_RETURN_STATE) {
PredictionContext newContext = config.context.getParent(i); // "pop" return state
ATNState returnState = atn.states.get(config.context.getReturnState(i));
LexerATNConfig c = new LexerATNConfig(returnState, config.alt, newContext);
LexerATNConfig c = new LexerATNConfig(config, returnState, newContext);
currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative);
}
}
@ -537,7 +537,7 @@ public class LexerATNSimulator extends ATNSimulator {
// getEpsilonTarget to return two configurations, so
// additional modifications are needed before we can support
// the split operation.
LexerActionExecutor lexerActionExecutor = LexerActionExecutor.append(config.lexerActionExecutor, atn.lexerActions[((ActionTransition)t).actionIndex]);
LexerActionExecutor lexerActionExecutor = LexerActionExecutor.append(config.getLexerActionExecutor(), atn.lexerActions[((ActionTransition)t).actionIndex]);
c = new LexerATNConfig(config, t.target, lexerActionExecutor);
break;
}
@ -684,7 +684,7 @@ public class LexerATNSimulator extends ATNSimulator {
if ( firstConfigWithRuleStopState!=null ) {
proposed.isAcceptState = true;
proposed.lexerActionExecutor = ((LexerATNConfig)firstConfigWithRuleStopState).lexerActionExecutor;
proposed.lexerActionExecutor = ((LexerATNConfig)firstConfigWithRuleStopState).getLexerActionExecutor();
proposed.prediction = atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex];
}

46
tool/test/org/antlr/v4/test/TestLexerActions.java
View File

@ -37,6 +37,52 @@ public class TestLexerActions extends BaseTest {
assertEquals(expecting, found);
}
/**
* This is a regressing test for antlr/antlr4#469 "Not all internal lexer
* rule actions are executed".
* https://github.com/antlr/antlr4/issues/469
*/
@Test public void testEvalMultipleActions() throws Exception {
String grammar =
"lexer grammar L;\n" +
"\n" +
"@lexer::members\n" +
"{\n" +
"class Marker\n" +
"{\n" +
" Marker (Lexer lexer) { this.lexer = lexer; }\n" +
"\n" +
" public String getText ()\n" +
" {\n" +
" return lexer._input.getText (new Interval (start_index, stop_index));\n" +
" }\n" +
"\n" +
" public void start () { start_index = lexer._input.index (); System.out.println (\"Start:\" + start_index);}\n" +
" public void stop () { stop_index = lexer._input.index (); System.out.println (\"Stop:\" + stop_index);}\n" +
"\n" +
" private int start_index = 0;\n" +
" private int stop_index = 0;\n" +
" private Lexer lexer;\n" +
"}\n" +
"\n" +
"Marker m_name = new Marker (this);\n" +
"}\n" +
"\n" +
"HELLO: 'hello' WS { m_name.start (); } NAME { m_name.stop (); } '\\n' { System.out.println (\"Hello: \" + m_name.getText ()); };\n" +
"NAME: ('a'..'z' | 'A'..'Z')+ ('\\n')?;\n" +
"\n" +
"fragment WS: [ \\r\\t\\n]+ ;\n";
String found = execLexer("L.g4", grammar, "L", "hello Steve\n");
String expecting =
"Start:6\n" +
"Stop:11\n" +
"Hello: Steve\n" +
"\n" +
"[@0,0:11='hello Steve\\n',<1>,1:0]\n" +
"[@1,12:11='<EOF>',<-1>,2:12]\n";
assertEquals(expecting, found);
}
@Test public void test2ActionsIn1Rule() throws Exception {
String grammar =
"lexer grammar L;\n"+