forked from jasder/antlr
v4: Initial tracing support in the lexer
[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9490]
This commit is contained in:
sharwell
parent
fb3de170d1
commit
1be2e2c556
|
|
@ -182,6 +182,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
|
|||
public void pushMode(int m) {
|
||||
if ( LexerATNSimulator.debug ) System.out.println("pushMode "+m);
|
||||
if ( modeStack==null ) modeStack = new ArrayDeque<Integer>();
|
||||
getInterpreter().tracePushMode(m);
|
||||
modeStack.push(mode);
|
||||
mode(m);
|
||||
}
|
||||
|
|
@ -189,6 +190,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
|
|||
public int popMode() {
|
||||
if ( modeStack==null ) throw new EmptyStackException();
|
||||
if ( LexerATNSimulator.debug ) System.out.println("popMode back to "+modeStack.peek());
|
||||
getInterpreter().tracePopMode();
|
||||
mode( modeStack.pop() );
|
||||
return mode;
|
||||
}
|
||||
|
|
@ -217,6 +219,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
|
|||
* than a single variable as this implementation does).
|
||||
*/
|
||||
public void emit(Token token) {
|
||||
getInterpreter().traceEmit(token);
|
||||
//System.err.println("emit "+token);
|
||||
this.token = token;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,12 +36,21 @@ import org.antlr.v4.runtime.misc.NotNull;
|
|||
import org.antlr.v4.runtime.misc.Nullable;
|
||||
import org.antlr.v4.runtime.misc.OrderedHashSet;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import org.antlr.v4.runtime.misc.Interval;
|
||||
import org.antlr.v4.runtime.misc.IntervalSet;
|
||||
|
||||
/** "dup" of ParserInterpreter */
|
||||
public class LexerATNSimulator extends ATNSimulator {
|
||||
public static boolean debug = false;
|
||||
public static boolean dfa_debug = false;
|
||||
public static final int NUM_EDGES = 255;
|
||||
|
||||
private boolean trace = false;
|
||||
private OutputStream traceStream = null;
|
||||
private boolean traceFailed = false;
|
||||
|
||||
/** When we hit an accept state in either the DFA or the ATN, we
|
||||
* have to notify the character stream to start buffering characters
|
||||
* via mark() and record the current state. The current state includes
|
||||
|
|
@ -131,10 +140,17 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
this.recog = recog;
|
||||
}
|
||||
|
||||
public void setTraceStream(OutputStream traceStream) {
|
||||
this.traceStream = traceStream;
|
||||
this.trace = traceStream != null;
|
||||
this.traceFailed = false;
|
||||
}
|
||||
|
||||
public int match(@NotNull CharStream input, int mode) {
|
||||
match_calls++;
|
||||
this.mode = mode;
|
||||
int mark = input.mark();
|
||||
traceBeginMatch(input, mode);
|
||||
try {
|
||||
if ( dfa[mode].s0==null ) {
|
||||
return matchATN(input);
|
||||
|
|
@ -144,6 +160,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
}
|
||||
}
|
||||
finally {
|
||||
traceEndMatch();
|
||||
input.release(mark);
|
||||
}
|
||||
}
|
||||
|
|
@ -159,6 +176,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
|
||||
// only called from test code from outside
|
||||
public int matchATN(@NotNull CharStream input) {
|
||||
traceMatchATN();
|
||||
startIndex = input.index();
|
||||
ATNState startState = atn.modeToStartState.get(mode);
|
||||
if ( debug ) System.out.println("mode "+ mode +" start: "+startState);
|
||||
|
|
@ -167,10 +185,12 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
dfa[mode].s0 = addDFAState(s0_closure);
|
||||
int predict = exec(input, s0_closure);
|
||||
if ( debug ) System.out.println("DFA after matchATN: "+dfa[old_mode].toLexerString());
|
||||
tracePredict(predict);
|
||||
return predict;
|
||||
}
|
||||
|
||||
protected int exec(@NotNull CharStream input, @NotNull DFAState s0) {
|
||||
traceMatchDFA();
|
||||
if ( dfa_debug ) System.out.println("DFA[mode "+(recog==null?0:recog.mode)+"] exec LA(1)=="+
|
||||
(char)input.LA(1));
|
||||
//System.out.println("DFA start of execDFA: "+dfa[mode].toLexerString());
|
||||
|
|
@ -178,6 +198,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
dfaPrevAccept.reset();
|
||||
LexerNoViableAltException atnException = null;
|
||||
DFAState s = s0;
|
||||
traceLookahead1();
|
||||
int t = input.LA(1);
|
||||
loop:
|
||||
while ( true ) {
|
||||
|
|
@ -211,6 +232,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
}
|
||||
|
||||
consume(input);
|
||||
traceLookahead1();
|
||||
t = input.LA(1);
|
||||
}
|
||||
if ( dfaPrevAccept.state==null ) {
|
||||
|
|
@ -224,6 +246,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
|
||||
int ruleIndex = dfaPrevAccept.state.ruleIndex;
|
||||
accept(input, ruleIndex, dfaPrevAccept);
|
||||
tracePredict(dfaPrevAccept.state.prediction);
|
||||
return dfaPrevAccept.state.prediction;
|
||||
}
|
||||
|
||||
|
|
@ -238,6 +261,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
OrderedHashSet<ATNConfig> reach = new OrderedHashSet<ATNConfig>();
|
||||
atnPrevAccept.reset();
|
||||
|
||||
traceLookahead1();
|
||||
int t = input.LA(1);
|
||||
|
||||
do { // while more work
|
||||
|
|
@ -272,6 +296,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
|
||||
consume(input);
|
||||
addDFAEdge(closure, t, reach);
|
||||
traceLookahead1();
|
||||
t = input.LA(1);
|
||||
|
||||
// swap to avoid reallocating space
|
||||
|
|
@ -309,6 +334,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
}
|
||||
int index = input.index();
|
||||
if ( index > atnPrevAccept.index ) {
|
||||
traceAcceptState(c.alt);
|
||||
// will favor prev accept at same index so "int" is keyword not ID
|
||||
markAcceptState(atnPrevAccept, input);
|
||||
atnPrevAccept.config = c;
|
||||
|
|
@ -340,6 +366,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
if ( actionIndex>=0 && recog!=null ) recog.action(null, ruleIndex, actionIndex);
|
||||
|
||||
// seek to after last char in token
|
||||
traceSeek(prevAccept.index);
|
||||
input.seek(prevAccept.index);
|
||||
line = prevAccept.line;
|
||||
charPositionInLine = prevAccept.charPos;
|
||||
|
|
@ -479,6 +506,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
}
|
||||
|
||||
int failOverToATN(@NotNull CharStream input, @NotNull DFAState s) {
|
||||
traceFailOverToATN();
|
||||
if ( dfa_debug ) System.out.println("no edge for "+(char)input.LA(1));
|
||||
if ( dfa_debug ) {
|
||||
System.out.println("ATN exec upon "+
|
||||
|
|
@ -493,6 +521,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
}
|
||||
// action already executed by ATN
|
||||
// we've updated DFA, exec'd action, and have our deepest answer
|
||||
tracePredict(ttype);
|
||||
return ttype;
|
||||
}
|
||||
|
||||
|
|
@ -610,6 +639,7 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
charPositionInLine++;
|
||||
}
|
||||
input.consume();
|
||||
traceConsume(input, curChar);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
|
|
@ -619,4 +649,243 @@ public class LexerATNSimulator extends ATNSimulator {
|
|||
return "'"+(char)t+"'";
|
||||
}
|
||||
|
||||
/*
|
||||
* Trace helpers (API and file format are work in progress)
|
||||
*/
|
||||
|
||||
public void traceEndMatch() {
|
||||
if (trace) {
|
||||
traceSlow(LexerOpCode.EndMatch);
|
||||
}
|
||||
}
|
||||
|
||||
public void traceMatchATN() {
|
||||
if (trace) {
|
||||
traceSlow(LexerOpCode.MatchATN);
|
||||
}
|
||||
}
|
||||
|
||||
public void traceMatchDFA() {
|
||||
if (trace) {
|
||||
traceSlow(LexerOpCode.MatchDFA);
|
||||
}
|
||||
}
|
||||
|
||||
public void traceLookahead1() {
|
||||
if (trace) {
|
||||
traceSlow(LexerOpCode.Lookahead1);
|
||||
}
|
||||
}
|
||||
|
||||
public void traceFailOverToATN() {
|
||||
if (trace) {
|
||||
traceSlow(LexerOpCode.FailOverToATN);
|
||||
}
|
||||
}
|
||||
|
||||
public void tracePredict(int prediction) {
|
||||
if (trace) {
|
||||
traceIntSlow(LexerOpCode.Predict, prediction);
|
||||
}
|
||||
}
|
||||
|
||||
public void traceAcceptState(int prediction) {
|
||||
if (trace) {
|
||||
traceIntSlow(LexerOpCode.AcceptState, prediction);
|
||||
}
|
||||
}
|
||||
|
||||
public void traceSeek(int index) {
|
||||
if (trace) {
|
||||
traceIntSlow(LexerOpCode.Seek, index);
|
||||
}
|
||||
}
|
||||
|
||||
public final void traceBeginMatch(CharStream input, int mode) {
|
||||
if (trace) {
|
||||
traceBeginMatchSlow(input, mode);
|
||||
}
|
||||
}
|
||||
|
||||
public final void traceConsume(CharStream input, int c) {
|
||||
if (trace) {
|
||||
traceConsumeSlow(input, c);
|
||||
}
|
||||
}
|
||||
|
||||
public final void tracePushMode(int mode) {
|
||||
if (trace) {
|
||||
traceByteSlow(LexerOpCode.PushMode, (byte)mode);
|
||||
}
|
||||
}
|
||||
|
||||
public final void tracePopMode() {
|
||||
if (trace) {
|
||||
traceSlow(LexerOpCode.PopMode);
|
||||
}
|
||||
}
|
||||
|
||||
public final void traceEmit(Token token) {
|
||||
if (trace) {
|
||||
traceEmitSlow(token);
|
||||
}
|
||||
}
|
||||
|
||||
private void traceSlow(LexerOpCode opcode) {
|
||||
assert traceStream != null;
|
||||
assert opcode.getArgumentSize() == 0;
|
||||
|
||||
if (!traceFailed) {
|
||||
try {
|
||||
traceStream.write(opcode.ordinal());
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
traceFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void traceByteSlow(LexerOpCode opcode, byte arg) {
|
||||
assert traceStream != null;
|
||||
assert opcode.getArgumentSize() == 1;
|
||||
|
||||
if (!traceFailed) {
|
||||
try {
|
||||
traceStream.write(opcode.ordinal());
|
||||
traceStream.write(arg);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
traceFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void traceByteIntSlow(LexerOpCode opcode, byte arg1, int arg2) {
|
||||
assert traceStream != null;
|
||||
assert opcode.getArgumentSize() == 5;
|
||||
|
||||
if (!traceFailed) {
|
||||
try {
|
||||
traceStream.write(opcode.ordinal());
|
||||
traceStream.write(arg1);
|
||||
traceIntSlow(arg2);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
traceFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void traceIntSlow(LexerOpCode opcode, int arg) {
|
||||
assert traceStream != null;
|
||||
assert opcode.getArgumentSize() == 4;
|
||||
|
||||
if (!traceFailed) {
|
||||
try {
|
||||
traceStream.write(opcode.ordinal());
|
||||
traceIntSlow(arg);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
traceFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void traceIntIntSlow(LexerOpCode opcode, int arg1, int arg2) {
|
||||
assert traceStream != null;
|
||||
assert opcode.getArgumentSize() == 8;
|
||||
|
||||
if (!traceFailed) {
|
||||
try {
|
||||
traceStream.write(opcode.ordinal());
|
||||
traceIntSlow(arg1);
|
||||
traceIntSlow(arg2);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
traceFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void traceIntIntIntIntSlow(LexerOpCode opcode, int arg1, int arg2, int arg3, int arg4) {
|
||||
assert traceStream != null;
|
||||
assert opcode.getArgumentSize() == 16;
|
||||
|
||||
if (!traceFailed) {
|
||||
try {
|
||||
traceStream.write(opcode.ordinal());
|
||||
traceIntSlow(arg1);
|
||||
traceIntSlow(arg2);
|
||||
traceIntSlow(arg3);
|
||||
traceIntSlow(arg4);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
traceFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void traceIntSlow(int arg) {
|
||||
assert traceStream != null;
|
||||
|
||||
if (!traceFailed) {
|
||||
try {
|
||||
traceStream.write(arg);
|
||||
traceStream.write(arg >> 8);
|
||||
traceStream.write(arg >> 16);
|
||||
traceStream.write(arg >> 24);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
traceFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void traceBeginMatchSlow(CharStream input, int mode) {
|
||||
traceByteIntSlow(LexerOpCode.BeginMatch, (byte)mode, input.index());
|
||||
}
|
||||
|
||||
private void traceConsumeSlow(CharStream input, int c) {
|
||||
assert traceStream != null;
|
||||
|
||||
if (!traceFailed) {
|
||||
traceIntIntSlow(LexerOpCode.Consume, c, input.index());
|
||||
}
|
||||
}
|
||||
|
||||
private void traceEmitSlow(Token token) {
|
||||
assert traceStream != null;
|
||||
|
||||
if (token != null && !traceFailed) {
|
||||
traceIntIntIntIntSlow(LexerOpCode.Emit, token.getStartIndex(), token.getStopIndex(), token.getType(), token.getChannel());
|
||||
}
|
||||
}
|
||||
|
||||
public enum LexerOpCode {
|
||||
BeginMatch(5),
|
||||
EndMatch(0),
|
||||
MatchATN(0),
|
||||
MatchDFA(0),
|
||||
FailOverToATN(0),
|
||||
AcceptState(4),
|
||||
Predict(4),
|
||||
|
||||
Seek(4),
|
||||
Consume(8),
|
||||
Lookahead1(0),
|
||||
|
||||
PushMode(1),
|
||||
PopMode(0),
|
||||
Emit(16);
|
||||
|
||||
private final int argumentSize;
|
||||
|
||||
private LexerOpCode(int argumentSize) {
|
||||
this.argumentSize = argumentSize;
|
||||
}
|
||||
|
||||
public int getArgumentSize() {
|
||||
return argumentSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue