forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

merging Sam\'s fix to full ctx pred etc...; effectively did pull parrt/antlr4#15, includes fixes antlr/antlr4#14, includes fixes antlr/antlr4#15

This commit is contained in:
Terence Parr 2012-02-20 11:48:03 -08:00
parent 7287f5a2d3 27806dc490
commit 1a2094b2dd
6 changed files with 97 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -225,7 +225,7 @@ import java.util.*;
* when closure operations fall off the end of the rule that * when closure operations fall off the end of the rule that
* holds the decision were evaluating * holds the decision were evaluating
*/ */
public class ParserATNSimulator<Symbol> extends ATNSimulator { public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
public static boolean debug = false; public static boolean debug = false;
public static boolean dfa_debug = false; public static boolean dfa_debug = false;
public static boolean retry_debug = false; public static boolean retry_debug = false;
@ -261,15 +261,14 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
public void reset() { public void reset() {
} }
public int adaptivePredict(@NotNull SymbolStream<Token> input, int decision, public int adaptivePredict(@NotNull SymbolStream<? extends Symbol> input, int decision,
@Nullable ParserRuleContext outerContext) @Nullable ParserRuleContext<?> outerContext)
{ {
predict_calls++; predict_calls++;
DFA dfa = decisionToDFA[decision]; DFA dfa = decisionToDFA[decision];
if ( dfa==null || dfa.s0==null ) { if ( dfa==null || dfa.s0==null ) {
DecisionState startState = atn.decisionToState.get(decision); DecisionState startState = atn.decisionToState.get(decision);
decisionToDFA[decision] = dfa = new DFA(startState); decisionToDFA[decision] = dfa = new DFA(startState, decision);
dfa.decision = decision;
return predictATN(dfa, input, outerContext); return predictATN(dfa, input, outerContext);
} }
else { else {
@ -288,8 +287,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
} }
public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<Token> input, public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<? extends Symbol> input,
@Nullable ParserRuleContext outerContext) @Nullable ParserRuleContext<?> outerContext)
{ {
if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
if ( debug ) System.out.println("ATN decision "+dfa.decision+ if ( debug ) System.out.println("ATN decision "+dfa.decision+
@ -320,8 +319,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0, public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0,
@NotNull SymbolStream<Token> input, int startIndex, @NotNull SymbolStream<? extends Symbol> input, int startIndex,
@Nullable ParserRuleContext outerContext) @Nullable ParserRuleContext<?> outerContext)
{ {
if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
@ -371,37 +370,32 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// if no edge, pop over to ATN interpreter, update DFA and return // if no edge, pop over to ATN interpreter, update DFA and return
if ( s.edges == null || t >= s.edges.length || t < -1 || s.edges[t+1] == null ) { if ( s.edges == null || t >= s.edges.length || t < -1 || s.edges[t+1] == null ) {
if ( dfa_debug && t>=0 ) System.out.println("no edge for "+parser.getTokenNames()[t]); if ( dfa_debug && t>=0 ) System.out.println("no edge for "+parser.getTokenNames()[t]);
int alt = -1; int alt;
if ( dfa_debug ) { if ( dfa_debug ) {
System.out.println("ATN exec upon "+ System.out.println("ATN exec upon "+
parser.getInputString(startIndex) + parser.getInputString(startIndex) +
" at DFA state "+s.stateNumber); " at DFA state "+s.stateNumber);
} }
try {
alt = execATN(dfa, s, input, startIndex, outerContext); alt = execATN(dfa, s, input, startIndex, outerContext);
// this adds edge even if next state is accept for // this adds edge even if next state is accept for
// same alt; e.g., s0-A->:s1=>2-B->:s2=>2 // same alt; e.g., s0-A->:s1=>2-B->:s2=>2
// TODO: This next stuff kills edge, but extra states remain. :( // TODO: This next stuff kills edge, but extra states remain. :(
if ( s.isAcceptState && alt!=-1 ) { if ( s.isAcceptState && alt!=-1 ) {
DFAState d = s.edges[input.LA(1)+1]; DFAState d = s.edges[input.LA(1)+1];
if ( d.isAcceptState && d.prediction==s.prediction ) { if ( d.isAcceptState && d.prediction==s.prediction ) {
// we can carve it out. // we can carve it out.
s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error
}
} }
if ( dfa_debug ) {
System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames()));
//dump(dfa);
}
// action already executed
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" predicts "+alt);
return alt; // we've updated DFA, exec'd action, and have our deepest answer
} }
catch (NoViableAltException nvae) { if ( dfa_debug ) {
addDFAEdge(s, t, ERROR); System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames()));
throw nvae; //dump(dfa);
} }
// action already executed
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" predicts "+alt);
return alt; // we've updated DFA, exec'd action, and have our deepest answer
} }
DFAState target = s.edges[t+1]; DFAState target = s.edges[t+1];
if ( target == ERROR ) { if ( target == ERROR ) {
@ -479,15 +473,15 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
*/ */
public int execATN(@NotNull DFA dfa, @NotNull DFAState s0, public int execATN(@NotNull DFA dfa, @NotNull DFAState s0,
@NotNull SymbolStream<Token> input, int startIndex, @NotNull SymbolStream<? extends Symbol> input, int startIndex,
ParserRuleContext outerContext) ParserRuleContext<?> outerContext)
{ {
if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input)); if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input));
ATN_failover++; ATN_failover++;
ATNConfigSet previous = s0.configset; ATNConfigSet previous = s0.configset;
DFAState D = null; DFAState D;
ATNConfigSet fullCtxSet = null; ATNConfigSet fullCtxSet;
if ( debug ) System.out.println("s0 = "+s0); if ( debug ) System.out.println("s0 = "+s0);
@ -600,8 +594,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
public ATNConfigSet execATNWithFullContext(DFA dfa, public ATNConfigSet execATNWithFullContext(DFA dfa,
DFAState D, // how far we got before failing over DFAState D, // how far we got before failing over
@NotNull ATNConfigSet s0, @NotNull ATNConfigSet s0,
@NotNull SymbolStream<Token> input, int startIndex, @NotNull SymbolStream<? extends Symbol> input, int startIndex,
ParserRuleContext outerContext, ParserRuleContext<?> outerContext,
int nalts, int nalts,
boolean greedy) boolean greedy)
{ {
@ -637,7 +631,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
if ( reach.hasSemanticContext ) { if ( reach.hasSemanticContext ) {
SemanticContext[] altToPred = getPredsForAmbigAlts(reach.conflictingAlts, reach, nalts); SemanticContext[] altToPred = getPredsForAmbigAlts(reach.conflictingAlts, reach, nalts);
// altToPred[uniqueAlt] is now our validating predicate (if any) // altToPred[uniqueAlt] is now our validating predicate (if any)
List<DFAState.PredPrediction> predPredictions = null; List<DFAState.PredPrediction> predPredictions;
if ( altToPred!=null ) { if ( altToPred!=null ) {
// we have a validating predicate; test it // we have a validating predicate; test it
predPredictions = getPredicatePredictions(reach.conflictingAlts, altToPred); predPredictions = getPredicatePredictions(reach.conflictingAlts, altToPred);
@ -799,7 +793,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
pairs.add(new DFAState.PredPrediction(pred, i)); pairs.add(new DFAState.PredPrediction(pred, i));
} }
} }
if ( pairs.size()==0 ) pairs = null; if ( pairs.isEmpty() ) pairs = null;
else if ( firstUnpredicated!=ATN.INVALID_ALT_NUMBER ) { else if ( firstUnpredicated!=ATN.INVALID_ALT_NUMBER ) {
// add default prediction if we found null predicate // add default prediction if we found null predicate
pairs.add(new DFAState.PredPrediction(null, firstUnpredicated)); pairs.add(new DFAState.PredPrediction(null, firstUnpredicated));
@ -813,7 +807,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
* prediction for disambiguating predicates. * prediction for disambiguating predicates.
*/ */
public int evalSemanticContext(List<DFAState.PredPrediction> predPredictions, public int evalSemanticContext(List<DFAState.PredPrediction> predPredictions,
ParserRuleContext outerContext) ParserRuleContext<?> outerContext)
{ {
int predictedAlt = ATN.INVALID_ALT_NUMBER; int predictedAlt = ATN.INVALID_ALT_NUMBER;
// List<DFAState.PredPrediction> predPredictions = D.predicates; // List<DFAState.PredPrediction> predPredictions = D.predicates;
@ -851,6 +845,17 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull Set<ATNConfig> closureBusy, @NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates, boolean collectPredicates,
boolean greedy, boolean loopsSimulateTailRecursion) boolean greedy, boolean loopsSimulateTailRecursion)
{
final int initialDepth = 0;
closure(config, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, initialDepth);
}
protected void closure(@NotNull ATNConfig config,
@NotNull ATNConfigSet configs,
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean greedy, boolean loopsSimulateTailRecursion,
int depth)
{ {
if ( debug ) System.out.println("closure("+config.toString(parser,true)+")"); if ( debug ) System.out.println("closure("+config.toString(parser,true)+")");
@ -875,7 +880,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// gotten that context AFTER having falling off a rule. // gotten that context AFTER having falling off a rule.
// Make sure we track that we are now out of context. // Make sure we track that we are now out of context.
c.reachesIntoOuterContext = config.reachesIntoOuterContext; c.reachesIntoOuterContext = config.reachesIntoOuterContext;
closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion); closure(c, configs, closureBusy, collectPredicates, greedy, loopsSimulateTailRecursion, depth - 1);
return; return;
} }
else { else {
@ -919,8 +924,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
Transition t = p.transition(i); Transition t = p.transition(i);
boolean continueCollecting = boolean continueCollecting =
!(t instanceof ActionTransition) && collectPredicates; !(t instanceof ActionTransition) && collectPredicates;
ATNConfig c = getEpsilonTarget(config, t, continueCollecting); ATNConfig c = getEpsilonTarget(config, t, continueCollecting, depth == 0);
if ( c!=null ) { if ( c!=null ) {
int newDepth = depth;
if ( config.state instanceof RuleStopState ) { if ( config.state instanceof RuleStopState ) {
// target fell off end of rule; mark resulting c as having dipped into outer context // target fell off end of rule; mark resulting c as having dipped into outer context
// We can't get here if incoming config was rule stop and we had context // We can't get here if incoming config was rule stop and we had context
@ -929,9 +935,16 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// preds if this is > 0. // preds if this is > 0.
c.reachesIntoOuterContext++; c.reachesIntoOuterContext++;
configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method
newDepth--;
if ( debug ) System.out.println("dips into outer ctx: "+c); if ( debug ) System.out.println("dips into outer ctx: "+c);
} }
closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion); else if (t instanceof RuleTransition) {
if (newDepth >= 0) {
newDepth++;
}
}
closure(c, configs, closureBusy, continueCollecting, greedy, loopsSimulateTailRecursion, newDepth);
} }
} }
} }
@ -943,12 +956,12 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
@Nullable @Nullable
public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates) { public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates, boolean inContext) {
if ( t instanceof RuleTransition ) { if ( t instanceof RuleTransition ) {
return ruleTransition(config, t); return ruleTransition(config, t);
} }
else if ( t instanceof PredicateTransition ) { else if ( t instanceof PredicateTransition ) {
return predTransition(config, (PredicateTransition)t, collectPredicates); return predTransition(config, (PredicateTransition)t, collectPredicates, inContext);
} }
else if ( t instanceof ActionTransition ) { else if ( t instanceof ActionTransition ) {
return actionTransition(config, (ActionTransition)t); return actionTransition(config, (ActionTransition)t);
@ -968,7 +981,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@Nullable @Nullable
public ATNConfig predTransition(@NotNull ATNConfig config, public ATNConfig predTransition(@NotNull ATNConfig config,
@NotNull PredicateTransition pt, @NotNull PredicateTransition pt,
boolean collectPredicates) boolean collectPredicates,
boolean inContext)
{ {
if ( debug ) { if ( debug ) {
System.out.println("PRED (collectPredicates="+collectPredicates+") "+ System.out.println("PRED (collectPredicates="+collectPredicates+") "+
@ -979,12 +993,6 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
parser.getRuleInvocationStack()); parser.getRuleInvocationStack());
} }
} }
// We know the correct context in exactly one spot: in the original
// rule that invokes the ATN simulation. We know we are in this rule
// when the context stack is empty and we've not dipped into
// the outer context.
boolean inContext =
config.context==ParserRuleContext.EMPTY && config.reachesIntoOuterContext==0;
ATNConfig c; ATNConfig c;
if ( collectPredicates && if ( collectPredicates &&
@ -1207,7 +1215,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull @NotNull
public String getTokenName(int t) { public String getTokenName(int t) {
if ( t==-1 ) return "EOF"; if ( t==Token.EOF ) return "EOF";
if ( parser!=null && parser.getTokenNames()!=null ) { if ( parser!=null && parser.getTokenNames()!=null ) {
String[] tokensNames = parser.getTokenNames(); String[] tokensNames = parser.getTokenNames();
if ( t>=tokensNames.length ) { if ( t>=tokensNames.length ) {
@ -1221,7 +1229,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return String.valueOf(t); return String.valueOf(t);
} }
public String getLookaheadName(SymbolStream<Token> input) { public String getLookaheadName(SymbolStream<? extends Symbol> input) {
return getTokenName(input.LA(1)); return getTokenName(input.LA(1));
} }
@ -1246,18 +1254,18 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
@NotNull @NotNull
public NoViableAltException noViableAlt(@NotNull SymbolStream<Token> input, public NoViableAltException noViableAlt(@NotNull SymbolStream<? extends Symbol> input,
@NotNull ParserRuleContext outerContext, @NotNull ParserRuleContext<?> outerContext,
@NotNull ATNConfigSet configs, @NotNull ATNConfigSet configs,
int startIndex) int startIndex)
{ {
return new NoViableAltException(parser, input, return new NoViableAltException(parser, input,
(Token)input.get(startIndex), input.get(startIndex),
(Token)input.LT(1), input.LT(1),
configs, outerContext); configs, outerContext);
} }
public static int getUniqueAlt(@NotNull Collection<ATNConfig> configs) { public int getUniqueAlt(@NotNull Collection<ATNConfig> configs) {
int alt = ATN.INVALID_ALT_NUMBER; int alt = ATN.INVALID_ALT_NUMBER;
for (ATNConfig c : configs) { for (ATNConfig c : configs) {
if ( alt == ATN.INVALID_ALT_NUMBER ) { if ( alt == ATN.INVALID_ALT_NUMBER ) {
@ -1282,6 +1290,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return false; return false;
} }
@NotNull
protected DFAState addDFAEdge(@NotNull DFA dfa, protected DFAState addDFAEdge(@NotNull DFA dfa,
@NotNull ATNConfigSet p, @NotNull ATNConfigSet p,
int t, int t,
@ -1304,7 +1313,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
/** See comment on LexerInterpreter.addDFAState. */ /** See comment on LexerInterpreter.addDFAState. */
@Nullable @NotNull
protected DFAState addDFAState(@NotNull DFA dfa, @NotNull ATNConfigSet configs) { protected DFAState addDFAState(@NotNull DFA dfa, @NotNull ATNConfigSet configs) {
DFAState proposed = new DFAState(configs); DFAState proposed = new DFAState(configs);
DFAState existing = dfa.states.get(proposed); DFAState existing = dfa.states.get(proposed);

12
runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java
View File

@ -43,7 +43,8 @@ public class DFA {
public final Map<DFAState, DFAState> states = new LinkedHashMap<DFAState, DFAState>(); public final Map<DFAState, DFAState> states = new LinkedHashMap<DFAState, DFAState>();
@Nullable @Nullable
public DFAState s0; public DFAState s0;
public int decision;
public final int decision;
/** From which ATN state did we create this DFA? */ /** From which ATN state did we create this DFA? */
@NotNull @NotNull
@ -54,7 +55,14 @@ public class DFA {
*/ */
// public OrderedHashSet<ATNConfig> conflictSet; // public OrderedHashSet<ATNConfig> conflictSet;
public DFA(@NotNull DecisionState atnStartState) { this.atnStartState = atnStartState; } public DFA(@NotNull DecisionState atnStartState) {
this(atnStartState, 0);
}
public DFA(@NotNull DecisionState atnStartState, int decision) {
this.atnStartState = atnStartState;
this.decision = decision;
}
/** Find the path in DFA from s0 to s, returning list of states encountered (inclusively) */ /** Find the path in DFA from s0 to s, returning list of states encountered (inclusively) */
// public List<DFAState> getPathToState(DFAState finalState, TokenStream input, int start, int stop) { // public List<DFAState> getPathToState(DFAState finalState, TokenStream input, int start, int stop) {

2
tool/test/org/antlr/v4/test/Java.g
View File

@ -181,7 +181,9 @@ compilationUnit
( packageDeclaration importDeclaration* typeDeclaration* ( packageDeclaration importDeclaration* typeDeclaration*
| classOrInterfaceDeclaration typeDeclaration* | classOrInterfaceDeclaration typeDeclaration*
) )
EOF
| packageDeclaration? importDeclaration* typeDeclaration* | packageDeclaration? importDeclaration* typeDeclaration*
EOF
; ;
packageDeclaration packageDeclaration

8
tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View File

@ -33,6 +33,7 @@ import org.antlr.v4.Tool;
import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.runtime.NoViableAltException; import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
@ -504,8 +505,7 @@ public class TestATNParserPrediction extends BaseTest {
TokenStream input = new IntTokenStream(types); TokenStream input = new IntTokenStream(types);
ParserInterpreter interp = new ParserInterpreter(g, input); ParserInterpreter interp = new ParserInterpreter(g, input);
DecisionState startState = atn.decisionToState.get(decision); DecisionState startState = atn.decisionToState.get(decision);
DFA dfa = new DFA(startState); DFA dfa = new DFA(startState, decision);
dfa.decision = decision;
int alt = interp.predictATN(dfa, input, ParserRuleContext.EMPTY, false); int alt = interp.predictATN(dfa, input, ParserRuleContext.EMPTY, false);
System.out.println(dot.getDOT(dfa, false)); System.out.println(dot.getDOT(dfa, false));
@ -523,7 +523,7 @@ public class TestATNParserPrediction extends BaseTest {
} }
public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName, public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
String inputString, ParserRuleContext ctx) String inputString, ParserRuleContext<?> ctx)
{ {
Tool.internalOption_ShowATNConfigsInDFA = true; Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg); ATN lexatn = createATN(lg);
@ -541,7 +541,7 @@ public class TestATNParserPrediction extends BaseTest {
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b")))); // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b"))));
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));
ParserATNSimulator interp = new ParserATNSimulator(atn); ParserATNSimulator<Token> interp = new ParserATNSimulator<Token>(atn);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp); List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types); System.out.println(types);
TokenStream input = new IntTokenStream(types); TokenStream input = new IntTokenStream(types);

17
tool/test/org/antlr/v4/test/TestCommonTokenStream.java
View File

@ -178,24 +178,17 @@ public class TestCommonTokenStream extends BaseTest {
new TokenSource() { new TokenSource() {
int i = 0; int i = 0;
WritableToken[] tokens = { WritableToken[] tokens = {
new CommonToken(1," "), new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
new CommonToken(1,"x"), new CommonToken(1,"x"),
new CommonToken(1," "), new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
new CommonToken(1,"="), new CommonToken(1,"="),
new CommonToken(1,"34"), new CommonToken(1,"34"),
new CommonToken(1," "), new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
new CommonToken(1," "), new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
new CommonToken(1,";"), new CommonToken(1,";"),
new CommonToken(1,"\n"), new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},
new CommonToken(Token.EOF,"") new CommonToken(Token.EOF,"")
}; };
{
tokens[0].setChannel(Lexer.HIDDEN);
tokens[2].setChannel(Lexer.HIDDEN);
tokens[5].setChannel(Lexer.HIDDEN);
tokens[6].setChannel(Lexer.HIDDEN);
tokens[8].setChannel(Lexer.HIDDEN);
}
public Token nextToken() { public Token nextToken() {
return tokens[i++]; return tokens[i++];
} }

8
tool/test/org/antlr/v4/test/TestPerformance.java
View File

@ -32,8 +32,12 @@ import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.*; import org.antlr.v4.runtime.dfa.*;
import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.*; import org.antlr.v4.runtime.tree.ParseTree;
import org.junit.*; import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import java.io.*; import java.io.*;
import java.lang.reflect.*; import java.lang.reflect.*;