forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Updated interpreter APIs 

* Add Grammar.createLexerInterpreter and Grammar.createParserInterpreter
* Rewrite LexerInterpreter to extend Lexer (greatly simplified)
This commit is contained in:
Sam Harwell 2013-11-14 13:12:08 -06:00
parent 508817a706
commit f6024aae0a
7 changed files with 113 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tool/src/org/antlr/v4/tool/Grammar.java
View File

@ -39,8 +39,10 @@ import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.parse.TokenVocabParser;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntSet;
@ -55,9 +57,12 @@ import org.antlr.v4.tool.ast.GrammarASTWithOptions;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.antlr.v4.tool.ast.PredAST;
import org.antlr.v4.tool.ast.TerminalAST;
import org.antlr.v4.tool.interp.LexerInterpreter;
import org.antlr.v4.tool.interp.ParserInterpreter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@ -863,4 +868,24 @@ public class Grammar implements AttributeResolver {
public void setLookaheadDFA(int decision, DFA lookaheadDFA) {
decisionDFAs.put(decision, lookaheadDFA);
}
public LexerInterpreter createLexerInterpreter(CharStream input) {
if (this.isParser()) {
throw new IllegalStateException("A lexer interpreter can only be created for a lexer or combined grammar.");
}
if (this.isCombined()) {
return implicitLexer.createLexerInterpreter(input);
}
return new LexerInterpreter(fileName, Arrays.asList(getTokenNames()), Arrays.asList(getRuleNames()), ((LexerGrammar)this).modes.keySet(), atn, input);
}
public ParserInterpreter createParserInterpreter(TokenStream tokenStream) {
if (this.isLexer()) {
throw new IllegalStateException("A parser interpreter can only be created for a parser or combined grammar.");
}
return new ParserInterpreter(fileName, Arrays.asList(getTokenNames()), Arrays.asList(getRuleNames()), atn, tokenStream);
}
}

99
tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java
View File

@ -30,101 +30,70 @@
package org.antlr.v4.tool.interp;
import org.antlr.v4.Tool;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonTokenFactory;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenFactory;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNType;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.PredictionContextCache;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.Pair;
import org.antlr.v4.tool.LexerGrammar;
public class LexerInterpreter implements TokenSource {
protected LexerGrammar g;
protected LexerATNSimulator interp;
protected CharStream input;
protected Pair<TokenSource, CharStream> tokenFactorySourcePair;
import java.util.Collection;
/** How to create token objects */
protected TokenFactory<?> _factory = CommonTokenFactory.DEFAULT;
public class LexerInterpreter extends Lexer {
protected final String grammarFileName;
protected final ATN atn;
protected final String[] tokenNames;
protected final String[] ruleNames;
protected final String[] modeNames;
protected final DFA[] _decisionToDFA;
protected final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
public LexerInterpreter(LexerGrammar g, String inputString) {
this(g);
setInput(inputString);
public LexerInterpreter(String grammarFileName, Collection<String> tokenNames, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) {
super(input);
if (atn.grammarType != ATNType.LEXER) {
throw new IllegalArgumentException("The ATN must be a lexer ATN.");
}
public LexerInterpreter(LexerGrammar g) {
Tool antlr = new Tool();
antlr.process(g,false);
_decisionToDFA = new DFA[g.atn.getNumberOfDecisions()];
this.grammarFileName = grammarFileName;
this.atn = atn;
this.tokenNames = tokenNames.toArray(new String[tokenNames.size()]);
this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]);
this.modeNames = modeNames.toArray(new String[modeNames.size()]);
this._decisionToDFA = new DFA[atn.getNumberOfDecisions()];
for (int i = 0; i < _decisionToDFA.length; i++) {
_decisionToDFA[i] = new DFA(g.atn.getDecisionState(i), i);
_decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
}
interp = new LexerATNSimulator(g.atn,_decisionToDFA,_sharedContextCache);
}
public void setInput(String inputString) {
setInput(new ANTLRInputStream(inputString));
}
public void setInput(CharStream input) {
this.input = input;
this.tokenFactorySourcePair = new Pair<TokenSource, CharStream>(this, input);
this._interp = new LexerATNSimulator(atn,_decisionToDFA,_sharedContextCache);
}
@Override
public String getSourceName() { return g.name; }
@Override
public void setTokenFactory(TokenFactory<?> factory) {
this._factory = factory;
public ATN getATN() {
return atn;
}
@Override
public TokenFactory<?> getTokenFactory() {
return _factory;
public String getGrammarFileName() {
return grammarFileName;
}
@Override
public int getCharPositionInLine() {
return 0;
public String[] getTokenNames() {
return tokenNames;
}
@Override
public int getLine() {
return 0;
public String[] getRuleNames() {
return ruleNames;
}
@Override
public CharStream getInputStream() {
return input;
}
@Override
public Token nextToken() {
// TODO: Deal with off channel tokens
int start = input.index();
int tokenStartCharPositionInLine = interp.getCharPositionInLine();
int tokenStartLine = interp.getLine();
int mark = input.mark(); // make sure unuffered stream holds chars long enough to get text
try {
int ttype = interp.match(input, Lexer.DEFAULT_MODE);
int stop = input.index()-1;
return _factory.create(tokenFactorySourcePair, ttype, null, Token.DEFAULT_CHANNEL, start, stop,
tokenStartLine, tokenStartCharPositionInLine);
}
finally {
input.release(mark);
}
public String[] getModeNames() {
return modeNames;
}
}

15
tool/test/org/antlr/v4/test/TestBufferedTokenStream.java
View File

@ -61,8 +61,7 @@ public class TestBufferedTokenStream extends BaseTest {
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
String result = tokens.LT(1).getText();
@ -83,8 +82,7 @@ public class TestBufferedTokenStream extends BaseTest {
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
String result = tokens.LT(2).getText();
@ -105,8 +103,7 @@ public class TestBufferedTokenStream extends BaseTest {
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
int i = 1;
@ -136,8 +133,7 @@ public class TestBufferedTokenStream extends BaseTest {
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
Token t = tokens.LT(1);
@ -164,8 +160,7 @@ public class TestBufferedTokenStream extends BaseTest {
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
tokens.consume(); // get x into buffer

5
tool/test/org/antlr/v4/test/TestParserInterpreter.java
View File

@ -30,9 +30,9 @@
package org.antlr.v4.test;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNSimulator;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
@ -41,7 +41,6 @@ import org.antlr.v4.tool.interp.ParserInterpreter;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import org.junit.Ignore;
import java.util.Arrays;
@ -200,7 +199,7 @@ public class TestParserInterpreter extends BaseTest {
String startRule, String input,
String parseTree)
{
LexerInterpreter lexEngine = new LexerInterpreter(lg, input);
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
ATN atn = createATN(g, true);

81
tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
View File

@ -29,6 +29,7 @@
*/
package org.antlr.v4.test;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.TokenStreamRewriter;
import org.antlr.v4.runtime.misc.Interval;
@ -50,7 +51,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
LexerInterpreter lexEngine = new LexerInterpreter(g, "abc");
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream("abc"));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -67,7 +68,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -84,7 +85,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -102,7 +103,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -119,7 +120,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -136,7 +137,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -158,7 +159,7 @@ public class TestTokenStreamRewriter extends BaseTest {
// Tokens: 0123456789
// Input: x = 3 * 0;
String input = "x = 3 * 0;";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -196,7 +197,7 @@ public class TestTokenStreamRewriter extends BaseTest {
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
String input = "x = 3 * 0 + 2 * 0;";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -248,7 +249,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -266,7 +267,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -285,7 +286,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -303,7 +304,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -328,7 +329,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -348,7 +349,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -366,7 +367,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -385,7 +386,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -403,7 +404,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -421,7 +422,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -439,7 +440,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcccba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -457,7 +458,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcccba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -483,7 +484,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcccba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -501,7 +502,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcccba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -518,7 +519,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcccba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -535,7 +536,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcccba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -562,7 +563,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcccba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -589,7 +590,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcba";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -607,7 +608,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -625,7 +626,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -644,7 +645,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -664,7 +665,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -686,7 +687,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -705,7 +706,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -725,7 +726,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -752,7 +753,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -772,7 +773,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -792,7 +793,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -812,7 +813,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -832,7 +833,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -850,7 +851,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abcc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
@ -868,7 +869,7 @@ public class TestTokenStreamRewriter extends BaseTest {
"B : 'b';\n" +
"C : 'c';\n");
String input = "abc";
LexerInterpreter lexEngine = new LexerInterpreter(g, input);
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);

3
tool/test/org/antlr/v4/test/TestUnbufferedCharStream.java
View File

@ -310,10 +310,9 @@ public class TestUnbufferedCharStream extends BaseTest {
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
TestingUnbufferedCharStream input = createStream("x = 302 * 91 + 20234234 * 0;");
LexerInterpreter lexEngine = new LexerInterpreter(g);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
// copy text into tokens from char stream
lexEngine.setTokenFactory(new CommonTokenFactory(true));
lexEngine.setInput(input);
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
String result = tokens.LT(1).getText();
String expecting = "x";

12
tool/test/org/antlr/v4/test/TestUnbufferedTokenStream.java
View File

@ -63,8 +63,7 @@ public class TestUnbufferedTokenStream extends BaseTest {
CharStream input = new ANTLRInputStream(
new StringReader("x = 302;")
);
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = new UnbufferedTokenStream<Token>(lexEngine);
assertEquals("x", tokens.LT(1).getText());
@ -90,8 +89,7 @@ public class TestUnbufferedTokenStream extends BaseTest {
CharStream input = new ANTLRInputStream(
new StringReader("x = 302;")
);
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
@ -128,8 +126,7 @@ public class TestUnbufferedTokenStream extends BaseTest {
CharStream input = new ANTLRInputStream(
new StringReader("x = 302;")
);
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
int m = tokens.mark();
@ -164,8 +161,7 @@ public class TestUnbufferedTokenStream extends BaseTest {
CharStream input = new ANTLRInputStream(
new StringReader("x = 302 + 1;")
);
LexerInterpreter lexEngine = new LexerInterpreter(g);
lexEngine.setInput(input);
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
int m = tokens.mark();