From f6024aae0a3ab9b679cebc5936d9308dc4704fb1 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 14 Nov 2013 13:12:08 -0600 Subject: [PATCH] Updated interpreter APIs * Add Grammar.createLexerInterpreter and Grammar.createParserInterpreter * Rewrite LexerInterpreter to extend Lexer (greatly simplified) --- tool/src/org/antlr/v4/tool/Grammar.java | 25 +++++ .../v4/tool/interp/LexerInterpreter.java | 101 ++++++------------ .../v4/test/TestBufferedTokenStream.java | 15 +-- .../antlr/v4/test/TestParserInterpreter.java | 5 +- .../v4/test/TestTokenStreamRewriter.java | 81 +++++++------- .../v4/test/TestUnbufferedCharStream.java | 3 +- .../v4/test/TestUnbufferedTokenStream.java | 12 +-- 7 files changed, 113 insertions(+), 129 deletions(-) diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index 8e64cdb34..badb749c6 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -39,8 +39,10 @@ import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.parse.GrammarASTAdaptor; import org.antlr.v4.parse.GrammarTreeVisitor; import org.antlr.v4.parse.TokenVocabParser; +import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.IntSet; @@ -55,9 +57,12 @@ import org.antlr.v4.tool.ast.GrammarASTWithOptions; import org.antlr.v4.tool.ast.GrammarRootAST; import org.antlr.v4.tool.ast.PredAST; import org.antlr.v4.tool.ast.TerminalAST; +import org.antlr.v4.tool.interp.LexerInterpreter; +import org.antlr.v4.tool.interp.ParserInterpreter; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -863,4 +868,24 @@ public class Grammar implements AttributeResolver { public void setLookaheadDFA(int decision, DFA lookaheadDFA) { decisionDFAs.put(decision, lookaheadDFA); } + + public LexerInterpreter createLexerInterpreter(CharStream input) { + if (this.isParser()) { + throw new IllegalStateException("A lexer interpreter can only be created for a lexer or combined grammar."); + } + + if (this.isCombined()) { + return implicitLexer.createLexerInterpreter(input); + } + + return new LexerInterpreter(fileName, Arrays.asList(getTokenNames()), Arrays.asList(getRuleNames()), ((LexerGrammar)this).modes.keySet(), atn, input); + } + + public ParserInterpreter createParserInterpreter(TokenStream tokenStream) { + if (this.isLexer()) { + throw new IllegalStateException("A parser interpreter can only be created for a parser or combined grammar."); + } + + return new ParserInterpreter(fileName, Arrays.asList(getTokenNames()), Arrays.asList(getRuleNames()), atn, tokenStream); + } } diff --git a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java index aca6a42ce..b7111881c 100644 --- a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java +++ b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java @@ -30,101 +30,70 @@ package org.antlr.v4.tool.interp; -import org.antlr.v4.Tool; -import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonTokenFactory; import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenFactory; -import org.antlr.v4.runtime.TokenSource; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNType; import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.Pair; -import org.antlr.v4.tool.LexerGrammar; -public class LexerInterpreter implements TokenSource { - protected LexerGrammar g; - protected LexerATNSimulator interp; - protected CharStream input; - protected Pair tokenFactorySourcePair; +import java.util.Collection; - /** How to create token objects */ - protected TokenFactory _factory = CommonTokenFactory.DEFAULT; +public class LexerInterpreter extends Lexer { + protected final String grammarFileName; + protected final ATN atn; + + protected final String[] tokenNames; + protected final String[] ruleNames; + protected final String[] modeNames; protected final DFA[] _decisionToDFA; protected final PredictionContextCache _sharedContextCache = new PredictionContextCache(); - public LexerInterpreter(LexerGrammar g, String inputString) { - this(g); - setInput(inputString); - } + public LexerInterpreter(String grammarFileName, Collection tokenNames, Collection ruleNames, Collection modeNames, ATN atn, CharStream input) { + super(input); - public LexerInterpreter(LexerGrammar g) { - Tool antlr = new Tool(); - antlr.process(g,false); - _decisionToDFA = new DFA[g.atn.getNumberOfDecisions()]; + if (atn.grammarType != ATNType.LEXER) { + throw new IllegalArgumentException("The ATN must be a lexer ATN."); + } + + this.grammarFileName = grammarFileName; + this.atn = atn; + this.tokenNames = tokenNames.toArray(new String[tokenNames.size()]); + this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]); + this.modeNames = modeNames.toArray(new String[modeNames.size()]); + + this._decisionToDFA = new DFA[atn.getNumberOfDecisions()]; for (int i = 0; i < _decisionToDFA.length; i++) { - _decisionToDFA[i] = new DFA(g.atn.getDecisionState(i), i); + _decisionToDFA[i] = new DFA(atn.getDecisionState(i), i); } - interp = new LexerATNSimulator(g.atn,_decisionToDFA,_sharedContextCache); - } - - public void setInput(String inputString) { - setInput(new ANTLRInputStream(inputString)); - } - - public void setInput(CharStream input) { - this.input = input; - this.tokenFactorySourcePair = new Pair(this, input); + this._interp = new LexerATNSimulator(atn,_decisionToDFA,_sharedContextCache); } @Override - public String getSourceName() { return g.name; } - - @Override - public void setTokenFactory(TokenFactory factory) { - this._factory = factory; + public ATN getATN() { + return atn; } @Override - public TokenFactory getTokenFactory() { - return _factory; + public String getGrammarFileName() { + return grammarFileName; } @Override - public int getCharPositionInLine() { - return 0; + public String[] getTokenNames() { + return tokenNames; } @Override - public int getLine() { - return 0; + public String[] getRuleNames() { + return ruleNames; } @Override - public CharStream getInputStream() { - return input; - } - - @Override - public Token nextToken() { - // TODO: Deal with off channel tokens - int start = input.index(); - int tokenStartCharPositionInLine = interp.getCharPositionInLine(); - int tokenStartLine = interp.getLine(); - int mark = input.mark(); // make sure unuffered stream holds chars long enough to get text - try { - int ttype = interp.match(input, Lexer.DEFAULT_MODE); - int stop = input.index()-1; - - return _factory.create(tokenFactorySourcePair, ttype, null, Token.DEFAULT_CHANNEL, start, stop, - tokenStartLine, tokenStartCharPositionInLine); - } - finally { - input.release(mark); - } + public String[] getModeNames() { + return modeNames; } } diff --git a/tool/test/org/antlr/v4/test/TestBufferedTokenStream.java b/tool/test/org/antlr/v4/test/TestBufferedTokenStream.java index 61fa64133..344f5af9c 100644 --- a/tool/test/org/antlr/v4/test/TestBufferedTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestBufferedTokenStream.java @@ -61,8 +61,7 @@ public class TestBufferedTokenStream extends BaseTest { // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;"); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TokenStream tokens = createTokenStream(lexEngine); String result = tokens.LT(1).getText(); @@ -83,8 +82,7 @@ public class TestBufferedTokenStream extends BaseTest { // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;"); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TokenStream tokens = createTokenStream(lexEngine); String result = tokens.LT(2).getText(); @@ -105,8 +103,7 @@ public class TestBufferedTokenStream extends BaseTest { // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;"); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TokenStream tokens = createTokenStream(lexEngine); int i = 1; @@ -136,8 +133,7 @@ public class TestBufferedTokenStream extends BaseTest { // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;"); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TokenStream tokens = createTokenStream(lexEngine); Token t = tokens.LT(1); @@ -164,8 +160,7 @@ public class TestBufferedTokenStream extends BaseTest { // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;"); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TokenStream tokens = createTokenStream(lexEngine); tokens.consume(); // get x into buffer diff --git a/tool/test/org/antlr/v4/test/TestParserInterpreter.java b/tool/test/org/antlr/v4/test/TestParserInterpreter.java index 5fcf6df5d..1a956a39d 100644 --- a/tool/test/org/antlr/v4/test/TestParserInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestParserInterpreter.java @@ -30,9 +30,9 @@ package org.antlr.v4.test; +import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNSimulator; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; @@ -41,7 +41,6 @@ import org.antlr.v4.tool.interp.ParserInterpreter; import org.junit.Test; import static org.junit.Assert.assertEquals; -import org.junit.Ignore; import java.util.Arrays; @@ -200,7 +199,7 @@ public class TestParserInterpreter extends BaseTest { String startRule, String input, String parseTree) { - LexerInterpreter lexEngine = new LexerInterpreter(lg, input); + LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream tokens = new CommonTokenStream(lexEngine); ATN atn = createATN(g, true); diff --git a/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java index c43a0c039..57955e11a 100644 --- a/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java +++ b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java @@ -29,6 +29,7 @@ */ package org.antlr.v4.test; +import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.TokenStreamRewriter; import org.antlr.v4.runtime.misc.Interval; @@ -50,7 +51,7 @@ public class TestTokenStreamRewriter extends BaseTest { "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); - LexerInterpreter lexEngine = new LexerInterpreter(g, "abc"); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream("abc")); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -67,7 +68,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -84,7 +85,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -102,7 +103,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -119,7 +120,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -136,7 +137,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -158,7 +159,7 @@ public class TestTokenStreamRewriter extends BaseTest { // Tokens: 0123456789 // Input: x = 3 * 0; String input = "x = 3 * 0;"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -196,7 +197,7 @@ public class TestTokenStreamRewriter extends BaseTest { // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; String input = "x = 3 * 0 + 2 * 0;"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -248,7 +249,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -266,7 +267,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -285,7 +286,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -303,7 +304,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -328,7 +329,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -348,7 +349,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -366,7 +367,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -385,7 +386,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -403,7 +404,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -421,7 +422,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -439,7 +440,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcccba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -457,7 +458,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcccba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -483,7 +484,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcccba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -501,7 +502,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcccba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -518,7 +519,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcccba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -535,7 +536,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcccba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -562,7 +563,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcccba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -589,7 +590,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcba"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -607,7 +608,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -625,7 +626,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -644,7 +645,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -664,7 +665,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -686,7 +687,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -705,7 +706,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -725,7 +726,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -752,7 +753,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -772,7 +773,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -792,7 +793,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -812,7 +813,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -832,7 +833,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -850,7 +851,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abcc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); @@ -868,7 +869,7 @@ public class TestTokenStreamRewriter extends BaseTest { "B : 'b';\n" + "C : 'c';\n"); String input = "abc"; - LexerInterpreter lexEngine = new LexerInterpreter(g, input); + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream stream = new CommonTokenStream(lexEngine); stream.fill(); TokenStreamRewriter tokens = new TokenStreamRewriter(stream); diff --git a/tool/test/org/antlr/v4/test/TestUnbufferedCharStream.java b/tool/test/org/antlr/v4/test/TestUnbufferedCharStream.java index 2211b42dd..4822e293a 100644 --- a/tool/test/org/antlr/v4/test/TestUnbufferedCharStream.java +++ b/tool/test/org/antlr/v4/test/TestUnbufferedCharStream.java @@ -310,10 +310,9 @@ public class TestUnbufferedCharStream extends BaseTest { // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; TestingUnbufferedCharStream input = createStream("x = 302 * 91 + 20234234 * 0;"); - LexerInterpreter lexEngine = new LexerInterpreter(g); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); // copy text into tokens from char stream lexEngine.setTokenFactory(new CommonTokenFactory(true)); - lexEngine.setInput(input); CommonTokenStream tokens = new CommonTokenStream(lexEngine); String result = tokens.LT(1).getText(); String expecting = "x"; diff --git a/tool/test/org/antlr/v4/test/TestUnbufferedTokenStream.java b/tool/test/org/antlr/v4/test/TestUnbufferedTokenStream.java index aa4aabc29..b919ea94a 100644 --- a/tool/test/org/antlr/v4/test/TestUnbufferedTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestUnbufferedTokenStream.java @@ -63,8 +63,7 @@ public class TestUnbufferedTokenStream extends BaseTest { CharStream input = new ANTLRInputStream( new StringReader("x = 302;") ); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TokenStream tokens = new UnbufferedTokenStream(lexEngine); assertEquals("x", tokens.LT(1).getText()); @@ -90,8 +89,7 @@ public class TestUnbufferedTokenStream extends BaseTest { CharStream input = new ANTLRInputStream( new StringReader("x = 302;") ); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TestingUnbufferedTokenStream tokens = new TestingUnbufferedTokenStream(lexEngine); assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString()); @@ -128,8 +126,7 @@ public class TestUnbufferedTokenStream extends BaseTest { CharStream input = new ANTLRInputStream( new StringReader("x = 302;") ); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TestingUnbufferedTokenStream tokens = new TestingUnbufferedTokenStream(lexEngine); int m = tokens.mark(); @@ -164,8 +161,7 @@ public class TestUnbufferedTokenStream extends BaseTest { CharStream input = new ANTLRInputStream( new StringReader("x = 302 + 1;") ); - LexerInterpreter lexEngine = new LexerInterpreter(g); - lexEngine.setInput(input); + LexerInterpreter lexEngine = g.createLexerInterpreter(input); TestingUnbufferedTokenStream tokens = new TestingUnbufferedTokenStream(lexEngine); int m = tokens.mark();