diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java new file mode 100644 index 000000000..8779da0ea --- /dev/null +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -0,0 +1,1095 @@ +/* + [The "BSD license"] + Copyright (c) 2005-2009 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.antlr.v4.test; + + +import org.antlr.v4.Tool; +import org.antlr.v4.misc.Utils; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.semantics.SemanticPipeline; +import org.antlr.v4.tool.*; +import org.junit.*; +import org.stringtemplate.v4.ST; + +import java.io.*; +import java.util.*; + + +public abstract class BaseTest { + public static final String newline = System.getProperty("line.separator"); + public static final String pathSep = System.getProperty("path.separator"); + + /** + * Build up the full classpath we need, including the surefire path (if present) + */ + public static final String CLASSPATH = System.getProperty("java.class.path"); + + public String tmpdir = null; + + /** reset during setUp and set to true if we find a problem */ + protected boolean lastTestFailed = false; + + /** If error during parser execution, store stderr here; can't return + * stdout and stderr. This doesn't trap errors from running antlr. + */ + protected String stderrDuringParse; + + @Before + public void setUp() throws Exception { + lastTestFailed = false; // hope for the best, but set to true in asserts that fail + // new output dir for each test + tmpdir = new File(System.getProperty("java.io.tmpdir"), + getClass().getSimpleName()+"-"+System.currentTimeMillis()).getAbsolutePath(); +// tmpdir = "/tmp"; + } + + @After + public void tearDown() throws Exception { + // remove tmpdir if no error. + if ( !lastTestFailed ) eraseTempDir(); + + } + + protected org.antlr.v4.Tool newTool(String[] args) { + Tool tool = new Tool(args); + tool.setOutputDirectory(tmpdir); + return tool; + } + + protected Tool newTool() { + org.antlr.v4.Tool tool = new Tool(); + tool.setOutputDirectory(tmpdir); + return tool; + } + + ATN createATN(Grammar g) { + if ( g.atn!=null ) return g.atn; + semanticProcess(g); + + ParserATNFactory f = new ParserATNFactory(g); + if ( g.isLexer() ) f = new LexerATNFactory((LexerGrammar)g); + g.atn = f.createATN(); + + return g.atn; + } + + protected void semanticProcess(Grammar g) { + if ( g.ast!=null && !g.ast.hasErrors ) { + System.out.println(g.ast.toStringTree()); + Tool antlr = new Tool(); + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) + for (Grammar imp : g.getImportedGrammars()) { + antlr.processNonCombinedGrammar(imp); + } + } + } + } + + public DFA createDFA(Grammar g, DecisionState s) { +// PredictionDFAFactory conv = new PredictionDFAFactory(g, s); +// DFA dfa = conv.createDFA(); +// conv.issueAmbiguityWarnings(); +// System.out.print("DFA="+dfa); +// return dfa; + return null; + } + +// public void minimizeDFA(DFA dfa) { +// DFAMinimizer dmin = new DFAMinimizer(dfa); +// dfa.minimized = dmin.minimize(); +// } + + List getTypesFromString(Grammar g, String expecting) { + List expectingTokenTypes = new ArrayList(); + if ( expecting!=null && !expecting.trim().equals("") ) { + for (String tname : expecting.replace(" ", "").split(",")) { + int ttype = g.getTokenType(tname); + expectingTokenTypes.add(ttype); + } + } + return expectingTokenTypes; + } + + public List getTokenTypes(String input, LexerInterpreter lexerATN) { + ANTLRStringStream in = new ANTLRStringStream(input); + List tokenTypes = new ArrayList(); + int ttype = 0; + do { + ttype = lexerATN.matchATN(in); + tokenTypes.add(ttype); + } while ( ttype!= Token.EOF ); + return tokenTypes; + } + + public List getTokenTypes(LexerGrammar lg, + ATN atn, + CharStream input, + boolean adaptive) + { + LexerInterpreter interp = new LexerInterpreter(atn); + List tokenTypes = new ArrayList(); + int ttype; + do { + if ( adaptive ) ttype = interp.match(input, Lexer.DEFAULT_MODE); + else ttype = interp.matchATN(input); + if ( ttype == Token.EOF ) tokenTypes.add("EOF"); + else { + tokenTypes.add(lg.typeToTokenList.get(ttype)); + } + } while ( ttype!=Token.EOF ); + return tokenTypes; + } + + List checkRuleDFA(String gtext, String ruleName, String expecting) + throws Exception + { + ErrorQueue equeue = new ErrorQueue(); + Grammar g = new Grammar(gtext, equeue); + ATN atn = createATN(g); + ATNState s = atn.ruleToStartState.get(g.getRule(ruleName)); + if ( s==null ) { + System.err.println("no such rule: "+ruleName); + return null; + } + ATNState t = s.transition(0).target; + if ( !(t instanceof DecisionState) ) { + System.out.println(ruleName+" has no decision"); + return null; + } + DecisionState blk = (DecisionState)t; + checkRuleDFA(g, blk, expecting); + return equeue.all; + } + + List checkRuleDFA(String gtext, int decision, String expecting) + throws Exception + { + ErrorQueue equeue = new ErrorQueue(); + Grammar g = new Grammar(gtext, equeue); + ATN atn = createATN(g); + DecisionState blk = atn.decisionToATNState.get(decision); + checkRuleDFA(g, blk, expecting); + return equeue.all; + } + + void checkRuleDFA(Grammar g, DecisionState blk, String expecting) + throws Exception + { + DFA dfa = createDFA(g, blk); + String result = null; + if ( dfa!=null ) result = dfa.toString(); + assertEquals(expecting, result); + } + + List checkLexerDFA(String gtext, String expecting) + throws Exception + { + return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting); + } + + List checkLexerDFA(String gtext, String modeName, String expecting) + throws Exception + { + ErrorQueue equeue = new ErrorQueue(); + LexerGrammar g = new LexerGrammar(gtext, equeue); + g.atn = createATN(g); +// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g); +// DFA dfa = conv.createDFA(modeName); +// g.setLookaheadDFA(0, dfa); // only one decision to worry about +// +// String result = null; +// if ( dfa!=null ) result = dfa.toString(); +// assertEquals(expecting, result); +// +// return equeue.all; + return null; + } + + protected boolean compile(String fileName) { + String compiler = "javac"; + String classpathOption = "-classpath"; + + String[] args = new String[] { + compiler, "-d", tmpdir, + classpathOption, tmpdir+pathSep+CLASSPATH, + tmpdir+"/"+fileName + }; + String cmdLine = compiler+" -d "+tmpdir+" "+classpathOption+" "+tmpdir+pathSep+CLASSPATH+" "+fileName; + //System.out.println("compile: "+cmdLine); + File outputDir = new File(tmpdir); + try { + Process process = + Runtime.getRuntime().exec(args, null, outputDir); + StreamVacuum stdout = new StreamVacuum(process.getInputStream()); + StreamVacuum stderr = new StreamVacuum(process.getErrorStream()); + stdout.start(); + stderr.start(); + process.waitFor(); + stdout.join(); + stderr.join(); + if ( stdout.toString().length()>0 ) { + System.err.println("compile stdout from: "+cmdLine); + System.err.println(stdout); + } + if ( stderr.toString().length()>0 ) { + System.err.println("compile stderr from: "+cmdLine); + System.err.println(stderr); + } + int ret = process.exitValue(); + return ret==0; + } + catch (Exception e) { + System.err.println("can't exec compilation"); + e.printStackTrace(System.err); + return false; + } + } + + /** Return true if all is ok, no errors */ + protected boolean antlr(String fileName, String grammarFileName, String grammarStr, boolean debug) { + boolean allIsWell = true; + mkdir(tmpdir); + writeFile(tmpdir, fileName, grammarStr); + try { + final List options = new ArrayList(); + if ( debug ) { + options.add("-debug"); + } + options.add("-o"); + options.add(tmpdir); + options.add("-lib"); + options.add(tmpdir); + options.add(new File(tmpdir,grammarFileName).toString()); + final String[] optionsA = new String[options.size()]; + options.toArray(optionsA); + ErrorQueue equeue = new ErrorQueue(); + Tool antlr = newTool(optionsA); + antlr.addListener(equeue); + antlr.processGrammarsOnCommandLine(); + if ( equeue.errors.size()>0 ) { + allIsWell = false; + System.err.println("antlr reports errors from "+options); + for (int i = 0; i < equeue.errors.size(); i++) { + ANTLRMessage msg = (ANTLRMessage) equeue.errors.get(i); + System.err.println(msg); + } + System.out.println("!!!\ngrammar:"); + System.out.println(grammarStr); + System.out.println("###"); + } + } + catch (Exception e) { + allIsWell = false; + System.err.println("problems building grammar: "+e); + e.printStackTrace(System.err); + } + return allIsWell; + } + + protected String execLexer(String grammarFileName, + String grammarStr, + String lexerName, + String input) + { + return execLexer(grammarFileName, grammarStr, lexerName, input, false); + } + + protected String execLexer(String grammarFileName, + String grammarStr, + String lexerName, + String input, + boolean showDFA) + { + rawGenerateAndBuildRecognizer(grammarFileName, + grammarStr, + null, + lexerName, + false); + writeFile(tmpdir, "input", input); + writeLexerTestFile(lexerName, showDFA); + compile("Test.java"); + String output = execClass("Test"); + if ( stderrDuringParse!=null && stderrDuringParse.length()>0 ) { + System.err.println(stderrDuringParse); + } + return output; + } + + protected String execParser(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String startRuleName, + String input, boolean debug) + { + rawGenerateAndBuildRecognizer(grammarFileName, + grammarStr, + parserName, + lexerName, + debug); + writeFile(tmpdir, "input", input); + boolean parserBuildsTrees = + grammarStr.indexOf("output=AST")>=0 || + grammarStr.indexOf("output = AST")>=0; + boolean parserBuildsTemplate = + grammarStr.indexOf("output=template")>=0 || + grammarStr.indexOf("output = template")>=0; + return rawExecRecognizer(parserName, + null, + lexerName, + startRuleName, + null, + parserBuildsTrees, + parserBuildsTemplate, + false, + debug); + } + + protected String execTreeParser(String parserGrammarFileName, + String parserGrammarStr, + String parserName, + String treeParserGrammarFileName, + String treeParserGrammarStr, + String treeParserName, + String lexerName, + String parserStartRuleName, + String treeParserStartRuleName, + String input) + { + return execTreeParser(parserGrammarFileName, + parserGrammarStr, + parserName, + treeParserGrammarFileName, + treeParserGrammarStr, + treeParserName, + lexerName, + parserStartRuleName, + treeParserStartRuleName, + input, + false); + } + + protected String execTreeParser(String parserGrammarFileName, + String parserGrammarStr, + String parserName, + String treeParserGrammarFileName, + String treeParserGrammarStr, + String treeParserName, + String lexerName, + String parserStartRuleName, + String treeParserStartRuleName, + String input, + boolean debug) + { + // build the parser + rawGenerateAndBuildRecognizer(parserGrammarFileName, + parserGrammarStr, + parserName, + lexerName, + debug); + + // build the tree parser + rawGenerateAndBuildRecognizer(treeParserGrammarFileName, + treeParserGrammarStr, + treeParserName, + lexerName, + debug); + + writeFile(tmpdir, "input", input); + + boolean parserBuildsTrees = + parserGrammarStr.indexOf("output=AST")>=0 || + parserGrammarStr.indexOf("output = AST")>=0; + boolean treeParserBuildsTrees = + treeParserGrammarStr.indexOf("output=AST")>=0 || + treeParserGrammarStr.indexOf("output = AST")>=0; + boolean parserBuildsTemplate = + parserGrammarStr.indexOf("output=template")>=0 || + parserGrammarStr.indexOf("output = template")>=0; + + return rawExecRecognizer(parserName, + treeParserName, + lexerName, + parserStartRuleName, + treeParserStartRuleName, + parserBuildsTrees, + parserBuildsTemplate, + treeParserBuildsTrees, + debug); + } + + /** Return true if all is well */ + protected boolean rawGenerateAndBuildRecognizer(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + boolean debug) + { + boolean allIsWell = + antlr(grammarFileName, grammarFileName, grammarStr, debug); + boolean ok; + if ( lexerName!=null ) { + ok = compile(lexerName+".java"); + if ( !ok ) { allIsWell = false; } + } + if ( parserName!=null ) { + ok = compile(parserName+".java"); + if ( !ok ) { allIsWell = false; } + } + return allIsWell; + } + + protected String rawExecRecognizer(String parserName, + String treeParserName, + String lexerName, + String parserStartRuleName, + String treeParserStartRuleName, + boolean parserBuildsTrees, + boolean parserBuildsTemplate, + boolean treeParserBuildsTrees, + boolean debug) + { + this.stderrDuringParse = null; + if ( treeParserBuildsTrees && parserBuildsTrees ) { + writeTreeAndTreeTestFile(parserName, + treeParserName, + lexerName, + parserStartRuleName, + treeParserStartRuleName, + debug); + } + else if ( parserBuildsTrees ) { + writeTreeTestFile(parserName, + treeParserName, + lexerName, + parserStartRuleName, + treeParserStartRuleName, + debug); + } + else if ( parserBuildsTemplate ) { + writeTemplateTestFile(parserName, + lexerName, + parserStartRuleName, + debug); + } + else if ( parserName==null ) { + writeLexerTestFile(lexerName, false); + } + else { + writeTestFile(parserName, + lexerName, + parserStartRuleName, + debug); + } + + compile("Test.java"); + return execClass("Test"); + } + + public String execClass(String className) { + try { + String[] args = new String[] { + "java", "-classpath", tmpdir+pathSep+CLASSPATH, + className, new File(tmpdir, "input").getAbsolutePath() + }; + //String cmdLine = "java -classpath "+CLASSPATH+pathSep+tmpdir+" Test " + new File(tmpdir, "input").getAbsolutePath(); + //System.out.println("execParser: "+cmdLine); + Process process = + Runtime.getRuntime().exec(args, null, new File(tmpdir)); + StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); + StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); + stdoutVacuum.start(); + stderrVacuum.start(); + process.waitFor(); + stdoutVacuum.join(); + stderrVacuum.join(); + String output = null; + output = stdoutVacuum.toString(); + if ( stderrVacuum.toString().length()>0 ) { + this.stderrDuringParse = stderrVacuum.toString(); + System.err.println("exec stderrVacuum: "+ stderrVacuum); + } + return output; + } + catch (Exception e) { + System.err.println("can't exec recognizer"); + e.printStackTrace(System.err); + } + return null; + } + + public void testErrors(String[] pairs, boolean printTree) { + for (int i = 0; i < pairs.length; i+=2) { + String input = pairs[i]; + String expect = pairs[i+1]; + ErrorQueue equeue = new ErrorQueue(); + Grammar g=null; + try { + String[] lines = input.split("\n"); + String fileName = getFilenameFromFirstLineOfGrammar(lines[0]); + g = new Grammar(fileName, input, equeue); + + if ( printTree ) { + if ( g.ast!=null ) System.out.println(g.ast.toStringTree()); + else System.out.println("null tree"); + } + + if ( g.ast!=null && !g.ast.hasErrors ) { + Tool antlr = new Tool(); + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) + for (Grammar imp : g.getImportedGrammars()) { + antlr.processNonCombinedGrammar(imp); + } + } + } + + //g.loadImportedGrammars(); + } + catch (org.antlr.runtime.RecognitionException re) { + re.printStackTrace(System.err); + } + String actual = equeue.toString(g.tool); + String msg = input; + msg = msg.replaceAll("\n","\\\\n"); + msg = msg.replaceAll("\r","\\\\r"); + msg = msg.replaceAll("\t","\\\\t"); + + // ignore error number + expect = stripErrorNum(expect); + actual = stripErrorNum(actual); + assertEquals("error in: "+msg,expect,actual); + } + } + + // can be multi-line + //error(29): A.g:2:11: unknown attribute reference a in $a + //error(29): A.g:2:11: unknown attribute reference a in $a + String stripErrorNum(String errs) { + String[] lines = errs.split("\n"); + for (int i=0; i=0 && semi>=0 ) { + int space = line.indexOf(' ', grIndex); + fileName = line.substring(space+1, semi)+".g"; + } + if ( fileName.length()==".g".length() ) fileName = ""; + return fileName; + } + +// void ambig(List msgs, int[] expectedAmbigAlts, String expectedAmbigInput) +// throws Exception +// { +// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput); +// } + +// void ambig(List msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput) +// throws Exception +// { +// List amsgs = getMessagesOfType(msgs, AmbiguityMessage.class); +// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i); +// if ( a==null ) assertNull(expectedAmbigAlts); +// else { +// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts)); +// } +// assertEquals(expectedAmbigInput, a.input); +// } + +// void unreachable(List msgs, int[] expectedUnreachableAlts) +// throws Exception +// { +// unreachable(msgs, 0, expectedUnreachableAlts); +// } + +// void unreachable(List msgs, int i, int[] expectedUnreachableAlts) +// throws Exception +// { +// List amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class); +// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i); +// if ( u==null ) assertNull(expectedUnreachableAlts); +// else { +// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts)); +// } +// } + + List getMessagesOfType(List msgs, Class c) { + List filtered = new ArrayList(); + for (ANTLRMessage m : msgs) { + if ( m.getClass() == c ) filtered.add(m); + } + return filtered; + } + + public static class StreamVacuum implements Runnable { + StringBuffer buf = new StringBuffer(); + BufferedReader in; + Thread sucker; + public StreamVacuum(InputStream in) { + this.in = new BufferedReader( new InputStreamReader(in) ); + } + public void start() { + sucker = new Thread(this); + sucker.start(); + } + public void run() { + try { + String line = in.readLine(); + while (line!=null) { + buf.append(line); + buf.append('\n'); + line = in.readLine(); + } + } + catch (IOException ioe) { + System.err.println("can't read output from process"); + } + } + /** wait for the thread to finish */ + public void join() throws InterruptedException { + sucker.join(); + } + public String toString() { + return buf.toString(); + } + } + + public static class FilteringTokenStream extends CommonTokenStream { + public FilteringTokenStream(TokenSource src) { super(src); } + Set hide = new HashSet(); + protected void sync(int i) { + super.sync(i); + if ( hide.contains(get(i).getType()) ) get(i).setChannel(Token.HIDDEN_CHANNEL); + } + public void setTokenTypeChannel(int ttype, int channel) { + hide.add(ttype); + } + } + + protected void writeFile(String dir, String fileName, String content) { + try { + File f = new File(dir, fileName); + FileWriter w = new FileWriter(f); + BufferedWriter bw = new BufferedWriter(w); + bw.write(content); + bw.close(); + w.close(); + } + catch (IOException ioe) { + System.err.println("can't write file"); + ioe.printStackTrace(System.err); + } + } + + protected void mkdir(String dir) { + File f = new File(dir); + f.mkdirs(); + } + + protected void writeTestFile(String parserName, + String lexerName, + String parserStartRuleName, + boolean debug) + { + ST outputFileST = new ST( + "import org.antlr.v4.runtime.*;\n" + + "import org.antlr.v4.runtime.tree.*;\n" + + //"import org.antlr.v4.runtime.debug.*;\n" + + "\n" + + "public class Test {\n" + + " public static void main(String[] args) throws Exception {\n" + + " CharStream input = new ANTLRFileStream(args[0]);\n" + + " lex = new (input);\n" + + " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + + " \n"+ + " parser.(null);\n" + + " }\n" + + "}" + ); + ST createParserST = + new ST( + "class Profiler2 extends Profiler {\n" + + " public void terminate() { ; }\n" + + "}\n"+ + " Profiler2 profiler = new Profiler2();\n"+ + " parser = new (tokens,profiler);\n" + + " profiler.setParser(parser);\n"); + if ( !debug ) { + createParserST = + new ST( + " parser = new (tokens);\n"); + } + outputFileST.add("createParser", createParserST); + outputFileST.add("parserName", parserName); + outputFileST.add("lexerName", lexerName); + outputFileST.add("parserStartRuleName", parserStartRuleName); + writeFile(tmpdir, "Test.java", outputFileST.render()); + } + + protected void writeLexerTestFile(String lexerName, boolean showDFA) { + ST outputFileST = new ST( + "import org.antlr.v4.runtime.*;\n" + + "\n" + + "public class Test {\n" + + " public static void main(String[] args) throws Exception {\n" + + " CharStream input = new ANTLRFileStream(args[0]);\n" + + " lex = new (input);\n" + + " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + + " tokens.fill();\n" + + " for (Object t : tokens.getTokens()) System.out.println(t);\n" + + (showDFA?"System.out.print(lex.getInterpreter().getDFA(Lexer.DEFAULT_MODE).toLexerString());\n":"")+ + " }\n" + + "}" + ); + + outputFileST.add("lexerName", lexerName); + writeFile(tmpdir, "Test.java", outputFileST.render()); + } + + protected void writeTreeTestFile(String parserName, + String treeParserName, + String lexerName, + String parserStartRuleName, + String treeParserStartRuleName, + boolean debug) + { + ST outputFileST = new ST( + "import org.antlr.v4.runtime.*;\n" + + "import org.antlr.v4.runtime.tree.*;\n" + +// "import org.antlr.v4.runtime.debug.*;\n" + + "\n" + + "public class Test {\n" + + " public static void main(String[] args) throws Exception {\n" + + " CharStream input = new ANTLRFileStream(args[0]);\n" + + " lex = new (input);\n" + + " TokenRewriteStream tokens = new TokenRewriteStream(lex);\n" + + " \n"+ + " ParserRuleContext r = parser.(null);\n" + + " \n" + + " if ( r.tree!=null ) {\n" + + " System.out.println(((Tree)r.tree).toStringTree());\n" + + " ((CommonTree)r.tree).sanityCheckParentAndChildIndexes();\n" + + " }\n" + + " \n" + + " CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree)r.tree);\n" + + " nodes.setTokenStream(tokens);\n" + + " walker = new (nodes);\n" + + " walker.();\n" + + " \n" + + " }\n" + + "}" + ); + ST createParserST = + new ST( + "class Profiler2 extends Profiler {\n" + + " public void terminate() { ; }\n" + + "}\n"+ + " Profiler2 profiler = new Profiler2();\n"+ + " parser = new (tokens,profiler);\n" + + " profiler.setParser(parser);\n"); + if ( !debug ) { + createParserST = + new ST( + " parser = new (tokens);\n"); + } + outputFileST.add("createParser", createParserST); + outputFileST.add("parserName", parserName); + outputFileST.add("treeParserName", treeParserName); + outputFileST.add("lexerName", lexerName); + outputFileST.add("parserStartRuleName", parserStartRuleName); + outputFileST.add("treeParserStartRuleName", treeParserStartRuleName); + writeFile(tmpdir, "Test.java", outputFileST.render()); + } + + /** Parser creates trees and so does the tree parser */ + protected void writeTreeAndTreeTestFile(String parserName, + String treeParserName, + String lexerName, + String parserStartRuleName, + String treeParserStartRuleName, + boolean debug) + { + ST outputFileST = new ST( + "import org.antlr.v4.runtime.*;\n" + + "import org.antlr.v4.runtime.tree.*;\n" + +// "import org.antlr.v4.runtime.debug.*;\n" + + "\n" + + "public class Test {\n" + + " public static void main(String[] args) throws Exception {\n" + + " CharStream input = new ANTLRFileStream(args[0]);\n" + + " lex = new (input);\n" + + " TokenRewriteStream tokens = new TokenRewriteStream(lex);\n" + + " \n"+ + " ParserRuleContext r = parser.(null);\n" + + " ((CommonTree)r.tree).sanityCheckParentAndChildIndexes();\n" + + " CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree)r.tree);\n" + + " nodes.setTokenStream(tokens);\n" + + " walker = new (nodes);\n" + + " ParserRuleContext r2 = walker.();\n" + + " CommonTree rt = ((CommonTree)r2.tree);\n" + + " if ( rt!=null ) System.out.println(((CommonTree)r2.tree).toStringTree());\n" + + " }\n" + + "}" + ); + ST createParserST = + new ST( + "class Profiler2 extends Profiler {\n" + + " public void terminate() { ; }\n" + + "}\n"+ + " Profiler2 profiler = new Profiler2();\n"+ + " parser = new (tokens,profiler);\n" + + " profiler.setParser(parser);\n"); + if ( !debug ) { + createParserST = + new ST( + " parser = new (tokens);\n"); + } + outputFileST.add("createParser", createParserST); + outputFileST.add("parserName", parserName); + outputFileST.add("treeParserName", treeParserName); + outputFileST.add("lexerName", lexerName); + outputFileST.add("parserStartRuleName", parserStartRuleName); + outputFileST.add("treeParserStartRuleName", treeParserStartRuleName); + writeFile(tmpdir, "Test.java", outputFileST.render()); + } + + protected void writeTemplateTestFile(String parserName, + String lexerName, + String parserStartRuleName, + boolean debug) + { + ST outputFileST = new ST( + "import org.antlr.v4.runtime.*;\n" + + "import org.antlr.v4.stringtemplate.*;\n" + + "import org.antlr.v4.stringtemplate.language.*;\n" + +// "import org.antlr.v4.runtime.debug.*;\n" + + "import java.io.*;\n" + + "\n" + + "public class Test {\n" + + " static String templates =\n" + + " \"group test;\"+" + + " \"foo(x,y) ::= \\\" \\\"\";\n"+ + " static STGroup group ="+ + " new STGroup(new StringReader(templates)," + + " AngleBracketTemplateLexer.class);"+ + " public static void main(String[] args) throws Exception {\n" + + " CharStream input = new ANTLRFileStream(args[0]);\n" + + " lex = new (input);\n" + + " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + + " \n"+ + " parser.setTemplateLib(group);\n"+ + " ParserRuleContext r = parser.(null);\n" + + " if ( r.st!=null )\n" + + " System.out.print(r.st.toString());\n" + + " else\n" + + " System.out.print(\"\");\n" + + " }\n" + + "}" + ); + ST createParserST = + new ST( + "class Profiler2 extends Profiler {\n" + + " public void terminate() { ; }\n" + + "}\n"+ + " Profiler2 profiler = new Profiler2();\n"+ + " parser = new (tokens,profiler);\n" + + " profiler.setParser(parser);\n"); + if ( !debug ) { + createParserST = + new ST( + " parser = new (tokens);\n"); + } + outputFileST.add("createParser", createParserST); + outputFileST.add("parserName", parserName); + outputFileST.add("lexerName", lexerName); + outputFileST.add("parserStartRuleName", parserStartRuleName); + writeFile(tmpdir, "Test.java", outputFileST.render()); + } + + protected void eraseFiles(final String filesEndingWith) { + File tmpdirF = new File(tmpdir); + String[] files = tmpdirF.list(); + for(int i = 0; files!=null && i < files.length; i++) { + if ( files[i].endsWith(filesEndingWith) ) { + new File(tmpdir+"/"+files[i]).delete(); + } + } + } + + protected void eraseFiles() { + File tmpdirF = new File(tmpdir); + String[] files = tmpdirF.list(); + for(int i = 0; files!=null && i < files.length; i++) { + new File(tmpdir+"/"+files[i]).delete(); + } + } + + protected void eraseTempDir() { + File tmpdirF = new File(tmpdir); + if ( tmpdirF.exists() ) { + eraseFiles(); + tmpdirF.delete(); + } + } + + public String getFirstLineOfException() { + if ( this.stderrDuringParse ==null ) { + return null; + } + String[] lines = this.stderrDuringParse.split("\n"); + String prefix="Exception in thread \"main\" "; + return lines[0].substring(prefix.length(),lines[0].length()); + } + + /** + * When looking at a result set that consists of a Map/HashTable + * we cannot rely on the output order, as the hashing algorithm or other aspects + * of the implementation may be different on differnt JDKs or platforms. Hence + * we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a + * bit of a hack, but guarantees that we get the same order on all systems. We assume that + * the keys are strings. + * + * @param m The Map that contains keys we wish to return in sorted order + * @return A string that represents all the keys in sorted order. + */ + public String sortMapToString(Map m) { + + System.out.println("Map toString looks like: " + m.toString()); + // Pass in crap, and get nothing back + // + if (m == null) { + return null; + } + + // Sort the keys in the Map + // + TreeMap nset = new TreeMap(m); + + System.out.println("Tree map looks like: " + nset.toString()); + return nset.toString(); + } + + // override to track errors + + public void assertEquals(String msg, Object a, Object b) { try {Assert.assertEquals(msg,a,b);} catch (Error e) {lastTestFailed=true; throw e;} } + public void assertEquals(Object a, Object b) { try {Assert.assertEquals(a,b);} catch (Error e) {lastTestFailed=true; throw e;} } + public void assertEquals(String msg, long a, long b) { try {Assert.assertEquals(msg,a,b);} catch (Error e) {lastTestFailed=true; throw e;} } + public void assertEquals(long a, long b) { try {Assert.assertEquals(a,b);} catch (Error e) {lastTestFailed=true; throw e;} } + + public void assertTrue(String msg, boolean b) { try {Assert.assertTrue(msg,b);} catch (Error e) {lastTestFailed=true; throw e;} } + public void assertTrue(boolean b) { try {Assert.assertTrue(b);} catch (Error e) {lastTestFailed=true; throw e;} } + + public void assertFalse(String msg, boolean b) { try {Assert.assertFalse(msg,b);} catch (Error e) {lastTestFailed=true; throw e;} } + public void assertFalse(boolean b) { try {Assert.assertFalse(b);} catch (Error e) {lastTestFailed=true; throw e;} } + + public void assertNotNull(String msg, Object p) { try {Assert.assertNotNull(msg, p);} catch (Error e) {lastTestFailed=true; throw e;} } + public void assertNotNull(Object p) { try {Assert.assertNotNull(p);} catch (Error e) {lastTestFailed=true; throw e;} } + + public void assertNull(String msg, Object p) { try {Assert.assertNull(msg, p);} catch (Error e) {lastTestFailed=true; throw e;} } + public void assertNull(Object p) { try {Assert.assertNull(p);} catch (Error e) {lastTestFailed=true; throw e;} } + + public static class IntTokenStream implements TokenStream { + List types; + int p=0; + public IntTokenStream(List types) { this.types = types; } + + public void consume() { p++; } + + public int LA(int i) { return LT(i).getType(); } + + public int mark() { + return index(); + } + + public int index() { return p; } + + public void rewind(int marker) { + seek(marker); + } + + public void rewind() { + } + + public void release(int marker) { + seek(marker); + } + + public void seek(int index) { + p = index; + } + + public int size() { + return types.size(); + } + + public String getSourceName() { + return null; + } + + public Token LT(int i) { + if ( (p+i-1)>=types.size() ) return new CommonToken(-1); + return new CommonToken(types.get(p+i-1)); + } + + public int range() { + return 0; + } + + public Token get(int i) { + return new org.antlr.v4.runtime.CommonToken(types.get(i)); + } + + public TokenSource getTokenSource() { + return null; + } + + public String toString(int start, int stop) { + return null; + } + + public String toString(Token start, Token stop) { + return null; + } + } +} diff --git a/tool/test/org/antlr/v4/test/ErrorQueue.java b/tool/test/org/antlr/v4/test/ErrorQueue.java new file mode 100644 index 000000000..1c4113f38 --- /dev/null +++ b/tool/test/org/antlr/v4/test/ErrorQueue.java @@ -0,0 +1,82 @@ +/* + [The "BSD license"] + Copyright (c) 2005-2009 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.antlr.v4.test; + +import org.antlr.v4.Tool; +import org.antlr.v4.misc.Utils; +import org.antlr.v4.tool.ANTLRToolListener; +import org.antlr.v4.tool.ANTLRMessage; +import org.antlr.v4.tool.ToolMessage; +import org.stringtemplate.v4.ST; + +import java.util.ArrayList; +import java.util.List; + +public class ErrorQueue implements ANTLRToolListener { + public List infos = new ArrayList(); + public List errors = new ArrayList(); + public List warnings = new ArrayList(); + public List all = new ArrayList(); + + public void info(String msg) { + infos.add(msg); + } + + public void error(ANTLRMessage msg) { + errors.add(msg); + all.add(msg); + } + + public void warning(ANTLRMessage msg) { + warnings.add(msg); + all.add(msg); + } + + public void error(ToolMessage msg) { + errors.add(msg); + all.add(msg); + } + + public int size() { + return all.size() + infos.size(); + } + + public String toString() { return Utils.join(all.iterator(), "\n"); } + + public String toString(Tool tool) { + StringBuilder buf = new StringBuilder(); + for (ANTLRMessage m : all) { + ST st = tool.errMgr.getMessageTemplate(m); + buf.append(st.render()); + buf.append("\n"); + } + return buf.toString(); + } + +} + diff --git a/tool/test/org/antlr/v4/test/TestASTStructure.java b/tool/test/org/antlr/v4/test/TestASTStructure.java new file mode 100644 index 000000000..430feb6a0 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestASTStructure.java @@ -0,0 +1,428 @@ +package org.antlr.v4.test; + +import org.antlr.runtime.*; +import org.antlr.runtime.tree.*; +import org.antlr.v4.gunit.gUnitBase; +import org.junit.Test; +import org.junit.Before; +import static org.junit.Assert.*; + +public class TestASTStructure extends gUnitBase { + @Before public void setup() { + lexerClassName = "org.antlr.v4.parse.ANTLRLexer"; + parserClassName = "org.antlr.v4.parse.ANTLRParser"; + adaptorClassName = "org.antlr.v4.parse.GrammarASTAdaptor"; } + @Test public void test_grammarSpec1() throws Exception { + // gunit test on line 15 + RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "parser grammar P; a : A;", 15); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(PARSER_GRAMMAR P (RULES (RULE a (BLOCK (ALT A)))))"; + assertEquals("testing rule grammarSpec", expecting, actual); + } + + @Test public void test_grammarSpec2() throws Exception { + // gunit test on line 18 + RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n options {k=2; output=AST;}\n scope S {int x}\n tokens { A; B='33'; }\n @header {foo}\n a : A;\n ", 18); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(PARSER_GRAMMAR P (OPTIONS (= k 2) (= output AST)) (scope S {int x}) (tokens { A (= B '33')) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))"; + assertEquals("testing rule grammarSpec", expecting, actual); + } + + @Test public void test_grammarSpec3() throws Exception { + // gunit test on line 34 + RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n @header {foo}\n tokens { A; B='33'; }\n options {k=2; ASTLabel=a.b.c; output=AST;}\n scope S {int x}\n a : A;\n ", 34); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A (= B '33')) (OPTIONS (= k 2) (= ASTLabel a.b.c) (= output AST)) (scope S {int x}) (RULES (RULE a (BLOCK (ALT A)))))"; + assertEquals("testing rule grammarSpec", expecting, actual); + } + + @Test public void test_grammarSpec4() throws Exception { + // gunit test on line 50 + RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n import A=B, C;\n a : A;\n ", 50); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(PARSER_GRAMMAR P (import (= A B) C) (RULES (RULE a (BLOCK (ALT A)))))"; + assertEquals("testing rule grammarSpec", expecting, actual); + } @Test public void test_delegateGrammars1() throws Exception { + // gunit test on line 61 + RuleReturnScope rstruct = (RuleReturnScope)execParser("delegateGrammars", "import A;", 61); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(import A)"; + assertEquals("testing rule delegateGrammars", expecting, actual); + } @Test public void test_rule1() throws Exception { + // gunit test on line 64 + RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "a : A;", 64); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(RULE a (BLOCK (ALT (A (ELEMENT_OPTIONS X (= Y a.b.c))))))"; + assertEquals("testing rule rule", expecting, actual); + } + + @Test public void test_rule2() throws Exception { + // gunit test on line 66 + RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "A : B+;", 66); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(RULE A (BLOCK (ALT (+ (BLOCK (ALT B))))))"; + assertEquals("testing rule rule", expecting, actual); + } + + @Test public void test_rule3() throws Exception { + // gunit test on line 68 + RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n public a[int i] returns [int y]\n options {backtrack=true;}\n scope {int ss;}\n scope S,T;\n @init {blort}\n : ID ;\n ", 68); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(RULE a (RULEMODIFIERS public) int i (returns int y) (OPTIONS (= backtrack true)) (scope {int ss;}) (scope S T) (@ init {blort}) (BLOCK (ALT ID)))"; + assertEquals("testing rule rule", expecting, actual); + } + + @Test public void test_rule4() throws Exception { + // gunit test on line 87 + RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n a[int i] returns [int y]\n @init {blort}\n scope {int ss;}\n options {backtrack=true;}\n scope S,T;\n : ID;\n ", 87); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(RULE a int i (returns int y) (@ init {blort}) (scope {int ss;}) (OPTIONS (= backtrack true)) (scope S T) (BLOCK (ALT ID)))"; + assertEquals("testing rule rule", expecting, actual); + } + + @Test public void test_rule5() throws Exception { + // gunit test on line 104 + RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n a : ID ;\n catch[A b] {foo}\n finally {bar}\n ", 104); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(RULE a (BLOCK (ALT ID)) (catch A b {foo}) (finally {bar}))"; + assertEquals("testing rule rule", expecting, actual); + } + + @Test public void test_rule6() throws Exception { + // gunit test on line 113 + RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n a : ID ;\n catch[A a] {foo}\n catch[B b] {fu}\n finally {bar}\n ", 113); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(RULE a (BLOCK (ALT ID)) (catch A a {foo}) (catch B b {fu}) (finally {bar}))"; + assertEquals("testing rule rule", expecting, actual); + } @Test public void test_block1() throws Exception { + // gunit test on line 124 + RuleReturnScope rstruct = (RuleReturnScope)execParser("block", "( ^(A B) | ^(b C) )", 124); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(BLOCK (ALT (^( A B)) (ALT (^( b C)))"; + assertEquals("testing rule block", expecting, actual); + } @Test public void test_alternative1() throws Exception { + // gunit test on line 127 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "x+=ID* -> $x*", 127); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT (* (BLOCK (ALT (+= x ID))))) (-> (ALT (* (REWRITE_BLOCK (ALT x))))))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative2() throws Exception { + // gunit test on line 132 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ...", 132); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> ...))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative3() throws Exception { + // gunit test on line 133 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ", 133); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> EPSILON))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative4() throws Exception { + // gunit test on line 135 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> foo(a={x}, b={y})", 135); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> (TEMPLATE foo (ARGLIST (= a {x}) (= b {y})))))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative5() throws Exception { + // gunit test on line 140 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> template(a={x}, b={y}) <>", 140); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> (TEMPLATE (ARGLIST (= a {x}) (= b {y})) <>)))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative6() throws Exception { + // gunit test on line 145 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ({name})()", 145); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> (TEMPLATE {name})))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative7() throws Exception { + // gunit test on line 147 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> {expr}", 147); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> {expr}))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative8() throws Exception { + // gunit test on line 149 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "\n A -> {p1}? {e1}\n -> {e2}\n ->\n ", 149); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> {p1}? {e1}) (-> {e2}))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative9() throws Exception { + // gunit test on line 160 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> A", 160); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> (ALT A)))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative10() throws Exception { + // gunit test on line 162 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "a -> a", 162); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT a) (-> (ALT a)))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative11() throws Exception { + // gunit test on line 164 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "a A X? Y* -> A a ^(TOP X)? Y*", 164); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT a A (? (BLOCK (ALT X))) (* (BLOCK (ALT Y)))) (-> (ALT A a (? (REWRITE_BLOCK (ALT (^( TOP X)))) (* (REWRITE_BLOCK (ALT Y))))))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative12() throws Exception { + // gunit test on line 172 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> A[33]", 172); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> (ALT (A 33))))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative13() throws Exception { + // gunit test on line 174 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> 'int' ^(A A)*", 174); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> (ALT 'int' (* (REWRITE_BLOCK (ALT (^( A A)))))))"; + assertEquals("testing rule alternative", expecting, actual); + } + + @Test public void test_alternative14() throws Exception { + // gunit test on line 179 + RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "\n A -> {p1}? A\n -> {p2}? B\n ->\n ", 179); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(ALT_REWRITE (ALT A) (-> {p1}? (ALT A)) (-> {p2}? (ALT B)) (-> EPSILON))"; + assertEquals("testing rule alternative", expecting, actual); + } @Test public void test_element1() throws Exception { + // gunit test on line 191 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b+", 191); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+ (BLOCK (ALT b)))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element2() throws Exception { + // gunit test on line 192 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)+", 192); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+ (BLOCK (ALT b)))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element3() throws Exception { + // gunit test on line 193 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b?", 193); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(? (BLOCK (ALT b)))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element4() throws Exception { + // gunit test on line 194 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)?", 194); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(? (BLOCK (ALT b)))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element5() throws Exception { + // gunit test on line 195 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)*", 195); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT b)))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element6() throws Exception { + // gunit test on line 196 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b*", 196); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT b)))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element7() throws Exception { + // gunit test on line 197 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'while'*", 197); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT 'while')))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element8() throws Exception { + // gunit test on line 198 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'a'+", 198); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+ (BLOCK (ALT 'a')))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element9() throws Exception { + // gunit test on line 199 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "a[3]", 199); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(a 3)"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element10() throws Exception { + // gunit test on line 200 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'a'..'z'+", 200); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+ (BLOCK (ALT (.. 'a' 'z'))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element11() throws Exception { + // gunit test on line 201 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID", 201); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(= x ID)"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element12() throws Exception { + // gunit test on line 202 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID?", 202); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(? (BLOCK (ALT (= x ID))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element13() throws Exception { + // gunit test on line 203 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID*", 203); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT (= x ID))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element14() throws Exception { + // gunit test on line 204 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b", 204); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(= x b)"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element15() throws Exception { + // gunit test on line 205 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=(A|B)", 205); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(= x (BLOCK (ALT A) (ALT B)))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element16() throws Exception { + // gunit test on line 206 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=~(A|B)", 206); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(= x (~ (BLOCK (ALT A) (ALT B))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element17() throws Exception { + // gunit test on line 207 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)", 207); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+= x (~ (BLOCK (ALT A) (ALT B))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element18() throws Exception { + // gunit test on line 208 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)+", 208); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+ (BLOCK (ALT (+= x (~ (BLOCK (ALT A) (ALT B)))))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element19() throws Exception { + // gunit test on line 209 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b+", 209); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+ (BLOCK (ALT (= x b))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element20() throws Exception { + // gunit test on line 210 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=ID*", 210); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT (+= x ID))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element21() throws Exception { + // gunit test on line 211 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+='int'*", 211); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT (+= x 'int'))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element22() throws Exception { + // gunit test on line 212 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=b+", 212); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(+ (BLOCK (ALT (+= x b))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element23() throws Exception { + // gunit test on line 213 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "('*'^)*", 213); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT (^ '*'))))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element24() throws Exception { + // gunit test on line 214 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "({blort} 'x')*", 214); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(* (BLOCK (ALT {blort} 'x')))"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element25() throws Exception { + // gunit test on line 215 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "A!", 215); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(! A)"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element26() throws Exception { + // gunit test on line 216 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "A^", 216); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(^ A)"; + assertEquals("testing rule element", expecting, actual); + } + + @Test public void test_element27() throws Exception { + // gunit test on line 217 + RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=A^", 217); + Object actual = ((Tree)rstruct.getTree()).toStringTree(); + Object expecting = "(= x (^ A))"; + assertEquals("testing rule element", expecting, actual); + } +} diff --git a/tool/test/org/antlr/v4/test/TestATNConstruction.java b/tool/test/org/antlr/v4/test/TestATNConstruction.java new file mode 100644 index 000000000..8682621c4 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestATNConstruction.java @@ -0,0 +1,954 @@ +package org.antlr.v4.test; + +import org.antlr.v4.Tool; +import org.antlr.v4.automata.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.semantics.SemanticPipeline; +import org.antlr.v4.tool.*; +import org.junit.Test; + +public class TestATNConstruction extends BaseTest { + @Test + public void testA() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A;"); + String expecting = + "RuleStart_a_0->s2\n" + + "s2-A->s3\n" + + "s3->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s4\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAB() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A B ;"); + String expecting = + "RuleStart_a_0->s2\n" + + "s2-A->s3\n" + + "s3->s4\n" + + "s4-B->s5\n" + + "s5->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s6\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAorB() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A | B {;} ;"); + String expecting = + "RuleStart_a_0->BlockStart_8\n" + + "BlockStart_8->s2\n" + + "BlockStart_8->s4\n" + + "s2-A->s3\n" + + "s4-B->s5\n" + + "s3->BlockEnd_9\n" + + "s5->s6\n" + + "BlockEnd_9->RuleStop_a_1\n" + + "s6-{;}->s7\n" + + "RuleStop_a_1-EOF->s10\n" + + "s7->BlockEnd_9\n"; + checkRule(g, "a", expecting); + } + + @Test public void testRange() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : 'a'..'c' ;" + ); + String expecting = + "RuleStart_A_1->s3\n" + + "s3-'a'..'c'->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, "A", expecting); + } + + @Test public void testRangeOrRange() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : ('a'..'c' 'h' | 'q' 'j'..'l') ;" + ); + String expecting = + "RuleStart_A_1->BlockStart_11\n" + + "BlockStart_11->s3\n" + + "BlockStart_11->s7\n" + + "s3-'a'..'c'->s4\n" + + "s7-'q'->s8\n" + + "s4->s5\n" + + "s8->s9\n" + + "s5-'h'->s6\n" + + "s9-'j'..'l'->s10\n" + + "s6->BlockEnd_12\n" + + "s10->BlockEnd_12\n" + + "BlockEnd_12->RuleStop_A_2\n"; + checkTokensRule(g, "A", expecting); + } + + @Test public void testStringLiteralInParser() throws Exception { + Grammar g = new Grammar( + "grammar P;\n"+ + "a : A|'b' ;" + ); + String expecting = + "RuleStart_a_0->BlockStart_6\n" + + "BlockStart_6->s2\n" + + "BlockStart_6->s4\n" + + "s2-A->s3\n" + + "s4-'b'->s5\n" + + "s3->BlockEnd_7\n" + + "s5->BlockEnd_7\n" + + "BlockEnd_7->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s8\n"; + checkRule(g, "a", expecting); + } + + @Test public void testABorCD() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A B | C D;"); + String expecting = + "RuleStart_a_0->BlockStart_10\n" + + "BlockStart_10->s2\n" + + "BlockStart_10->s6\n" + + "s2-A->s3\n" + + "s6-C->s7\n" + + "s3->s4\n" + + "s7->s8\n" + + "s4-B->s5\n" + + "s8-D->s9\n" + + "s5->BlockEnd_11\n" + + "s9->BlockEnd_11\n" + + "BlockEnd_11->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s12\n"; + checkRule(g, "a", expecting); + } + + @Test public void testbA() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : b A ;\n"+ + "b : B ;"); + String expecting = + "RuleStart_a_0->s4\n" + + "s4->RuleStart_b_2\n" + + "s5->s6\n" + + "s6-A->s7\n" + + "s7->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s10\n"; + checkRule(g, "a", expecting); + expecting = + "RuleStart_b_2->s8\n" + + "s8-B->s9\n" + + "s9->RuleStop_b_3\n" + + "RuleStop_b_3->s5\n"; + checkRule(g, "b", expecting); + } + + @Test public void testFollow() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : b A ;\n"+ + "b : B ;\n"+ + "c : b C;"); + String expecting = + "RuleStart_b_2->s10\n" + + "s10-B->s11\n" + + "s11->RuleStop_b_3\n" + + "RuleStop_b_3->s7\n" + + "RuleStop_b_3->s13\n"; + checkRule(g, "b", expecting); + } + + @Test public void testAorEpsilon() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A | ;"); + String expecting = + "RuleStart_a_0->BlockStart_6\n" + + "BlockStart_6->s2\n" + + "BlockStart_6->s4\n" + + "s2-A->s3\n" + + "s4->s5\n" + + "s3->BlockEnd_7\n" + + "s5->BlockEnd_7\n" + + "BlockEnd_7->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s8\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAOptional() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A?;"); + String expecting = + "RuleStart_a_0->BlockStart_4\n" + + "BlockStart_4->s2\n" + + "BlockStart_4->BlockEnd_5\n" + + "s2-A->s3\n" + + "BlockEnd_5->RuleStop_a_1\n" + + "s3->BlockEnd_5\n" + + "RuleStop_a_1-EOF->s6\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAorBoptional() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : (A|B)?;"); + String expecting = + "RuleStart_a_0->BlockStart_6\n" + + "BlockStart_6->s2\n" + + "BlockStart_6->s4\n" + + "BlockStart_6->BlockEnd_7\n" + + "s2-A->s3\n" + + "s4-B->s5\n" + + "BlockEnd_7->RuleStop_a_1\n" + + "s3->BlockEnd_7\n" + + "s5->BlockEnd_7\n" + + "RuleStop_a_1-EOF->s8\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAorBthenC() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : (A | B) C;"); + String expecting = + "RuleStart_a_0->BlockStart_6\n" + + "BlockStart_6->s2\n" + + "BlockStart_6->s4\n" + + "s2-A->s3\n" + + "s4-B->s5\n" + + "s3->BlockEnd_7\n" + + "s5->BlockEnd_7\n" + + "BlockEnd_7->s8\n" + + "s8-C->s9\n" + + "s9->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s10\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAplus() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A+;"); + String expecting = + "RuleStart_a_0->PlusBlockStart_4\n" + + "PlusBlockStart_4->s2\n" + + "s2-A->s3\n" + + "s3->BlockEnd_5\n" + + "BlockEnd_5->PlusLoopBack_6\n" + + "PlusLoopBack_6->s2\n" + + "PlusLoopBack_6->s7\n" + + "s7->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s8\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAorBplus() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : (A|B)+;"); + String expecting = + "RuleStart_a_0->PlusBlockStart_6\n" + + "PlusBlockStart_6->s2\n" + + "PlusBlockStart_6->s4\n" + + "s2-A->s3\n" + + "s4-B->s5\n" + + "s3->BlockEnd_7\n" + + "s5->BlockEnd_7\n" + + "BlockEnd_7->PlusLoopBack_8\n" + + "PlusLoopBack_8->s2\n" + + "PlusLoopBack_8->s4\n" + + "PlusLoopBack_8->s9\n" + + "s9->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s10\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAorBorEmptyPlus() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : (A | B | )+ ;"); + String expecting = + "RuleStart_a_0->PlusBlockStart_8\n" + + "PlusBlockStart_8->s2\n" + + "PlusBlockStart_8->s4\n" + + "PlusBlockStart_8->s6\n" + + "s2-A->s3\n" + + "s4-B->s5\n" + + "s6->s7\n" + + "s3->BlockEnd_9\n" + + "s5->BlockEnd_9\n" + + "s7->BlockEnd_9\n" + + "BlockEnd_9->PlusLoopBack_10\n" + + "PlusLoopBack_10->s2\n" + + "PlusLoopBack_10->s4\n" + + "PlusLoopBack_10->s6\n" + + "PlusLoopBack_10->s11\n" + + "s11->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s12\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAStar() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : A*;"); + String expecting = + "RuleStart_a_0->StarBlockStart_4\n" + + "StarBlockStart_4->s2\n" + + "StarBlockStart_4->s7\n" + + "s2-A->s3\n" + + "s7->RuleStop_a_1\n" + + "s3->BlockEnd_5\n" + + "RuleStop_a_1-EOF->s8\n" + + "BlockEnd_5->StarLoopBack_6\n" + + "StarLoopBack_6->StarBlockStart_4\n"; + checkRule(g, "a", expecting); + } + + @Test public void testNestedAstar() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : (',' ID*)*;"); + String expecting = + "RuleStart_a_0->StarBlockStart_10\n" + + "StarBlockStart_10->s2\n" + + "StarBlockStart_10->s13\n" + + "s2-','->s3\n" + + "s13->RuleStop_a_1\n" + + "s3->StarBlockStart_6\n" + + "RuleStop_a_1-EOF->s14\n" + + "StarBlockStart_6->s4\n" + + "StarBlockStart_6->s9\n" + + "s4-ID->s5\n" + + "s9->BlockEnd_11\n" + + "s5->BlockEnd_7\n" + + "BlockEnd_11->StarLoopBack_12\n" + + "BlockEnd_7->StarLoopBack_8\n" + + "StarLoopBack_12->StarBlockStart_10\n" + + "StarLoopBack_8->StarBlockStart_6\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAorBstar() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : (A | B)* ;"); + String expecting = + "RuleStart_a_0->StarBlockStart_6\n" + + "StarBlockStart_6->s2\n" + + "StarBlockStart_6->s4\n" + + "StarBlockStart_6->s9\n" + + "s2-A->s3\n" + + "s4-B->s5\n" + + "s9->RuleStop_a_1\n" + + "s3->BlockEnd_7\n" + + "s5->BlockEnd_7\n" + + "RuleStop_a_1-EOF->s10\n" + + "BlockEnd_7->StarLoopBack_8\n" + + "StarLoopBack_8->StarBlockStart_6\n"; + checkRule(g, "a", expecting); + } + + @Test public void testPredicatedAorB() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : {p1}? A | {p2}? B ;"); + String expecting = + "RuleStart_a_0->BlockStart_10\n" + + "BlockStart_10->s2\n" + + "BlockStart_10->s6\n" + + "s2-{p1}?->s3\n" + + "s6-{p2}?->s7\n" + + "s3->s4\n" + + "s7->s8\n" + + "s4-A->s5\n" + + "s8-B->s9\n" + + "s5->BlockEnd_11\n" + + "s9->BlockEnd_11\n" + + "BlockEnd_11->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s12\n"; + checkRule(g, "a", expecting); + } + +/* + @Test public void testMultiplePredicates() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : {p1}? {p1a}? A | {p2}? B | {p3} b;\n" + + "b : {p4}? B ;"); + String expecting = + "\n"; + checkRule(g, "a", expecting); + } + + @Test public void testSets() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "a : ( A | B )+ ;\n" + + "b : ( A | B{;} )+ ;\n" + + "c : (A|B) (A|B) ;\n" + + "d : ( A | B )* ;\n" + + "e : ( A | B )? ;"); + String expecting = + "\n"; + checkRule(g, "a", expecting); + expecting = + "\n"; + checkRule(g, "b", expecting); + expecting = + "\n"; + checkRule(g, "c", expecting); + expecting = + "\n"; + checkRule(g, "d", expecting); + expecting = + "\n"; + checkRule(g, "e", expecting); + } + + @Test public void testNotSet() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "tokens { A; B; C; }\n"+ + "a : ~A ;\n"); + String expecting = + "\n"; + checkRule(g, "a", expecting); + + } + + @Test public void testNotSingletonBlockSet() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "tokens { A; B; C; }\n"+ + "a : ~(A) ;\n"); + String expecting = + "\n"; + checkRule(g, "a", expecting); + } + + @Test public void testNotCharSet() throws Exception { + Grammar g = new Grammar( + "lexer grammar P;\n"+ + "A : ~'3' ;\n"); + String expecting = + "RuleStart_A_1->s5\n" + + "s5-{'\\u0000'..'2', '4'..'\\uFFFE'}->s6\n" + + "s6->RuleStop_A_2\n"; + checkRule(g, "A", expecting); + } + + @Test public void testNotBlockSet() throws Exception { + Grammar g = new Grammar( + "lexer grammar P;\n"+ + "A : ~('3'|'b') ;\n"); + String expecting = + "\n"; + checkRule(g, "A", expecting); + } + + @Test public void testNotSetLoop() throws Exception { + Grammar g = new Grammar( + "lexer grammar P;\n"+ + "A : ~('3')* ;\n"); + String expecting = + "\n"; + checkRule(g, "A", expecting); + } + + @Test public void testNotBlockSetLoop() throws Exception { + Grammar g = new Grammar( + "lexer grammar P;\n"+ + "A : ~('3'|'b')* ;\n"); + String expecting = + "\n"; + checkRule(g, "A", expecting); + } + + @Test public void testLabeledNotSet() throws Exception { + Grammar g = new Grammar( + "parser grammar P;\n"+ + "tokens { A; B; C; }\n"+ + "a : t=~A ;\n"); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2-B..C->.s3\n" + + ".s3->:s4\n" + + ":s4-EOF->.s5\n"; + checkRule(g, "a", expecting); + } + + @Test public void testLabeledNotCharSet() throws Exception { + Grammar g = new Grammar( + "lexer grammar P;\n"+ + "A : t=~'3' ;\n"); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2-{'\\u0000'..'2', '4'..'\\uFFFF'}->.s3\n" + + ".s3->:s4\n" + + ":s4-->.s5\n"; + checkRule(g, "A", expecting); + } + + @Test public void testLabeledNotBlockSet() throws Exception { + Grammar g = new Grammar( + "lexer grammar P;\n"+ + "A : t=~('3'|'b') ;\n"); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2-{'\\u0000'..'2', '4'..'a', 'c'..'\\uFFFF'}->.s3\n" + + ".s3->:s4\n" + + ":s4-->.s5\n"; + checkRule(g, "A", expecting); + } + + @Test public void testEscapedCharLiteral() throws Exception { + Grammar g = new Grammar( + "grammar P;\n"+ + "a : '\\n';"); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2-'\\n'->.s3\n" + + ".s3->:s4\n" + + ":s4-EOF->.s5\n"; + checkRule(g, "a", expecting); + } + + @Test public void testEscapedStringLiteral() throws Exception { + Grammar g = new Grammar( + "grammar P;\n"+ + "a : 'a\\nb\\u0030c\\'';"); + String expecting = + "RuleStart_a_0->s2\n" + + "s2-'a\\nb\\u0030c\\''->s3\n" + + "s3->RuleStop_a_1\n" + + "RuleStop_a_1-EOF->s4\n"; + checkRule(g, "a", expecting); + } + + // AUTO BACKTRACKING STUFF + + @Test public void testAutoBacktracking_RuleBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : 'a'{;}|'b';" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s1->.s9\n" + + ".s10-'b'->.s11\n" + + ".s11->.s6\n" + + ".s2-{synpred1_t}?->.s3\n" + + ".s3-'a'->.s4\n" + + ".s4-{}->.s5\n" + + ".s5->.s6\n" + + ".s6->:s7\n" + + ".s9->.s10\n" + + ":s7-EOF->.s8\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_RuleSetBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : 'a'|'b';" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2-'a'..'b'->.s3\n" + + ".s3->:s4\n" + + ":s4-EOF->.s5\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_SimpleBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'{;}|'b') ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s10->.s11\n" + + ".s11-'b'->.s12\n" + + ".s12->.s7\n" + + ".s2->.s10\n" + + ".s2->.s3\n" + + ".s3-{synpred1_t}?->.s4\n" + + ".s4-'a'->.s5\n" + + ".s5-{}->.s6\n" + + ".s6->.s7\n" + + ".s7->:s8\n" + + ":s8-EOF->.s9\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_SetBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'|'b') ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2-'a'..'b'->.s3\n" + + ".s3->:s4\n" + + ":s4-EOF->.s5\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_StarBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'{;}|'b')* ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s12->.s13\n" + + ".s13-{synpred2_t}?->.s14\n" + + ".s14-'b'->.s15\n" + + ".s15->.s8\n" + + ".s16->.s9\n" + + ".s2->.s16\n" + + ".s2->.s3\n" + + ".s3->.s12\n" + + ".s3->.s4\n" + + ".s4-{synpred1_t}?->.s5\n" + + ".s5-'a'->.s6\n" + + ".s6-{}->.s7\n" + + ".s7->.s8\n" + + ".s8->.s3\n" + + ".s8->.s9\n" + + ".s9->:s10\n" + + ":s10-EOF->.s11\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_StarSetBlock_IgnoresPreds() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'|'b')* ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2->.s3\n" + + ".s2->.s9\n" + + ".s3->.s4\n" + + ".s4-'a'..'b'->.s5\n" + + ".s5->.s3\n" + + ".s5->.s6\n" + + ".s6->:s7\n" + + ".s9->.s6\n" + + ":s7-EOF->.s8\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_StarSetBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'|'b'{;})* ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s11->.s12\n" + + ".s12-{synpred2_t}?->.s13\n" + + ".s13-'b'->.s14\n" + + ".s14-{}->.s15\n" + + ".s15->.s7\n" + + ".s16->.s8\n" + + ".s2->.s16\n" + + ".s2->.s3\n" + + ".s3->.s11\n" + + ".s3->.s4\n" + + ".s4-{synpred1_t}?->.s5\n" + + ".s5-'a'->.s6\n" + + ".s6->.s7\n" + + ".s7->.s3\n" + + ".s7->.s8\n" + + ".s8->:s9\n" + + ":s9-EOF->.s10\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_StarBlock1Alt() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a')* ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s10->.s7\n" + + ".s2->.s10\n" + + ".s2->.s3\n" + + ".s3->.s4\n" + + ".s4-{synpred1_t}?->.s5\n" + + ".s5-'a'->.s6\n" + + ".s6->.s3\n" + + ".s6->.s7\n" + + ".s7->:s8\n" + + ":s8-EOF->.s9\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_PlusBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'{;}|'b')+ ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s12->.s13\n" + + ".s13-{synpred2_t}?->.s14\n" + + ".s14-'b'->.s15\n" + + ".s15->.s8\n" + + ".s2->.s3\n" + + ".s3->.s12\n" + + ".s3->.s4\n" + + ".s4-{synpred1_t}?->.s5\n" + + ".s5-'a'->.s6\n" + + ".s6-{}->.s7\n" + + ".s7->.s8\n" + + ".s8->.s3\n" + + ".s8->.s9\n" + + ".s9->:s10\n" + + ":s10-EOF->.s11\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_PlusSetBlock() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'|'b'{;})+ ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s11->.s12\n" + + ".s12-{synpred2_t}?->.s13\n" + + ".s13-'b'->.s14\n" + + ".s14-{}->.s15\n" + + ".s15->.s7\n" + + ".s2->.s3\n" + + ".s3->.s11\n" + + ".s3->.s4\n" + + ".s4-{synpred1_t}?->.s5\n" + + ".s5-'a'->.s6\n" + + ".s6->.s7\n" + + ".s7->.s3\n" + + ".s7->.s8\n" + + ".s8->:s9\n" + + ":s9-EOF->.s10\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_PlusBlock1Alt() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a')+ ;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2->.s3\n" + + ".s3->.s4\n" + + ".s4-{synpred1_t}?->.s5\n" + + ".s5-'a'->.s6\n" + + ".s6->.s3\n" + + ".s6->.s7\n" + + ".s7->:s8\n" + + ":s8-EOF->.s9\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_OptionalBlock2Alts() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a'{;}|'b')?;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s10->.s11\n" + + ".s10->.s14\n" + + ".s11-{synpred2_t}?->.s12\n" + + ".s12-'b'->.s13\n" + + ".s13->.s7\n" + + ".s14->.s7\n" + + ".s2->.s10\n" + + ".s2->.s3\n" + + ".s3-{synpred1_t}?->.s4\n" + + ".s4-'a'->.s5\n" + + ".s5-{}->.s6\n" + + ".s6->.s7\n" + + ".s7->:s8\n" + + ":s8-EOF->.s9\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_OptionalBlock1Alt() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a')?;" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s2->.s3\n" + + ".s2->.s9\n" + + ".s3-{synpred1_t}?->.s4\n" + + ".s4-'a'->.s5\n" + + ".s5->.s6\n" + + ".s6->:s7\n" + + ".s9->.s6\n" + + ":s7-EOF->.s8\n"; + checkRule(g, "a", expecting); + } + + @Test public void testAutoBacktracking_ExistingPred() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "options {backtrack=true;}\n"+ + "a : ('a')=> 'a' | 'b';" + ); + String expecting = + ".s0->.s1\n" + + ".s1->.s2\n" + + ".s1->.s8\n" + + ".s10->.s5\n" + + ".s2-{synpred1_t}?->.s3\n" + + ".s3-'a'->.s4\n" + + ".s4->.s5\n" + + ".s5->:s6\n" + + ".s8->.s9\n" + + ".s9-'b'->.s10\n" + + ":s6-EOF->.s7\n"; + checkRule(g, "a", expecting); + } +*/ + + @Test public void testDefaultMode() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a' ;\n" + + "X : 'x' ;\n" + + "mode FOO;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + String expecting = + "BlockStart_0->RuleStart_A_2\n" + + "BlockStart_0->RuleStart_X_4\n" + + "RuleStart_A_2->s10\n" + + "RuleStart_X_4->s12\n" + + "s10-'a'->s11\n" + + "s12-'x'->s13\n" + + "s11->RuleStop_A_3\n" + + "s13->RuleStop_X_5\n"; + checkTokensRule(g, "DEFAULT_MODE", expecting); + } + + @Test public void testMode() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a' ;\n" + + "X : 'x' ;\n" + + "mode FOO;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + String expecting = + "BlockStart_1->RuleStart_B_6\n" + + "BlockStart_1->RuleStart_C_8\n" + + "RuleStart_B_6->s14\n" + + "RuleStart_C_8->s16\n" + + "s14-'b'->s15\n" + + "s16-'c'->s17\n" + + "s15->RuleStop_B_7\n" + + "s17->RuleStop_C_9\n"; + checkTokensRule(g, "FOO", expecting); + } + + void checkTokensRule(LexerGrammar g, String modeName, String expecting) { + if ( g.ast!=null && !g.ast.hasErrors ) { + System.out.println(g.ast.toStringTree()); + Tool antlr = new Tool(); + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) + for (Grammar imp : g.getImportedGrammars()) { + antlr.processNonCombinedGrammar(imp); + } + } + } + + if ( g.modes.get(modeName)==null ) { + System.err.println("no such mode "+modeName); + return; + } + + ParserATNFactory f = new LexerATNFactory((LexerGrammar)g); + ATN nfa = f.createATN(); + ATNState startState = nfa.modeNameToStartState.get(modeName); + ATNPrinter serializer = new ATNPrinter(g, startState); + String result = serializer.toString(); + + //System.out.print(result); + assertEquals(expecting, result); + } + + void checkRule(Grammar g, String ruleName, String expecting) { + if ( g.ast!=null && !g.ast.hasErrors ) { + System.out.println(g.ast.toStringTree()); + Tool antlr = new Tool(); + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) + for (Grammar imp : g.getImportedGrammars()) { + antlr.processNonCombinedGrammar(imp); + } + } + } + + ParserATNFactory f = new ParserATNFactory(g); + ATN atn = f.createATN(); + + DOTGenerator dot = new DOTGenerator(g); + System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule(ruleName)))); + + Rule r = g.getRule(ruleName); + ATNState startState = atn.ruleToStartState.get(r); + ATNPrinter serializer = new ATNPrinter(g, startState); + String result = serializer.toString(); + + //System.out.print(result); + assertEquals(expecting, result); + } +} diff --git a/tool/test/org/antlr/v4/test/TestATNDeserialization.java b/tool/test/org/antlr/v4/test/TestATNDeserialization.java new file mode 100644 index 000000000..affca5566 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestATNDeserialization.java @@ -0,0 +1,127 @@ +package org.antlr.v4.test; + +import org.antlr.v4.misc.Utils; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ParserInterpreter; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; +import org.junit.Test; + +public class TestATNDeserialization extends BaseTest { + @Test public void testSimpleNoBlock() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B ;"); + checkDeserializationIsStable(g); + } + + @Test public void testNot() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "tokens {A; B; C;}\n" + + "a : ~A ;"); + checkDeserializationIsStable(g); + } + + @Test public void testWildcard() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "tokens {A; B; C;}\n" + + "a : . ;"); + checkDeserializationIsStable(g); + } + + @Test public void testPEGAchillesHeel() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B ;"); + checkDeserializationIsStable(g); + } + + @Test public void test3Alts() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B | A B C ;"); + checkDeserializationIsStable(g); + } + + @Test public void testSimpleLoop() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A+ B ;"); + checkDeserializationIsStable(g); + } + + @Test public void testRuleRef() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : e ;\n" + + "e : E ;\n"); + checkDeserializationIsStable(g); + } + + @Test public void testLexerTwoRules() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a' ;\n" + + "B : 'b' ;\n"); + checkDeserializationIsStable(lg); + } + + @Test public void testLexerRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : '0'..'9' ;\n"); + checkDeserializationIsStable(lg); + } + + @Test public void testLexerLoops() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : '0'..'9'+ ;\n"); + checkDeserializationIsStable(lg); + } + + @Test public void testLexerNotSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b')\n ;"); + checkDeserializationIsStable(lg); + } + + @Test public void testLexerNotSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b'|'e'|'p'..'t')\n ;"); + checkDeserializationIsStable(lg); + } + + @Test public void testLexerNotSetWithRange2() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b') ~('e'|'p'..'t')\n ;"); + checkDeserializationIsStable(lg); + } + + @Test public void test2ModesInLexer() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a'\n ;\n" + + "mode M;\n" + + "B : 'b';\n" + + "mode M2;\n" + + "C : 'c';\n"); + checkDeserializationIsStable(lg); + } + + protected void checkDeserializationIsStable(Grammar g) { + ATN atn = createATN(g); + char[] data = Utils.toCharArray(atn.getSerialized()); + String atnData = atn.getDecoded(); + ATN atn2 = ParserInterpreter.deserialize(data); + atn2.g = g; + String atn2Data = atn2.getDecoded(); + + assertEquals(atnData, atn2Data); + } +} diff --git a/tool/test/org/antlr/v4/test/TestATNInterpreter.java b/tool/test/org/antlr/v4/test/TestATNInterpreter.java new file mode 100644 index 000000000..ba42cf48a --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestATNInterpreter.java @@ -0,0 +1,289 @@ +package org.antlr.v4.test; + +import org.antlr.v4.automata.ParserATNFactory; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.tool.*; +import org.junit.Test; + +import java.util.List; + +/** */ +public class TestATNInterpreter extends BaseTest { + @Test public void testSimpleNoBlock() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B ;"); + checkMatchedAlt(lg, g, "ab", 1); + } + + @Test public void testSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "tokens {A; B; C;}\n" + + "a : ~A ;"); + checkMatchedAlt(lg, g, "b", 1); + } + + @Test public void testPEGAchillesHeel() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B ;"); + checkMatchedAlt(lg, g, "a", 1); + checkMatchedAlt(lg, g, "ab", 2); + checkMatchedAlt(lg, g, "abc", 2); + } + + @Test public void testMustTrackPreviousGoodAlt() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B ;"); + int errorIndex = 0; + int errorTokenType = 0; + try { + checkMatchedAlt(lg, g, "ac", 1); + } + catch (NoViableAltException re) { + errorIndex = re.index; + errorTokenType = re.token.getType(); + } + assertEquals(1, errorIndex); + assertEquals(errorTokenType, 5); + } + + @Test public void testMustTrackPreviousGoodAlt2() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "D : 'd' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B | A B C ;"); + checkMatchedAlt(lg, g, "a", 1 ); + checkMatchedAlt(lg, g, "ab", 2); + checkMatchedAlt(lg, g, "abc", 3); + int errorIndex = 0; + int errorTokenType = 0; + try { + checkMatchedAlt(lg, g, "abd", 1); + } + catch (NoViableAltException re) { + errorIndex = re.index; + errorTokenType = re.token.getType(); + } + assertEquals(2, errorIndex); + assertEquals(errorTokenType, 6); + } + + @Test public void testMustTrackPreviousGoodAlt3() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "D : 'd' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B | A | A B C ;"); + int errorIndex = 0; + int errorTokenType = 0; + try { + checkMatchedAlt(lg, g, "abd", 1); + } + catch (NoViableAltException re) { + errorIndex = re.index; + errorTokenType = re.token.getType(); + } + assertEquals(2, errorIndex); + assertEquals(errorTokenType, 6); + } + + @Test public void testAmbigAltChooseFirst() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "D : 'd' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B | A B ;"); // first alt + checkMatchedAlt(lg, g, "ab", 1); + checkMatchedAlt(lg, g, "abc", 1); + } + + @Test public void testAmbigAltChooseFirstWithFollowingToken() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "D : 'd' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : (A B | A B) C ;"); // first alt + checkMatchedAlt(lg, g, "abc", 1); + checkMatchedAlt(lg, g, "abcd", 1); + } + + @Test public void testAmbigAltChooseFirstWithFollowingToken2() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "D : 'd' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : (A B | A B | D) C ;"); + checkMatchedAlt(lg, g, "abc", 1); + checkMatchedAlt(lg, g, "abcd", 1); + checkMatchedAlt(lg, g, "dc", 3); + } + + @Test public void testAmbigAltChooseFirst2() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "D : 'd' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B | A B | A B C ;"); + checkMatchedAlt(lg, g, "ab", 1); + checkMatchedAlt(lg, g, "abc", 3); + + int errorIndex = 0; + int errorTokenType = 0; + try { + checkMatchedAlt(lg, g, "abd", 1); + } + catch (NoViableAltException re) { + errorIndex = re.index; + errorTokenType = re.token.getType(); + } + assertEquals(2, errorIndex); + assertEquals(6, errorTokenType); + + checkMatchedAlt(lg, g, "abcd", 3); // ignores d on end + } + + @Test public void testSimpleLoop() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "D : 'd' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A+ B ;"); + checkMatchedAlt(lg, g, "ab", 1); + checkMatchedAlt(lg, g, "aab", 1); + checkMatchedAlt(lg, g, "aaaaaab", 1); + checkMatchedAlt(lg, g, "aabd", 1); + } + + @Test public void testCommonLeftPrefix() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B | A C ;"); + checkMatchedAlt(lg, g, "ab", 1); + checkMatchedAlt(lg, g, "ac", 2); + } + + @Test public void testArbitraryLeftPrefix() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A+ B | A+ C ;"); + checkMatchedAlt(lg, g, "aac", 2); + } + + @Test public void testRecursiveLeftPrefix() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "LP : '(' ;\n" + + "RP : ')' ;\n" + + "INT : '0'..'9'+ ;\n" + ); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : e B | e C ;\n" + + "e : LP e RP\n" + + " | INT\n" + + " ;"); + checkMatchedAlt(lg, g, "34b", 1); + checkMatchedAlt(lg, g, "34c", 2); + checkMatchedAlt(lg, g, "(34)b", 1); + checkMatchedAlt(lg, g, "(34)c", 2); + checkMatchedAlt(lg, g, "((34))b", 1); + checkMatchedAlt(lg, g, "((34))c", 2); + } + + public void checkMatchedAlt(LexerGrammar lg, Grammar g, + String inputString, + int expected) + { + ATN lexatn = createATN(lg); + LexerInterpreter lexInterp = new LexerInterpreter(lexatn); + List types = getTokenTypes(inputString, lexInterp); + System.out.println(types); + + semanticProcess(lg); + g.importVocab(lg); + semanticProcess(g); + + ParserATNFactory f = new ParserATNFactory(g); + ATN atn = f.createATN(); + + ParserInterpreter interp = new ParserInterpreter(atn); + TokenStream input = new IntTokenStream(types); + ATNState startState = atn.ruleToStartState.get(g.getRule("a")); + if ( startState.transition(0).target instanceof BlockStartState ) { + startState = startState.transition(0).target; + } + + DOTGenerator dot = new DOTGenerator(g); + System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("a")))); + System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); + + int result = interp.matchATN(input, startState); + assertEquals(expected, result); + } +} diff --git a/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java new file mode 100644 index 000000000..57a0770cd --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java @@ -0,0 +1,144 @@ +package org.antlr.v4.test; + +import org.antlr.v4.misc.Utils; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.tool.*; +import org.junit.*; + +import java.util.List; + +public class TestATNLexerInterpreter extends BaseTest { + @Test public void testLexerTwoRules() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a' ;\n" + + "B : 'b' ;\n"); + String expecting = "A, B, A, B, EOF"; + checkLexerMatches(lg, "abab", expecting); + } + + @Test public void testShortLongRule() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'xy'\n" + + " | 'xyz'\n" + + " ;\n"); + checkLexerMatches(lg, "xy", "A, EOF"); + checkLexerMatches(lg, "xyz", "A, EOF"); + } + + @Test public void testLexerLoops() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : '0'..'9'+ ;\n" + + "ID : 'a'..'z'+ ;\n"); + String expecting = "ID, INT, ID, INT, EOF"; + checkLexerMatches(lg, "a34bde3", expecting); + } + + @Test public void testLexerNotSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, "c", expecting); + } + + @Test public void testLexerKeywordIDAmbiguity() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "KEND : 'end' ;\n" + + "ID : 'a'..'z'+ ;\n" + + "WS : (' '|'\n')+ ;"); + String expecting = "ID, EOF"; + //checkLexerMatches(lg, "e", expecting); + expecting = "KEND, EOF"; + checkLexerMatches(lg, "end", expecting); + expecting = "ID, EOF"; + checkLexerMatches(lg, "ending", expecting); + expecting = "ID, WS, KEND, WS, ID, EOF"; + checkLexerMatches(lg, "a end bcd", expecting); + } + + @Test public void testLexerRuleRef() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : DIGIT+ ;\n" + + "fragment DIGIT : '0'..'9' ;\n" + + "WS : (' '|'\n')+ ;"); + String expecting = "INT, WS, INT, EOF"; + checkLexerMatches(lg, "32 99", expecting); + } + + @Test public void testRecursiveLexerRuleRef() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "CMT : '/*' (CMT | ~'*')+ '*/' ;\n" + + "WS : (' '|'\n')+ ;"); + String expecting = "CMT, WS, CMT, EOF"; + checkLexerMatches(lg, "/* ick */\n/* /*nested*/ */", expecting); + } + + @Ignore public void testLexerWildcardNonGreedyLoopByDefault() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "CMT : '//' .* '\\n' ;\n"); + String expecting = "CMT, CMT, EOF"; + checkLexerMatches(lg, "//x\n//y\n", expecting); + } + + // should not work. no priority within a single rule. the subrule won't work. need modes + @Ignore + public void testLexerEscapeInString() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "STR : '\"' ('\\\\' '\"' | .)* '\"' ;\n"); // STR : '"' ('\\' '"' | .)* '"' + checkLexerMatches(lg, "\"a\\\"b\"", "STR, EOF"); + checkLexerMatches(lg, "\"a\"", "STR, EOF"); + } + + @Ignore public void testLexerWildcardNonGreedyPlusLoopByDefault() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "CMT : '//' .+ '\\n' ;\n"); + String expecting = "CMT, CMT, EOF"; + checkLexerMatches(lg, "//x\n//y\n", expecting); + } + + @Ignore public void testLexerGreedyOptionalShouldWorkAsWeExpect() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "CMT : '/*' ('*/')? '*/' ;\n"); + String expecting = "CMT, EOF"; + checkLexerMatches(lg, "/**/", expecting); + } + + @Ignore public void testNonGreedyBetweenRules() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : '' ;\n" + + "B : '<' .+ '>' ;\n"); + String expecting = "A, B, EOF"; + checkLexerMatches(lg, "", expecting); + } + + protected void checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { + ATN atn = createATN(lg); + CharStream input = new ANTLRStringStream(inputString); + ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE"); + DOTGenerator dot = new DOTGenerator(lg); + System.out.println(dot.getDOT(startState)); + + List tokenTypes = getTokenTypes(lg, atn, input, false); + String result = Utils.join(tokenTypes.iterator(), ", "); + System.out.println(tokenTypes); + assertEquals(expecting, result); + + // try now adaptive DFA + input.seek(0); + List tokenTypes2 = getTokenTypes(lg, atn, input, true); + assertEquals("interp vs adaptive types differ", tokenTypes, tokenTypes2); + } + +} diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java new file mode 100644 index 000000000..40c63adfa --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -0,0 +1,586 @@ +package org.antlr.v4.test; + +import org.antlr.v4.Tool; +import org.antlr.v4.automata.ParserATNFactory; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.tool.*; +import org.junit.Test; + +import java.util.List; + +public class TestATNParserPrediction extends BaseTest { + @Test public void testAorB() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | B ;"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "a", 1); + checkPredictedAlt(lg, g, decision, "b", 2); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "a", + "b", + "a" + }; + String[] dfa = { + "s0-'a'->:s1=>1\n", + + "s0-'a'->:s1=>1\n" + + "s0-'b'->:s2=>2\n", + + "s0-'a'->:s1=>1\n" + // don't change after it works + "s0-'b'->:s2=>2\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + @Test public void testEmptyInput() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | ;"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "a", 1); + checkPredictedAlt(lg, g, decision, "", 2); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "a", + "", + }; + String[] dfa = { + "s0-'a'->:s1=>1\n", + + "s0-EOF->:s2=>2\n" + + "s0-'a'->:s1=>1\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + @Test public void testPEGAchillesHeel() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B ;"); + checkPredictedAlt(lg, g, 0, "a", 1); + checkPredictedAlt(lg, g, 0, "ab", 2); + checkPredictedAlt(lg, g, 0, "abc", 2); + + String[] inputs = { + "a", + "ab", + "abc" + }; + String[] dfa = { + "s0-'a'->s1\n" + + "s1-EOF->:s2=>1\n", + + "s0-'a'->s1\n" + + "s1-EOF->:s2=>1\n" + + "s1-'b'->:s3=>2\n", + + "s0-'a'->s1\n" + + "s1-EOF->:s2=>1\n" + + "s1-'b'->:s3=>2\n" + }; + checkDFAConstruction(lg, g, 0, inputs, dfa); + } + + @Test public void testRuleRefxory() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : x | y ;\n" + + "x : A ;\n" + + "y : B ;\n"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "a", 1); + checkPredictedAlt(lg, g, decision, "b", 2); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "a", + "b", + "a" + }; + String[] dfa = { + "s0-'a'->:s1=>1\n", + + "s0-'a'->:s1=>1\n" + + "s0-'b'->:s2=>2\n", + + "s0-'a'->:s1=>1\n" + // don't change after it works + "s0-'b'->:s2=>2\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + @Test public void testOptionalRuleChasesGlobalFollow() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : x B ;\n" + + "b : x C ;\n" + + "x : A | ;\n"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "a", 1); + checkPredictedAlt(lg, g, decision, "b", 2); + checkPredictedAlt(lg, g, decision, "c", 2); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "a", + "b", + "c", + "c", + }; + String[] dfa = { + "s0-'a'->:s1=>1\n", + + "s0-'a'->:s1=>1\n" + + "s0-'b'->:s2=>2\n", + + "s0-'a'->:s1=>1\n" + + "s0-'b'->:s2=>2\n" + + "s0-'c'->:s3=>2\n", + + "s0-'a'->:s1=>1\n" + + "s0-'b'->:s2=>2\n" + + "s0-'c'->:s3=>2\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + @Test public void testLL1Ambig() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A | A B ;"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "a", 1); + checkPredictedAlt(lg, g, decision, "ab", 3); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "a", + "ab", + "ab" + }; + String[] dfa = { + "s0-'a'->s1\n" + + "s1-EOF->:s2=>1\n", + + "s0-'a'->s1\n" + + "s1-EOF->:s2=>1\n" + + "s1-'b'->:s3=>3\n", + + "s0-'a'->s1\n" + + "s1-EOF->:s2=>1\n" + + "s1-'b'->:s3=>3\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + @Test public void testLL2Ambig() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B | A B | A B C ;"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "ab", 1); + checkPredictedAlt(lg, g, decision, "abc", 3); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "ab", + "abc", + "ab" + }; + String[] dfa = { + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3=>1\n", + + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3=>1\n" + + "s2-'c'->:s4=>3\n", + + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3=>1\n" + + "s2-'c'->:s4=>3\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + @Test public void testFullLLContextParse() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n"); + // AB predicted in both alts of e but in diff contexts. + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : e B ;\n" + + "b : e A B ;\n" + + "e : A | ;\n"); // TODO: try with three alts + + ATN lexatn = createATN(lg); + LexerInterpreter lexInterp = new LexerInterpreter(lexatn); + + semanticProcess(lg); + g.importVocab(lg); + semanticProcess(g); + + ParserATNFactory f = new ParserATNFactory(g); + ATN atn = f.createATN(); + + RuleStartState aStart = atn.ruleToStartState.get(g.getRule("a")); + RuleStartState bStart = atn.ruleToStartState.get(g.getRule("b")); + RuleStartState eStart = atn.ruleToStartState.get(g.getRule("e")); + ATNState a_e_invoke = aStart.transition(0).target; // + ATNState b_e_invoke = bStart.transition(0).target; // + RuleContext a_ctx = new RuleContext(null, -1, a_e_invoke.stateNumber); + RuleContext b_ctx = new RuleContext(null, -1, b_e_invoke.stateNumber); + RuleContext a_e_ctx = new RuleContext(a_ctx, a_e_invoke.stateNumber, bStart.stateNumber); + RuleContext b_e_ctx = new RuleContext(b_ctx, b_e_invoke.stateNumber, bStart.stateNumber); + + ParserInterpreter interp = new ParserInterpreter(atn); + interp.setContextSensitive(true); + List types = getTokenTypes("ab", lexInterp); + System.out.println(types); + TokenStream input = new IntTokenStream(types); + int alt = interp.adaptivePredict(input, 0, b_e_ctx); + assertEquals(alt, 2); + DFA dfa = interp.decisionToDFA[0]; + String expecting = + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3@{[10]=2}\n"; + assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); + + alt = interp.adaptivePredict(input, 0, b_e_ctx); // cached + assertEquals(alt, 2); + expecting = + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3@{[10]=2}\n"; + assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); + + alt = interp.adaptivePredict(input, 0, a_e_ctx); // forces new context-sens ATN match + assertEquals(alt, 1); + expecting = + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3@{[10]=2, [6]=1}\n"; + assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); + + alt = interp.adaptivePredict(input, 0, b_e_ctx); // cached + assertEquals(alt, 2); + expecting = + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3@{[10]=2, [6]=1}\n"; + assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); + + alt = interp.adaptivePredict(input, 0, a_e_ctx); // cached + assertEquals(alt, 1); + expecting = + "s0-'a'->s1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3@{[10]=2, [6]=1}\n"; + assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); + + types = getTokenTypes("b", lexInterp); + System.out.println(types); + input = new IntTokenStream(types); + alt = interp.adaptivePredict(input, 0, null); // ctx irrelevant + assertEquals(alt, 2); + expecting = + "s0-'a'->s1\n" + + "s0-'b'->:s4=>2\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3@{[10]=2, [6]=1}\n"; + assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); + + types = getTokenTypes("aab", lexInterp); + System.out.println(types); + input = new IntTokenStream(types); + alt = interp.adaptivePredict(input, 0, null); + assertEquals(alt, 1); + expecting = + "s0-'a'->s1\n" + + "s0-'b'->:s4=>2\n" + + "s1-'a'->:s5=>1\n" + + "s1-'b'->s2\n" + + "s2-EOF->:s3@{[10]=2, [6]=1}\n"; + assertEquals(expecting, dfa.toString(g.getTokenDisplayNames())); + } + + @Test public void testRecursiveLeftPrefix() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "LP : '(' ;\n" + + "RP : ')' ;\n" + + "INT : '0'..'9'+ ;\n" + ); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : e B | e C ;\n" + + "e : LP e RP\n" + + " | INT\n" + + " ;"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "34b", 1); + checkPredictedAlt(lg, g, decision, "34c", 2); + checkPredictedAlt(lg, g, decision, "((34))b", 1); + checkPredictedAlt(lg, g, decision, "((34))c", 2); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "34b", + "34c", + "((34))b", + "((34))c" + }; + String[] dfa = { + "s0-INT->s1\n" + + "s1-'b'->:s2=>1\n", + + "s0-INT->s1\n" + + "s1-'b'->:s2=>1\n" + + "s1-'c'->:s3=>2\n", + + "s0-'('->s4\n" + + "s0-INT->s1\n" + + "s1-'b'->:s2=>1\n" + + "s1-'c'->:s3=>2\n" + + "s4-'('->s5\n" + + "s5-INT->s6\n" + + "s6-')'->s7\n" + + "s7-')'->s1\n", + + "s0-'('->s4\n" + + "s0-INT->s1\n" + + "s1-'b'->:s2=>1\n" + + "s1-'c'->:s3=>2\n" + + "s4-'('->s5\n" + + "s5-INT->s6\n" + + "s6-')'->s7\n" + + "s7-')'->s1\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + @Test public void testRecursiveLeftPrefixWithAorABIssue() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n" + + "C : 'c' ;\n" + + "LP : '(' ;\n" + + "RP : ')' ;\n" + + "INT : '0'..'9'+ ;\n" + ); + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : e A | e A B ;\n" + + "e : LP e RP\n" + + " | INT\n" + + " ;"); + int decision = 0; + checkPredictedAlt(lg, g, decision, "34a", 1); + checkPredictedAlt(lg, g, decision, "34ab", 2); // PEG would miss this one! + checkPredictedAlt(lg, g, decision, "((34))a", 1); + checkPredictedAlt(lg, g, decision, "((34))ab", 2); + + // After matching these inputs for decision, what is DFA after each prediction? + String[] inputs = { + "34a", + "34ab", + "((34))a", + "((34))ab", + }; + String[] dfa = { + "s0-INT->s1\n" + + "s1-'a'->s2\n" + + "s2-EOF->:s3=>1\n", + + "s0-INT->s1\n" + + "s1-'a'->s2\n" + + "s2-EOF->:s3=>1\n" + + "s2-'b'->:s4=>2\n", + + "s0-'('->s5\n" + + "s0-INT->s1\n" + + "s1-'a'->s2\n" + + "s2-EOF->:s3=>1\n" + + "s2-'b'->:s4=>2\n" + + "s5-'('->s6\n" + + "s6-INT->s7\n" + + "s7-')'->s8\n" + + "s8-')'->s1\n", + + "s0-'('->s5\n" + + "s0-INT->s1\n" + + "s1-'a'->s2\n" + + "s2-EOF->:s3=>1\n" + + "s2-'b'->:s4=>2\n" + + "s5-'('->s6\n" + + "s6-INT->s7\n" + + "s7-')'->s8\n" + + "s8-')'->s1\n", + }; + checkDFAConstruction(lg, g, decision, inputs, dfa); + } + + /** first check that the ATN predicts right alt. + * Then check adaptive prediction. + */ + public void checkPredictedAlt(LexerGrammar lg, Grammar g, int decision, + String inputString, int expectedAlt) + { + Tool.internalOption_ShowATNConfigsInDFA = true; + ATN lexatn = createATN(lg); + LexerInterpreter lexInterp = new LexerInterpreter(lexatn); + List types = getTokenTypes(inputString, lexInterp); + System.out.println(types); + + semanticProcess(lg); + g.importVocab(lg); + semanticProcess(g); + + ParserATNFactory f = new ParserATNFactory(g); + ATN atn = f.createATN(); + + DOTGenerator dot = new DOTGenerator(g); + System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("a")))); + System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b")))); + System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); + + // Check ATN prediction + ParserInterpreter interp = new ParserInterpreter(atn); + TokenStream input = new IntTokenStream(types); + ATNState startState = atn.decisionToATNState.get(decision); + DFA dfa = new DFA(startState); + int alt = interp.predictATN(dfa, input, decision, RuleContext.EMPTY, false); + + System.out.println(dot.getDOT(dfa, false)); + + assertEquals(expectedAlt, alt); + + // Check adaptive prediction + input.seek(0); + alt = interp.adaptivePredict(input, decision, null); + assertEquals(expectedAlt, alt); + // run 2x; first time creates DFA in atn + input.seek(0); + alt = interp.adaptivePredict(input, decision, null); + assertEquals(expectedAlt, alt); + } + + public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName, + String inputString, RuleContext ctx) + { + Tool.internalOption_ShowATNConfigsInDFA = true; + ATN lexatn = createATN(lg); + LexerInterpreter lexInterp = new LexerInterpreter(lexatn); + + semanticProcess(lg); + g.importVocab(lg); + semanticProcess(g); + + ParserATNFactory f = new ParserATNFactory(g); + ATN atn = f.createATN(); + +// DOTGenerator dot = new DOTGenerator(g); +// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("a")))); +// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b")))); +// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); + + ParserInterpreter interp = new ParserInterpreter(atn); + List types = getTokenTypes(inputString, lexInterp); + System.out.println(types); + TokenStream input = new IntTokenStream(types); + try { + ATNState startState = atn.decisionToATNState.get(0); + DFA dfa = new DFA(startState); +// Rule r = g.getRule(ruleName); + //ATNState startState = atn.ruleToStartState.get(r); + interp.predictATN(dfa, input, 0, ctx, false); + } + catch (NoViableAltException nvae) { + nvae.printStackTrace(System.err); + } + return null; + } + + public void checkDFAConstruction(LexerGrammar lg, Grammar g, int decision, + String[] inputString, String[] dfaString) + { +// Tool.internalOption_ShowATNConfigsInDFA = true; + ATN lexatn = createATN(lg); + LexerInterpreter lexInterp = new LexerInterpreter(lexatn); + + semanticProcess(lg); + g.importVocab(lg); + semanticProcess(g); + + ParserATNFactory f = new ParserATNFactory(g); + ATN atn = f.createATN(); + + ParserInterpreter interp = new ParserInterpreter(atn); + for (int i=0; i types = getTokenTypes(inputString[i], lexInterp); + System.out.println(types); + TokenStream input = new IntTokenStream(types); + try { + interp.adaptivePredict(input, decision, RuleContext.EMPTY); + } + catch (NoViableAltException nvae) { + nvae.printStackTrace(System.err); + } + DFA dfa = interp.decisionToDFA[decision]; + ATNInterpreter.dump(dfa,g); + assertEquals(dfaString[i], dfa.toString(g.getTokenDisplayNames())); + } + } +} diff --git a/tool/test/org/antlr/v4/test/TestATNSerialization.java b/tool/test/org/antlr/v4/test/TestATNSerialization.java new file mode 100644 index 000000000..bbee4e1a6 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestATNSerialization.java @@ -0,0 +1,494 @@ +package org.antlr.v4.test; + +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.tool.*; +import org.junit.Test; + +public class TestATNSerialization extends BaseTest { + @Test public void testSimpleNoBlock() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A B ;"); + String expecting = + "max type 4\n" + + "0:RULE_START 1\n" + + "1:RULE_STOP 1\n" + + "2:BASIC 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "5:BASIC 1\n" + + "6:BASIC 1\n" + + "rule 1:0 0,0\n" + + "0->2 EPSILON 0,0\n" + + "1->6 ATOM -1,0\n" + + "2->3 ATOM 3,0\n" + + "3->4 EPSILON 0,0\n" + + "4->5 ATOM 4,0\n" + + "5->1 EPSILON 0,0\n"; + ATN atn = createATN(g); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testNot() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "tokens {A; B; C;}\n" + + "a : ~A ;"); + String expecting = + "max type 5\n" + + "0:RULE_START 1\n" + + "1:RULE_STOP 1\n" + + "2:BASIC 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "rule 1:0 0,0\n" + + "0->2 EPSILON 0,0\n" + + "1->4 ATOM -1,0\n" + + "2->3 NOT_ATOM 3,0\n" + + "3->1 EPSILON 0,0\n"; + ATN atn = createATN(g); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testWildcard() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "tokens {A; B; C;}\n" + + "a : . ;"); + String expecting = + "max type 5\n" + + "0:RULE_START 1\n" + + "1:RULE_STOP 1\n" + + "2:BASIC 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "rule 1:0 0,0\n" + + "0->2 EPSILON 0,0\n" + + "1->4 ATOM -1,0\n" + + "2->3 WILDCARD 0,0\n" + + "3->1 EPSILON 0,0\n"; + ATN atn = createATN(g); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testPEGAchillesHeel() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B ;"); + String expecting = + "max type 4\n" + + "0:RULE_START 1\n" + + "1:RULE_STOP 1\n" + + "2:BASIC 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "5:BASIC 1\n" + + "6:BASIC 1\n" + + "7:BASIC 1\n" + + "8:BLOCK_START 1\n" + + "9:BLOCK_END 1\n" + + "10:BASIC 1\n" + + "rule 1:0 0,0\n" + + "0->8 EPSILON 0,0\n" + + "1->10 ATOM -1,0\n" + + "2->3 ATOM 3,0\n" + + "3->9 EPSILON 0,0\n" + + "4->5 ATOM 3,0\n" + + "5->6 EPSILON 0,0\n" + + "6->7 ATOM 4,0\n" + + "7->9 EPSILON 0,0\n" + + "8->2 EPSILON 0,0\n" + + "8->4 EPSILON 0,0\n" + + "9->1 EPSILON 0,0\n" + + "0:8\n"; + ATN atn = createATN(g); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void test3Alts() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A | A B | A B C ;"); + String expecting = + "max type 5\n" + + "0:RULE_START 1\n" + + "1:RULE_STOP 1\n" + + "2:BASIC 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "5:BASIC 1\n" + + "6:BASIC 1\n" + + "7:BASIC 1\n" + + "8:BASIC 1\n" + + "9:BASIC 1\n" + + "10:BASIC 1\n" + + "11:BASIC 1\n" + + "12:BASIC 1\n" + + "13:BASIC 1\n" + + "14:BLOCK_START 1\n" + + "15:BLOCK_END 1\n" + + "16:BASIC 1\n" + + "rule 1:0 0,0\n" + + "0->14 EPSILON 0,0\n" + + "1->16 ATOM -1,0\n" + + "2->3 ATOM 3,0\n" + + "3->15 EPSILON 0,0\n" + + "4->5 ATOM 3,0\n" + + "5->6 EPSILON 0,0\n" + + "6->7 ATOM 4,0\n" + + "7->15 EPSILON 0,0\n" + + "8->9 ATOM 3,0\n" + + "9->10 EPSILON 0,0\n" + + "10->11 ATOM 4,0\n" + + "11->12 EPSILON 0,0\n" + + "12->13 ATOM 5,0\n" + + "13->15 EPSILON 0,0\n" + + "14->2 EPSILON 0,0\n" + + "14->4 EPSILON 0,0\n" + + "14->8 EPSILON 0,0\n" + + "15->1 EPSILON 0,0\n" + + "0:14\n"; + ATN atn = createATN(g); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testSimpleLoop() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : A+ B ;"); + String expecting = + "max type 4\n" + + "0:RULE_START 1\n" + + "1:RULE_STOP 1\n" + + "2:BASIC 1\n" + + "3:BASIC 1\n" + + "4:PLUS_BLOCK_START 1\n" + + "5:BLOCK_END 1\n" + + "6:PLUS_LOOP_BACK 1\n" + + "7:BASIC 1\n" + + "8:BASIC 1\n" + + "9:BASIC 1\n" + + "10:BASIC 1\n" + + "rule 1:0 0,0\n" + + "0->4 EPSILON 0,0\n" + + "1->10 ATOM -1,0\n" + + "2->3 ATOM 3,0\n" + + "3->5 EPSILON 0,0\n" + + "4->2 EPSILON 0,0\n" + + "5->6 EPSILON 0,0\n" + + "6->2 EPSILON 0,0\n" + + "6->7 EPSILON 0,0\n" + + "7->8 EPSILON 0,0\n" + + "8->9 ATOM 4,0\n" + + "9->1 EPSILON 0,0\n" + + "0:4\n" + + "1:4\n" + + "2:6\n"; + ATN atn = createATN(g); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testRuleRef() throws Exception { + Grammar g = new Grammar( + "parser grammar T;\n"+ + "a : e ;\n" + + "e : E ;\n"); + String expecting = + "max type 3\n" + + "0:RULE_START 1\n" + + "1:RULE_STOP 1\n" + + "2:RULE_START 2\n" + + "3:RULE_STOP 2\n" + + "4:BASIC 1\n" + + "5:BASIC 1\n" + + "6:BASIC 2\n" + + "7:BASIC 2\n" + + "8:BASIC 2\n" + + "rule 1:0 0,0\n" + + "rule 2:2 0,0\n" + + "0->4 EPSILON 0,0\n" + + "1->8 ATOM -1,0\n" + + "2->6 EPSILON 0,0\n" + + "3->5 EPSILON 0,0\n" + + "4->5 RULE 2,2\n" + + "5->1 EPSILON 0,0\n" + + "6->7 ATOM 3,0\n" + + "7->3 EPSILON 0,0\n"; + ATN atn = createATN(g); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testLexerTwoRules() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a' ;\n" + + "B : 'b' ;\n"); + String expecting = + "max type 4\n" + + "0:TOKEN_START 0\n" + + "1:RULE_START 1\n" + + "2:RULE_STOP 1\n" + + "3:RULE_START 2\n" + + "4:RULE_STOP 2\n" + + "5:BASIC 1\n" + + "6:BASIC 1\n" + + "7:BASIC 2\n" + + "8:BASIC 2\n" + + "rule 1:1 3,0\n" + + "rule 2:3 4,0\n" + + "mode 0:0\n" + + "0->1 EPSILON 0,0\n" + + "0->3 EPSILON 0,0\n" + + "1->5 EPSILON 0,0\n" + + "3->7 EPSILON 0,0\n" + + "5->6 ATOM 97,0\n" + + "6->2 EPSILON 0,0\n" + + "7->8 ATOM 98,0\n" + + "8->4 EPSILON 0,0\n" + + "0:0\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testLexerRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : '0'..'9' ;\n"); + String expecting = + "max type 3\n" + + "0:TOKEN_START 0\n" + + "1:RULE_START 1\n" + + "2:RULE_STOP 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "rule 1:1 3,0\n" + + "mode 0:0\n" + + "0->1 EPSILON 0,0\n" + + "1->3 EPSILON 0,0\n" + + "3->4 RANGE 48,57\n" + + "4->2 EPSILON 0,0\n" + + "0:0\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testLexerLoops() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : '0'..'9'+ ;\n"); + String expecting = + "max type 3\n" + + "0:TOKEN_START 0\n" + + "1:RULE_START 1\n" + + "2:RULE_STOP 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "5:PLUS_BLOCK_START 1\n" + + "6:BLOCK_END 1\n" + + "7:PLUS_LOOP_BACK 1\n" + + "8:BASIC 1\n" + + "rule 1:1 3,0\n" + + "mode 0:0\n" + + "0->1 EPSILON 0,0\n" + + "1->5 EPSILON 0,0\n" + + "3->4 RANGE 48,57\n" + + "4->6 EPSILON 0,0\n" + + "5->3 EPSILON 0,0\n" + + "6->7 EPSILON 0,0\n" + + "7->3 EPSILON 0,0\n" + + "7->8 EPSILON 0,0\n" + + "8->2 EPSILON 0,0\n" + + "0:0\n" + + "1:5\n" + + "2:5\n" + + "3:7\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testLexerAction() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a' {a} ;\n" + + "B : 'b' ;\n" + + "C : 'c' {c} ;\n"); + String expecting = + "max type 5\n" + + "0:TOKEN_START 0\n" + + "1:RULE_START 1\n" + + "2:RULE_STOP 1\n" + + "3:RULE_START 2\n" + + "4:RULE_STOP 2\n" + + "5:RULE_START 3\n" + + "6:RULE_STOP 3\n" + + "7:BASIC 1\n" + + "8:BASIC 1\n" + + "9:BASIC 1\n" + + "10:BASIC 2\n" + + "11:BASIC 2\n" + + "12:BASIC 3\n" + + "13:BASIC 3\n" + + "14:BASIC 3\n" + + "rule 1:1 3,1\n" + + "rule 2:3 4,0\n" + + "rule 3:5 5,2\n" + + "mode 0:0\n" + + "0->1 EPSILON 0,0\n" + + "0->3 EPSILON 0,0\n" + + "0->5 EPSILON 0,0\n" + + "1->7 EPSILON 0,0\n" + + "3->10 EPSILON 0,0\n" + + "5->12 EPSILON 0,0\n" + + "7->8 ATOM 97,0\n" + + "8->9 EPSILON 0,0\n" + + "9->2 EPSILON 0,0\n" + + "10->11 ATOM 98,0\n" + + "11->4 EPSILON 0,0\n" + + "12->13 ATOM 99,0\n" + + "13->14 EPSILON 0,0\n" + + "14->6 EPSILON 0,0\n" + + "0:0\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testLexerNotSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b')\n ;"); + String expecting = + "max type 3\n" + + "0:TOKEN_START 0\n" + + "1:RULE_START 1\n" + + "2:RULE_STOP 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "rule 1:1 3,0\n" + + "mode 0:0\n" + + "0:'a'..'b'\n" + + "0->1 EPSILON 0,0\n" + + "1->3 EPSILON 0,0\n" + + "3->4 NOT_SET 0,0\n" + + "4->2 EPSILON 0,0\n" + + "0:0\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testLexerNotSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b'|'e'|'p'..'t')\n ;"); + String expecting = + "max type 3\n" + + "0:TOKEN_START 0\n" + + "1:RULE_START 1\n" + + "2:RULE_STOP 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "rule 1:1 3,0\n" + + "mode 0:0\n" + + "0:'a'..'b', 'e'..'e', 'p'..'t'\n" + + "0->1 EPSILON 0,0\n" + + "1->3 EPSILON 0,0\n" + + "3->4 NOT_SET 0,0\n" + + "4->2 EPSILON 0,0\n" + + "0:0\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void testLexerNotSetWithRange2() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b') ~('e'|'p'..'t')\n ;"); + String expecting = + "max type 3\n" + + "0:TOKEN_START 0\n" + + "1:RULE_START 1\n" + + "2:RULE_STOP 1\n" + + "3:BASIC 1\n" + + "4:BASIC 1\n" + + "5:BASIC 1\n" + + "6:BASIC 1\n" + + "rule 1:1 3,0\n" + + "mode 0:0\n" + + "0:'a'..'b'\n" + + "1:'e'..'e', 'p'..'t'\n" + + "0->1 EPSILON 0,0\n" + + "1->3 EPSILON 0,0\n" + + "3->4 NOT_SET 0,0\n" + + "4->5 EPSILON 0,0\n" + + "5->6 NOT_SET 1,0\n" + + "6->2 EPSILON 0,0\n" + + "0:0\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + + @Test public void test2ModesInLexer() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a'\n ;\n" + + "mode M;\n" + + "B : 'b';\n" + + "mode M2;\n" + + "C : 'c';\n"); + String expecting = + "max type 5\n" + + "0:TOKEN_START 0\n" + + "1:TOKEN_START 0\n" + + "2:TOKEN_START 0\n" + + "3:RULE_START 1\n" + + "4:RULE_STOP 1\n" + + "5:RULE_START 2\n" + + "6:RULE_STOP 2\n" + + "7:RULE_START 3\n" + + "8:RULE_STOP 3\n" + + "9:BASIC 1\n" + + "10:BASIC 1\n" + + "11:BASIC 2\n" + + "12:BASIC 2\n" + + "13:BASIC 3\n" + + "14:BASIC 3\n" + + "rule 1:3 3,0\n" + + "rule 2:5 4,0\n" + + "rule 3:7 5,0\n" + + "mode 0:0\n" + + "mode 1:1\n" + + "mode 2:2\n" + + "0->3 EPSILON 0,0\n" + + "1->5 EPSILON 0,0\n" + + "2->7 EPSILON 0,0\n" + + "3->9 EPSILON 0,0\n" + + "5->11 EPSILON 0,0\n" + + "7->13 EPSILON 0,0\n" + + "9->10 ATOM 97,0\n" + + "10->4 EPSILON 0,0\n" + + "11->12 ATOM 98,0\n" + + "12->6 EPSILON 0,0\n" + + "13->14 ATOM 99,0\n" + + "14->8 EPSILON 0,0\n" + + "0:0\n" + + "1:1\n" + + "2:2\n"; + ATN atn = createATN(lg); + String result = atn.getDecoded(); + assertEquals(expecting, result); + } + +} diff --git a/tool/test/org/antlr/v4/test/TestActionSplitter.java b/tool/test/org/antlr/v4/test/TestActionSplitter.java new file mode 100644 index 000000000..b6945483d --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestActionSplitter.java @@ -0,0 +1,53 @@ +package org.antlr.v4.test; + +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.Token; +import org.antlr.v4.parse.ActionSplitter; +import org.antlr.v4.semantics.BlankActionSplitterListener; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class TestActionSplitter extends BaseTest { + static String[] exprs = { + "foo", "['foo'<29>]", + "$x", "['$x'<20>]", + "\\$x", "['\\$'<6>, 'x'<29>]", + "$x.y", "['$x.y'<11>]", + "$ID.text", "['$ID.text'<11>]", + "$ID", "['$ID'<20>]", + "$ID.getText()", "['$ID'<20>, '.getText()'<29>]", + "$ID.text = \"test\";", "['$ID.text = \"test\";'<10>]", + "$a.line == $b.line", "['$a.line'<11>, ' == '<29>, '$b.line'<11>]", + "$r.tree", "['$r.tree'<11>]", + "foo $a::n bar", "['foo '<29>, '$a::n'<13>, ' bar'<29>]", + "$Symbols[-1]::names.add($id.text);", "['$Symbols[-1]::names'<16>, '.add('<29>, '$id.text'<11>, ');'<29>]", + "$Symbols[0]::names.add($id.text);", "['$Symbols[0]::names'<18>, '.add('<29>, '$id.text'<11>, ');'<29>]", + "$Symbols::x;", "['$Symbols::x'<13>, ';'<29>]", + "$Symbols.size()>0", "['$Symbols'<20>, '.size()>0'<29>]", + "$field::x = $field.st;", "['$field::x = $field.st;'<12>]", + "$foo.get(\"ick\");", "['$foo'<20>, '.get(\"ick\");'<29>]", + }; + + @Test public void testExprs() { + for (int i = 0; i < exprs.length; i+=2) { + String input = exprs[i]; + String expect = exprs[i+1]; + List chunks = getActionChunks(input); + assertEquals("input: "+input, expect, chunks.toString()); + } + } + + public static List getActionChunks(String a) { + List chunks = new ArrayList(); + ActionSplitter splitter = new ActionSplitter(new ANTLRStringStream(a), + new BlankActionSplitterListener()); + Token t = splitter.nextToken(); + while ( t.getType()!=Token.EOF ) { + chunks.add("'"+t.getText()+"'<"+t.getType()+">"); + t = splitter.nextToken(); + } + return chunks; + } +} diff --git a/tool/test/org/antlr/v4/test/TestActionTranslation.java b/tool/test/org/antlr/v4/test/TestActionTranslation.java new file mode 100644 index 000000000..9c7039e76 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestActionTranslation.java @@ -0,0 +1,522 @@ +package org.antlr.v4.test; + +import org.antlr.runtime.RecognitionException; +import org.antlr.v4.automata.ATNFactory; +import org.antlr.v4.automata.LexerATNFactory; +import org.antlr.v4.automata.ParserATNFactory; +import org.antlr.v4.codegen.CodeGenerator; +import org.antlr.v4.semantics.SemanticPipeline; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; +import org.junit.Test; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupString; + +/** */ +public class TestActionTranslation extends BaseTest { + String attributeTemplate = + "attributeTemplate(members,init,inline,finally,inline2) ::= <<\n" + + "parser grammar A;\n"+ + "@members {#members##end-members#}\n" + + "a[int x, int x1] returns [int y]\n" + + "@init {#init##end-init#}\n" + + " : id=ID ids+=ID lab=b[34] c d {\n" + + " #inline##end-inline#\n" + + " }\n" + + " c\n" + + " ;\n" + + " finally {#finally##end-finally#}\n" + + "b[int d] returns [int e]\n" + + " : {#inline2##end-inline2#}\n" + + " ;\n" + + "c returns [int x, int y] : ;\n" + + "d : ;\n" + + ">>"; + + String scopeTemplate = + "scopeTemplate(members,init,inline,finally,inline2) ::= <<\n" + + "parser grammar A;\n"+ + "@members {\n" + + "#members##end-members#\n" + + "}\n" + + "scope S { int i; }\n" + + "a\n" + + "scope { int z; }\n" + + "scope S;\n" + + "@init {#init##end-init#}\n" + + " : {\n" + + " #inline##end-inline#" + + " }\n" + + " ;\n" + + " finally {#finally##end-finally#}\n" + + ">>"; + + @Test public void testEscapedLessThanInAction() throws Exception { + String action = "i<3; ''"; + String expected = "i<3; ''"; + testActions(attributeTemplate, "members", action, expected); + testActions(attributeTemplate, "init", action, expected); + testActions(attributeTemplate, "inline", action, expected); + testActions(attributeTemplate, "finally", action, expected); + testActions(attributeTemplate, "inline2", action, expected); + } + + @Test public void testEscaped$InAction() throws Exception { + String action = "int \\$n; \"\\$in string\\$\""; + String expected = "int \\$n; \"\\$in string\\$\""; + testActions(attributeTemplate, "members", action, expected); + testActions(attributeTemplate, "init", action, expected); + testActions(attributeTemplate, "inline", action, expected); + testActions(attributeTemplate, "finally", action, expected); + testActions(attributeTemplate, "inline2", action, expected); + } + + @Test public void testComplicatedArgParsing() throws Exception { + String action = "x, (*a).foo(21,33), 3.2+1, '\\n', "+ + "\"a,oo\\nick\", {bl, \"fdkj\"eck}"; + String expected = "x, (*a).foo(21,33), 3.2+1, '\\n', "+ + "\"a,oo\\nick\", {bl, \"fdkj\"eck}"; + testActions(attributeTemplate, "members", action, expected); + testActions(attributeTemplate, "init", action, expected); + testActions(attributeTemplate, "inline", action, expected); + testActions(attributeTemplate, "finally", action, expected); + testActions(attributeTemplate, "inline2", action, expected); + } + + @Test public void testComplicatedArgParsingWithTranslation() throws Exception { + String action = "x, $ID.text+\"3242\", (*$ID).foo(21,33), 3.2+1, '\\n', "+ + "\"a,oo\\nick\", {bl, \"fdkj\"eck}"; + String expected = "x, (_rID!=null?_rID.getText():null)+\"3242\"," + + " (*_tID).foo(21,33), 3.2+1, '\\n', \"a,oo\\nick\", {bl, \"fdkj\"eck}"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testArguments() throws Exception { + String action = "$x; $a.x"; + String expected = "_ctx.x; _ctx.x"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testReturnValue() throws Exception { + String action = "$x; $a.x"; + String expected = "_ctx.x; _ctx.x"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testReturnValueWithNumber() throws Exception { + String action = "$a.x1"; + String expected = "_ctx.x1"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testReturnValuesCurrentRule() throws Exception { + String action = "$y; $a.y;"; + String expected = "_ctx.y; _ctx.y;"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testReturnValues() throws Exception { + String action = "$lab.e; $b.e;"; + String expected = "lab.e; _rb.e;"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testReturnWithMultipleRuleRefs() throws Exception { + String action = "$c.x; $c.y;"; + String expected = "_rc.x; _rc.y;"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testTokenRefs() throws Exception { + String action = "$id; $ID; $id.text; $id.getText(); $id.line;"; + String expected = "id; _tID; (id!=null?id.getText():null); id.getText(); (id!=null?id.getLine():0);"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testRuleRefs() throws Exception { + String action = "$lab.start; $c.tree;"; + String expected = "(lab!=null?(()lab.start):null); (_rc!=null?(()_rc.tree):null);"; + testActions(attributeTemplate, "inline", action, expected); + } + + @Test public void testBasicGlobalScope() throws Exception { + String action = "$S::i"; + String expected = "S_stack.peek().i"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void test0IndexedGlobalScope() throws Exception { + String action = "$S[0]::i"; + String expected = "S_stack.get(0).i"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testAbsoluteIndexedGlobalScope() throws Exception { + String action = "$S[3]::i"; + String expected = "S_stack.get(3).i"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testNegIndexedGlobalScope() throws Exception { + String action = "$S[-1]::i"; + String expected = "S_stack.get(S_stack.size()-1-1).i"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testNegIndexedGlobalScope2() throws Exception { + String action = "$S[-$S::i]::i"; + String expected = "S_stack.get(S_stack.size()-S_stack.peek().i-1).i"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testBasicRuleScope() throws Exception { + String action = "$a::z"; + String expected = "a_scope_stack.peek().z"; + testActions(scopeTemplate, "inline", action, expected); + } + + @Test public void testBasicGlobalScopeInRule() throws Exception { + String action = "$S::i"; + String expected = "S_stack.peek().i"; + testActions(scopeTemplate, "inline", action, expected); + } + + @Test public void testSetBasicRuleScope() throws Exception { + String action = "$a::z = 3;"; + String expected = "a_scope_stack.peek().z = 3;"; + testActions(scopeTemplate, "inline", action, expected); + } + + @Test public void testSetBasicGlobalScopeInRule() throws Exception { + String action = "$S::i = 3;"; + String expected = "S_stack.peek().i = 3;"; + testActions(scopeTemplate, "inline", action, expected); + } + + @Test public void testSet0IndexedGlobalScope() throws Exception { + String action = "$S[0]::i = $S::i;"; + String expected = "S_stack.get(0).i = S_stack.peek().i;"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testSetAbsoluteIndexedGlobalScope() throws Exception { + String action = "$S[3]::i = $S::i;"; + String expected = "S_stack.get(3).i = S_stack.peek().i;"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testSetNegIndexedGlobalScope() throws Exception { + String action = "$S[-1]::i = $S::i;"; + String expected = "S_stack.get(S_stack.size()-1-1).i = S_stack.peek().i;"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testSetNegIndexedGlobalScope2() throws Exception { + String action = "$S[-$S::i]::i = $S::i;"; + String expected = "S_stack.get(S_stack.size()-S_stack.peek().i-1).i = S_stack.peek().i;"; + testActions(scopeTemplate, "members", action, expected); + } + + @Test public void testIsolatedDynamicRuleScopeRef() throws Exception { + String action = "$a;"; // refers to stack not top of stack + String expected = "a_scope_stack;"; + testActions(scopeTemplate, "inline", action, expected); + } + + @Test public void testIsolatedGlobalScopeRef() throws Exception { + String action = "$S;"; + String expected = "S_stack;"; + testActions(scopeTemplate, "inline", action, expected); + } + + + + @Test public void testDynamicRuleScopeRefInSubrule() throws Exception { + String action = "$a::n;"; + } + @Test public void testRuleScopeFromAnotherRule() throws Exception { + String action = "$a::n;"; // must be qualified + } + @Test public void testFullyQualifiedRefToCurrentRuleParameter() throws Exception { + String action = "$a.i;"; + } + @Test public void testFullyQualifiedRefToCurrentRuleRetVal() throws Exception { + String action = "$a.i;"; + } + @Test public void testSetFullyQualifiedRefToCurrentRuleRetVal() throws Exception { + String action = "$a.i = 1;"; + } + @Test public void testIsolatedRefToCurrentRule() throws Exception { + String action = "$a;"; + } + @Test public void testIsolatedRefToRule() throws Exception { + String action = "$x;"; + } + @Test public void testFullyQualifiedRefToLabelInCurrentRule() throws Exception { + String action = "$a.x;"; + } + @Test public void testFullyQualifiedRefToListLabelInCurrentRule() throws Exception { + String action = "$a.x;"; // must be qualified + } + @Test public void testFullyQualifiedRefToTemplateAttributeInCurrentRule() throws Exception { + String action = "$a.st;"; // can be qualified + } + @Test public void testRuleRefWhenRuleHasScope() throws Exception { + String action = "$b.start;"; + } + @Test public void testDynamicScopeRefOkEvenThoughRuleRefExists() throws Exception { + String action = "$b::n;"; + } + @Test public void testRefToTemplateAttributeForCurrentRule() throws Exception { + String action = "$st=null;"; + } + @Test public void testRefToTextAttributeForCurrentRule() throws Exception { + String action = "$text"; + } + @Test public void testRefToStartAttributeForCurrentRule() throws Exception { + String action = "$start;"; + } + + @Test public void testTokenLabelFromMultipleAlts() throws Exception { + String action = "$ID.text;"; // must be qualified + } + @Test public void testRuleLabelFromMultipleAlts() throws Exception { + String action = "$b.text;"; // must be qualified + } + @Test public void testUnqualifiedRuleScopeAttribute() throws Exception { + String action = "$n;"; // must be qualified + } + @Test public void testRuleAndTokenLabelTypeMismatch() throws Exception { + } + @Test public void testListAndTokenLabelTypeMismatch() throws Exception { + } + @Test public void testListAndRuleLabelTypeMismatch() throws Exception { + } + @Test public void testArgReturnValueMismatch() throws Exception { + } + @Test public void testSimplePlusEqualLabel() throws Exception { + String action = "$ids.size();"; // must be qualified + } + @Test public void testPlusEqualStringLabel() throws Exception { + String action = "$ids.size();"; // must be qualified + } + @Test public void testPlusEqualSetLabel() throws Exception { + String action = "$ids.size();"; // must be qualified + } + @Test public void testPlusEqualWildcardLabel() throws Exception { + String action = "$ids.size();"; // must be qualified + } + @Test public void testImplicitTokenLabel() throws Exception { + String action = "$ID; $ID.text; $ID.getText()"; + } + + @Test public void testImplicitRuleLabel() throws Exception { + String action = "$r.start;"; + } + + @Test public void testReuseExistingLabelWithImplicitRuleLabel() throws Exception { + String action = "$r.start;"; + } + + @Test public void testReuseExistingListLabelWithImplicitRuleLabel() throws Exception { + String action = "$r.start;"; + } + + @Test public void testReuseExistingLabelWithImplicitTokenLabel() throws Exception { + String action = "$ID.text;"; + } + + @Test public void testReuseExistingListLabelWithImplicitTokenLabel() throws Exception { + String action = "$ID.text;"; + } + + @Test public void testRuleLabelWithoutOutputOption() throws Exception { + } + @Test public void testRuleLabelOnTwoDifferentRulesAST() throws Exception { + } + @Test public void testRuleLabelOnTwoDifferentRulesTemplate() throws Exception { + } + @Test public void testMissingArgs() throws Exception { + } + @Test public void testArgsWhenNoneDefined() throws Exception { + } + @Test public void testReturnInitValue() throws Exception { + } + @Test public void testMultipleReturnInitValue() throws Exception { + } + @Test public void testCStyleReturnInitValue() throws Exception { + } + @Test public void testArgsWithInitValues() throws Exception { + } + @Test public void testArgsOnToken() throws Exception { + } + @Test public void testArgsOnTokenInLexer() throws Exception { + } + @Test public void testLabelOnRuleRefInLexer() throws Exception { + String action = "$i.text"; + } + + @Test public void testRefToRuleRefInLexer() throws Exception { + String action = "$ID.text"; + } + + @Test public void testRefToRuleRefInLexerNoAttribute() throws Exception { + String action = "$ID"; + } + + @Test public void testCharLabelInLexer() throws Exception { + } + @Test public void testCharListLabelInLexer() throws Exception { + } + @Test public void testWildcardCharLabelInLexer() throws Exception { + } + @Test public void testWildcardCharListLabelInLexer() throws Exception { + } + @Test public void testMissingArgsInLexer() throws Exception { + } + @Test public void testLexerRulePropertyRefs() throws Exception { + String action = "$text $type $line $pos $channel $index $start $stop"; + } + + @Test public void testLexerLabelRefs() throws Exception { + String action = "$a $b.text $c $d.text"; + } + + @Test public void testSettingLexerRulePropertyRefs() throws Exception { + String action = "$text $type=1 $line=1 $pos=1 $channel=1 $index"; + } + + @Test public void testArgsOnTokenInLexerRuleOfCombined() throws Exception { + } + @Test public void testMissingArgsOnTokenInLexerRuleOfCombined() throws Exception { + } + @Test public void testTokenLabelTreeProperty() throws Exception { + String action = "$id.tree;"; + } + + @Test public void testTokenRefTreeProperty() throws Exception { + String action = "$ID.tree;"; + } + + @Test public void testAmbiguousTokenRef() throws Exception { + String action = "$ID;"; + } + + @Test public void testAmbiguousTokenRefWithProp() throws Exception { + String action = "$ID.text;"; + } + + @Test public void testRuleRefWithDynamicScope() throws Exception { + String action = "$field::x = $field.st;"; + } + + @Test public void testAssignToOwnRulenameAttr() throws Exception { + String action = "$rule.tree = null;"; + } + + @Test public void testAssignToOwnParamAttr() throws Exception { + String action = "$rule.i = 42; $i = 23;"; + } + + @Test public void testIllegalAssignToOwnRulenameAttr() throws Exception { + String action = "$rule.stop = 0;"; + } + + @Test public void testIllegalAssignToLocalAttr() throws Exception { + String action = "$tree = null; $st = null; $start = 0; $stop = 0; $text = 0;"; + } + + @Test public void testIllegalAssignRuleRefAttr() throws Exception { + String action = "$other.tree = null;"; + } + + @Test public void testIllegalAssignTokenRefAttr() throws Exception { + String action = "$ID.text = \"test\";"; + } + + @Test public void testAssignToTreeNodeAttribute() throws Exception { + String action = "$tree.scope = localScope;"; + } + + @Test public void testDoNotTranslateAttributeCompare() throws Exception { + String action = "$a.line == $b.line"; + } + + @Test public void testDoNotTranslateScopeAttributeCompare() throws Exception { + String action = "if ($rule::foo == \"foo\" || 1) { System.out.println(\"ouch\"); }"; + } + + @Test public void testTreeRuleStopAttributeIsInvalid() throws Exception { + String action = "$r.x; $r.start; $r.stop"; + } + + @Test public void testRefToTextAttributeForCurrentTreeRule() throws Exception { + String action = "$text"; + } + + @Test public void testTypeOfGuardedAttributeRefIsCorrect() throws Exception { + String action = "int x = $b::n;"; + } + + @Test public void testBracketArgParsing() throws Exception { + } + + @Test public void testStringArgParsing() throws Exception { + String action = "34, '{', \"it's<\", '\"', \"\\\"\", 19"; + } + @Test public void testComplicatedSingleArgParsing() throws Exception { + String action = "(*a).foo(21,33,\",\")"; + } + @Test public void testArgWithLT() throws Exception { + String action = "34<50"; + } + @Test public void testGenericsAsArgumentDefinition() throws Exception { + String action = "$foo.get(\"ick\");"; + } + @Test public void testGenericsAsArgumentDefinition2() throws Exception { + String action = "$foo.get(\"ick\"); x=3;"; + } + @Test public void testGenericsAsReturnValue() throws Exception { + } + + + + public void testActions(String templates, String actionName, String action, String expected) { + int lp = templates.indexOf('('); + String name = templates.substring(0, lp); + STGroup group = new STGroupString(templates); + ST st = group.getInstanceOf(name); + st.add(actionName, action); + String grammar = st.render(); + try { + ErrorQueue equeue = new ErrorQueue(); + Grammar g = new Grammar(grammar); + if ( g.ast!=null && !g.ast.hasErrors ) { + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + + ATNFactory factory = new ParserATNFactory(g); + if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g); + g.atn = factory.createATN(); + + CodeGenerator gen = new CodeGenerator(g); + ST outputFileST = gen.generate(); + String output = outputFileST.render(); + //System.out.println(output); + String b = "#" + actionName + "#"; + int start = output.indexOf(b); + String e = "#end-" + actionName + "#"; + int end = output.indexOf(e); + String snippet = output.substring(start+b.length(),end); + assertEquals(expected, snippet); + } + if ( equeue.size()>0 ) { + System.err.println(equeue.toString(g.tool)); + } + } + catch (RecognitionException re) { + re.printStackTrace(System.err); + } + } +} diff --git a/tool/test/org/antlr/v4/test/TestAttributeChecks.java b/tool/test/org/antlr/v4/test/TestAttributeChecks.java new file mode 100644 index 000000000..bc702b1e7 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestAttributeChecks.java @@ -0,0 +1,280 @@ +package org.antlr.v4.test; + +import org.antlr.runtime.RecognitionException; +import org.junit.Test; +import org.stringtemplate.v4.ST; + +/** */ +public class TestAttributeChecks extends BaseTest { + String attributeTemplate = + "parser grammar A;\n"+ + "@members {}\n" + + "a[int x] returns [int y]\n" + + "@init {}\n" + + " : id=ID ids+=ID lab=b[34] {\n" + + " \n" + + " }\n" + + " c\n" + + " ;\n" + + " finally {}\n" + + "b[int d] returns [int e]\n" + + " : {}\n" + + " ;\n" + + "c : ;\n" + + "d : ;\n"; + + String scopeTemplate = + "parser grammar A;\n"+ + "@members {\n" + + "\n" + + "}\n" + + "scope S { int i; }\n" + + "a[int x] returns [int y]\n" + + "scope { int z; }\n" + + "scope S;\n" + + "@init {}\n" + + " : lab=b[34] {\n" + + " " + + " }\n" + + " ;\n" + + " finally {}\n" + + "b[int d] returns [int e]\n" + + "scope { int f; }\n" + + " : {}\n" + + " ;\n" + + "c : ;"; + + String[] membersChecks = { + "$a", "error(29): A.g:2:11: unknown attribute reference a in $a\n", + "$a.y", "error(29): A.g:2:11: unknown attribute reference a in $a.y\n", + }; + + String[] initChecks = { + "$text", "", + "$start", "", + "$x = $y", "", + "$y = $x", "", + "$lab.e", "", + "$ids", "", + + "$a", "error(33): A.g:4:8: missing attribute access on rule reference a in $a\n", + "$c", "error(29): A.g:4:8: unknown attribute reference c in $c\n", + "$a.q", "error(31): A.g:4:10: unknown attribute q for rule a in $a.q\n", + }; + + String[] inlineChecks = { + "$text", "", + "$start", "", + "$x = $y", "", + "$y = $x", "", + "$a.x = $a.y", "", + "$lab.e", "", + "$lab.text", "", + "$b.e", "", + "$c.text", "", + "$ID", "", + "$ID.text", "", + "$id", "", + "$id.text", "", + "$ids", "", + }; + + String[] bad_inlineChecks = { + "$a", "error(33): A.g:6:4: missing attribute access on rule reference a in $a\n", + "$b", "error(33): A.g:6:4: missing attribute access on rule reference b in $b\n", + "$lab", "error(33): A.g:6:4: missing attribute access on rule reference lab in $lab\n", + "$c", "error(33): A.g:6:4: missing attribute access on rule reference c in $c\n", // no scope + "$q", "error(29): A.g:6:4: unknown attribute reference q in $q\n", + "$q.y", "error(29): A.g:6:4: unknown attribute reference q in $q.y\n", + "$q = 3", "error(29): A.g:6:4: unknown attribute reference q in $q\n", + "$q = 3;", "error(29): A.g:6:4: unknown attribute reference q in $q = 3;\n", + "$q.y = 3;", "error(29): A.g:6:4: unknown attribute reference q in $q.y = 3;\n", + "$q = $blort;", "error(29): A.g:6:4: unknown attribute reference q in $q = $blort;\n" + + "error(29): A.g:6:9: unknown attribute reference blort in $blort\n", + "$a.ick", "error(31): A.g:6:6: unknown attribute ick for rule a in $a.ick\n", + "$a.ick = 3;", "error(31): A.g:6:6: unknown attribute ick for rule a in $a.ick = 3;\n", + "$b.d", "error(30): A.g:6:6: cannot access rule d's parameter: $b.d\n", // can't see rule ref's arg + "$d.text", "error(29): A.g:6:4: unknown attribute reference d in $d.text\n", // valid rule, but no ref + "$lab.d", "error(30): A.g:6:8: cannot access rule d's parameter: $lab.d\n", + }; + + String[] finallyChecks = { + "$text", "", + "$start", "", + "$x = $y", "", + "$y = $x", "", + "$lab.e", "", + "$lab.text", "", + "$id", "", + "$id.text", "", + "$ids", "", + + "$lab", "error(33): A.g:9:14: missing attribute access on rule reference lab in $lab\n", + "$a", "error(33): A.g:9:14: missing attribute access on rule reference a in $a\n", + "$q", "error(29): A.g:9:14: unknown attribute reference q in $q\n", + "$q.y", "error(29): A.g:9:14: unknown attribute reference q in $q.y\n", + "$q = 3", "error(29): A.g:9:14: unknown attribute reference q in $q\n", + "$q = 3;", "error(29): A.g:9:14: unknown attribute reference q in $q = 3;\n", + "$q.y = 3;", "error(29): A.g:9:14: unknown attribute reference q in $q.y = 3;\n", + "$q = $blort;", "error(29): A.g:9:14: unknown attribute reference q in $q = $blort;\n" + + "error(29): A.g:9:19: unknown attribute reference blort in $blort\n", + "$a.ick", "error(31): A.g:9:16: unknown attribute ick for rule a in $a.ick\n", + "$a.ick = 3;", "error(31): A.g:9:16: unknown attribute ick for rule a in $a.ick = 3;\n", + "$b", "error(29): A.g:9:14: unknown attribute reference b in $b\n", + "$b.e", "error(29): A.g:9:14: unknown attribute reference b in $b.e\n", // can't see rule refs outside alts + "$b.d", "error(29): A.g:9:14: unknown attribute reference b in $b.d\n", + "$c.text", "error(29): A.g:9:14: unknown attribute reference c in $c.text\n", + "$lab.d", "error(30): A.g:9:18: cannot access rule d's parameter: $lab.d\n", + }; + + String[] dynMembersChecks = { + "$S", "", + "$S::i", "", + "$S::i=$S::i", "", + + "$b::f", "error(54): A.g:3:1: unknown dynamic scope: b in $b::f\n", + "$S::j", "error(55): A.g:3:4: unknown dynamically-scoped attribute for scope S: j in $S::j\n", + "$S::j = 3;", "error(55): A.g:3:4: unknown dynamically-scoped attribute for scope S: j in $S::j = 3;\n", + "$S::j = $S::k;", "error(55): A.g:3:4: unknown dynamically-scoped attribute for scope S: j in $S::j = $S::k;\n" + + "error(55): A.g:3:12: unknown dynamically-scoped attribute for scope S: k in $S::k\n", + }; + + String[] dynInitChecks = { + "$a", "", + "$b", "", + "$lab", "", + "$b::f", "", + "$S::i", "", + "$S::i=$S::i", "", + "$a::z", "", + "$S", "", + + "$S::j", "error(55): A.g:8:11: unknown dynamically-scoped attribute for scope S: j in $S::j\n", + "$S::j = 3;", "error(55): A.g:8:11: unknown dynamically-scoped attribute for scope S: j in $S::j = 3;\n", + "$S::j = $S::k;", "error(55): A.g:8:11: unknown dynamically-scoped attribute for scope S: j in $S::j = $S::k;\n" + + "error(55): A.g:8:19: unknown dynamically-scoped attribute for scope S: k in $S::k\n", + }; + + String[] dynInlineChecks = { + "$a", "", + "$b", "", + "$lab", "", + "$b::f", "", + "$S", "", + "$S::i", "", + "$S::i=$S::i", "", + "$a::z", "", + + "$S::j", "error(55): A.g:10:7: unknown dynamically-scoped attribute for scope S: j in $S::j\n", + "$S::j = 3;", "error(55): A.g:10:7: unknown dynamically-scoped attribute for scope S: j in $S::j = 3;\n", + "$S::j = $S::k;", "error(55): A.g:10:7: unknown dynamically-scoped attribute for scope S: j in $S::j = $S::k;\n" + + "error(55): A.g:10:15: unknown dynamically-scoped attribute for scope S: k in $S::k\n", + "$Q[-1]::y", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[-1]::y\n", + "$Q[-i]::y", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[-i]::y\n", + "$Q[i]::y", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[i]::y\n", + "$Q[0]::y", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[0]::y\n", + "$Q[-1]::y = 23;", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[-1]::y = 23;\n", + "$Q[-i]::y = 23;", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[-i]::y = 23;\n", + "$Q[i]::y = 23;", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[i]::y = 23;\n", + "$Q[0]::y = 23;", "error(54): A.g:10:4: unknown dynamic scope: Q in $Q[0]::y = 23;\n", + "$S[-1]::y", "error(55): A.g:10:11: unknown dynamically-scoped attribute for scope S: y in $S[-1]::y\n", + "$S[-i]::y", "error(55): A.g:10:11: unknown dynamically-scoped attribute for scope S: y in $S[-i]::y\n", + "$S[i]::y", "error(55): A.g:10:10: unknown dynamically-scoped attribute for scope S: y in $S[i]::y\n", + "$S[0]::y", "error(55): A.g:10:10: unknown dynamically-scoped attribute for scope S: y in $S[0]::y\n", + "$S[-1]::y = 23;", "error(55): A.g:10:11: unknown dynamically-scoped attribute for scope S: y in $S[-1]::y = 23;\n", + "$S[-i]::y = 23;", "error(55): A.g:10:11: unknown dynamically-scoped attribute for scope S: y in $S[-i]::y = 23;\n", + "$S[i]::y = 23;", "error(55): A.g:10:10: unknown dynamically-scoped attribute for scope S: y in $S[i]::y = 23;\n", + "$S[0]::y = 23;", "error(55): A.g:10:10: unknown dynamically-scoped attribute for scope S: y in $S[0]::y = 23;\n", + "$S[$S::y]::i", "error(55): A.g:10:10: unknown dynamically-scoped attribute for scope S: y in $S::y\n" + }; + + String[] dynFinallyChecks = { + "$a", "", + "$b", "", + "$lab", "", + "$b::f", "", + "$S", "", + "$S::i", "", + "$S::i=$S::i", "", + "$a::z", "", + + "$S::j", "error(55): A.g:12:17: unknown dynamically-scoped attribute for scope S: j in $S::j\n", + "$S::j = 3;", "error(55): A.g:12:17: unknown dynamically-scoped attribute for scope S: j in $S::j = 3;\n", + "$S::j = $S::k;", "error(55): A.g:12:17: unknown dynamically-scoped attribute for scope S: j in $S::j = $S::k;\n" + + "error(55): A.g:12:25: unknown dynamically-scoped attribute for scope S: k in $S::k\n", + }; + + @Test public void testMembersActions() throws RecognitionException { + testActions("members", membersChecks, attributeTemplate); + } + + @Test public void testInitActions() throws RecognitionException { + testActions("init", initChecks, attributeTemplate); + } + + @Test public void testInlineActions() throws RecognitionException { + testActions("inline", inlineChecks, attributeTemplate); + } + + @Test public void testBadInlineActions() throws RecognitionException { + testActions("inline", bad_inlineChecks, attributeTemplate); + } + + @Test public void testFinallyActions() throws RecognitionException { + testActions("finally", finallyChecks, attributeTemplate); + } + + @Test public void testDynMembersActions() throws RecognitionException { + testActions("members", dynMembersChecks, scopeTemplate); + } + + @Test public void testDynInitActions() throws RecognitionException { + testActions("init", dynInitChecks, scopeTemplate); + } + + @Test public void testDynInlineActions() throws RecognitionException { + testActions("inline", dynInlineChecks, scopeTemplate); + } + + @Test public void testDynFinallyActions() throws RecognitionException { + testActions("finally", dynFinallyChecks, scopeTemplate); + } + + @Test public void testTokenRef() throws RecognitionException { + String grammar = + "parser grammar S;\n" + + "a : x=ID {Token t = $x; t = $ID;} ;\n"; + String expected = + ""; + testErrors(new String[] {grammar, expected}, false); + } + + @Test public void testNonDynamicAttributeOutsideRule() throws Exception { + String action = "public void foo() { $x; }"; + } + @Test public void testNonDynamicAttributeOutsideRule2() throws Exception { + String action = "public void foo() { $x.y; }"; + } + @Test public void testUnknownGlobalScope() throws Exception { + String action = "$Symbols::names.add($id.text);"; + } + @Test public void testUnknownDynamicAttribute() throws Exception { + String action = "$a::x"; + } + + @Test public void testUnknownGlobalDynamicAttribute() throws Exception { + String action = "$Symbols::x"; + } + + + public void testActions(String location, String[] pairs, String template) { + for (int i = 0; i < pairs.length; i+=2) { + String action = pairs[i]; + String expected = pairs[i+1]; + ST st = new ST(template); + st.add(location, action); + String grammar = st.render(); + testErrors(new String[] {grammar, expected}, false); + } + } +} diff --git a/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java b/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java new file mode 100644 index 000000000..5ac16efc6 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java @@ -0,0 +1,103 @@ +package org.antlr.v4.test; + +import org.junit.Test; + +public class TestBasicSemanticErrors extends BaseTest { + static String[] A = { + // INPUT + "grammar A;\n" + + "\n" + + "options {\n" + + " output=template;\n" + + "}\n" + + "\n" + + "a : ID -> ID ;\n" + + "\n" + + "b : A^ | ((B!|C)) -> C;", + // YIELDS + "error(68): A.g:7:7: alts with rewrites can't use heterogeneous types left of ->\n" + + "error(78): A.g:9:4: AST operator with non-AST output option: ^\n" + + "error(78): A.g:9:11: AST operator with non-AST output option: !\n" + + "error(79): A.g:9:11: rule b alt 2 uses rewrite syntax and also an AST operator\n", + + // INPUT + "tree grammar B;\n" + + "options {\n" + + "\tfilter=true;\n" + + "\tbacktrack=false;\n" + + "\toutput=template;\n" + + "}\n" + + "\n" + + "a : A;\n" + + "\n" + + "b : ^(. A) ;", + // YIELDS + "error(80): B.g:10:6: Wildcard invalid as root; wildcard can itself be a tree\n" + + "error(81): B.g:1:5: option backtrack=false conflicts with tree grammar filter mode\n" + + "error(81): B.g:1:5: option output=template conflicts with tree grammar filter mode\n" + }; + + static String[] U = { + // INPUT + "parser grammar U;\n" + + "options { foo=bar; k=*; backtrack=true;}\n" + + "tokens {\n" + + " f='fkj';\n" + + " S = 'a';\n" + + "}\n" + + "tokens { A; }\n" + + "options { x=y; }\n" + + "\n" + + "a\n" + + "options { blech=bar; greedy=true; }\n" + + " : ID\n" + + " ;\n" + + "b : ( options { ick=bar; greedy=true; } : ID )+ ;\n" + + "c : ID ID ;", + // YIELDS + "error(21): U.g:8:0: repeated grammar prequel spec (option, token, or import); please merge\n" + + "error(21): U.g:7:0: repeated grammar prequel spec (option, token, or import); please merge\n" + + "error(49): U.g:2:10: illegal option foo\n" + + "error(26): U.g:4:8: token names must start with an uppercase letter: f\n" + + "error(25): U.g:4:8: can't assign string value to token name f in non-combined grammar\n" + + "error(25): U.g:5:8: can't assign string value to token name S in non-combined grammar\n" + + "error(49): U.g:8:10: illegal option x\n" + + "error(49): U.g:11:10: illegal option blech\n" + + "error(49): U.g:14:16: illegal option ick\n" + + "error(49): U.g:15:16: illegal option x\n", + + // INPUT + "tree grammar V;\n" + + "options {\n" + + " rewrite=true;\n" + + " output=template;\n" + + "}\n" + + "a : A\n" + + " | A B -> template() \"kjsfdkdsj\" \n" + + " ;", + // YIELDS + "error(66): V.g:7:4: with rewrite=true, alt 2 not simple node or obvious tree element; text attribute for rule not guaranteed to be correct\n", + + // INPUT + "tree grammar V;\n" + + "options { rewrite=true; }\n" + + "a : A\n" + + " | A B -> template() \"kjsfdkdsj\" \n" + + " ;", + // YIELDS + "error(62): V.g:4:8: rule a uses rewrite syntax or operator with no output option\n", + }; + + static String[] C = { + "parser grammar C;\n" + + "options {output=AST;}\n" + + "tokens { A; B; C; }\n" + + "a : A -> B $a A ;", // no problem with or $a. + + "" + }; + + @Test public void testA() { super.testErrors(A, false); } + @Test public void testU() { super.testErrors(U, false); } + @Test public void testE() { super.testErrors(C, false); } +} \ No newline at end of file diff --git a/tool/test/org/antlr/v4/test/TestLexerExec.java b/tool/test/org/antlr/v4/test/TestLexerExec.java new file mode 100644 index 000000000..91f41b998 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestLexerExec.java @@ -0,0 +1,70 @@ +package org.antlr.v4.test; + +import org.junit.Test; + +public class TestLexerExec extends BaseTest { + @Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "A : '-' I ;\n" + + "I : '0'..'9'+ ;\n"+ + "WS : (' '|'\\n') {skip();} ;"; + String found = execLexer("L.g", grammar, "L", "34 -21 3"); + String expecting = + "[@0,0:1='34',<4>,1:0]\n" + + "[@1,3:5='-21',<3>,1:3]\n" + + "[@2,7:7='3',<4>,1:7]\n" + + "[@3,8:8='',<-1>,1:8]\n"; + assertEquals(expecting, found); + } + + @Test public void testActionExecutedInDFA() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+ + "WS : (' '|'\\n') {skip();} ;"; + String found = execLexer("L.g", grammar, "L", "34 34"); + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<3>,1:0]\n" + + "[@1,3:4='34',<3>,1:3]\n" + + "[@2,5:5='',<-1>,1:5]\n"; + assertEquals(expecting, found); + } + + @Test public void testLexerMode() throws Exception { + String grammar = + "lexer grammar L;\n" + + "STRING_START : '\"' {pushMode(STRING_MODE); more();} ;\n" + + "WS : ' '|'\n' {skip();} ;\n"+ + "mode STRING_MODE;\n"+ + "STRING : '\"' {popMode();} ;\n"+ + "ANY : . {more();} ;\n"; + String found = execLexer("L.g", grammar, "L", "\"abc\" \"ab\""); + String expecting = + "[@0,0:4='\"abc\"',<5>,1:0]\n" + + "[@1,6:9='\"ab\"',<5>,1:6]\n" + + "[@2,10:10='',<-1>,1:10]\n"; + assertEquals(expecting, found); + } + + @Test public void testKeywordID() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "KEND : 'end' ;\n" + // has priority + "ID : 'a'..'z'+ ;\n" + + "WS : (' '|'\n')+ ;"; + String found = execLexer("L.g", grammar, "L", "end eend ending a"); + String expecting = + "[@0,0:2='end',<3>,1:0]\n" + + "[@1,3:3=' ',<5>,1:3]\n" + + "[@2,4:7='eend',<4>,1:4]\n" + + "[@3,8:8=' ',<5>,1:8]\n" + + "[@4,9:14='ending',<4>,1:9]\n" + + "[@5,15:15=' ',<5>,1:15]\n" + + "[@6,16:16='a',<4>,1:16]\n" + + "[@7,17:17='',<-1>,1:17]\n"; + assertEquals(expecting, found); + } +} diff --git a/tool/test/org/antlr/v4/test/TestParserExec.java b/tool/test/org/antlr/v4/test/TestParserExec.java new file mode 100644 index 000000000..21c5c8f5b --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestParserExec.java @@ -0,0 +1,20 @@ +package org.antlr.v4.test; + +import org.junit.Test; + +public class TestParserExec extends BaseTest { + + @Test public void testBasic() throws Exception { + String grammar = + "grammar T;\n" + + "a : ID INT {System.out.println(state.input);} ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "a", + "abc 34", false); + assertEquals("abc34\n", found); + } + +} diff --git a/tool/test/org/antlr/v4/test/TestScopeParsing.java b/tool/test/org/antlr/v4/test/TestScopeParsing.java new file mode 100644 index 000000000..d654d8a33 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestScopeParsing.java @@ -0,0 +1,56 @@ +package org.antlr.v4.test; + +import org.antlr.v4.parse.ScopeParser; +import org.junit.Test; + +public class TestScopeParsing extends BaseTest { + String[] argPairs = { + "", "{}", + " ", "{}", + "int i", "{i=int i}", + "int[] i, int j[]", "{i=int[] i, j=int [] j}", + "Map[] i, int j[]", "{i=Map[] i, j=int [] j}", + "int i = 34+a[3], int j[] = new int[34]", + "{i=int i= 34+a[3], j=int [] j= new int[34]}", + "char *foo32[3] = {1\\,2\\,3}", "{3=char *foo32[] 3= {1,2,3}}", + + // python/ruby style + "i", "{i=null i}", + "i,j", "{i=null i, j=null j}", + "i,j, k", "{i=null i, j=null j, k=null k}", + }; + + String[] scopePairs = { + "int i;", "{i=int i}", + "int[] i; int j[];", "{i=int[] i, j=int [] j}", + "Map[] i; int j[];", "{i=Map[] i, j=int [] j}", + "int i = 34+a[3]; int j[] = new int[34];", + "{i=int i= 34+a[3], j=int [] j= new int[34]}", + "char *foo32[] = {1,2,3};", "{foo32=char *[] foo32= {1,2,3}}", + " int i; int c; int k; ", "{i=int i, c=int c, k=int k}", + " { int i; int c; int k; }", "{i=int i, c=int c, k=int k}", + + // python/ruby style + "i", "{i=null i}", + " i ; j ;", "{i=null i, j=null j}", + "i; j; k;", "{i=null i, j=null j, k=null k}", + }; + + @Test public void testArgs() { + for (int i = 0; i < argPairs.length; i+=2) { + String input = argPairs[i]; + String expected = argPairs[i+1]; + String actual = ScopeParser.parseTypeList(input).attributes.toString(); + assertEquals(expected, actual); + } + } + + @Test public void testScopes() { + for (int i = 0; i < scopePairs.length; i+=2) { + String input = scopePairs[i]; + String expected = scopePairs[i+1]; + String actual = ScopeParser.parseDynamicScope(input).attributes.toString(); + assertEquals(expected, actual); + } + } +} diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java b/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java new file mode 100644 index 000000000..de516e257 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java @@ -0,0 +1,61 @@ +package org.antlr.v4.test; + +import org.junit.*; + +public class TestSemPredEvalLexer extends BaseTest { + + @Test public void testDisableRule() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "E1 : {false}? 'enum' ;\n" + + "E2 : {true}? 'enum' ;\n" + // winner not E1 or ID + "ID : 'a'..'z'+ ;\n"+ + "WS : (' '|'\\n') {skip();} ;"; + String found = execLexer("L.g", grammar, "L", "enum abc", true); + String expecting = + "[@0,0:3='enum',<4>,1:0]\n" + + "[@1,5:7='abc',<5>,1:5]\n" + + "[@2,8:8='',<-1>,1:8]\n"; // no dfa since preds on left edge + assertEquals(expecting, found); + } + + @Test public void testDisableRuleAfterMatch() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "E1 : 'enum' {false}? ;\n" + + "E2 : 'enum' {true}? ;\n" + // winner not E1 or ID + "ID : 'a'..'z'+ ;\n"+ + "WS : (' '|'\\n') {skip();} ;"; + String found = execLexer("L.g", grammar, "L", "enum abc enum", true); + String expecting = + "[@0,0:3='enum',<4>,1:0]\n" + + "[@1,5:7='abc',<5>,1:5]\n" + + "[@2,9:12='enum',<4>,1:9]\n" + + "[@3,13:13='',<-1>,1:13]\n" + + "s0-' '->:s4=>6\n" + + "s0-'a'->:s5=>5\n" + + "s0-'e'->:s1=>5\n" + + ":s1=>5-'n'->:s2=>5\n" + + ":s2=>5-'u'->:s3=>5\n" + + ":s5=>5-'b'->:s5=>5\n" + + ":s5=>5-'c'->:s5=>5\n"; + // didn't even created DFA 2nd time; old target of 'u' has "pred" flag set + assertEquals(expecting, found); + } + + @Ignore + public void testMatchNChar() throws Exception { // can't do locals yet + String grammar = + "lexer grammar L;\n"+ + "B : {int n=0;} ({n<=2}? DIGIT {n++})+ ;\n" + + "fragment DIGIT : '0'..'9' ;\n"+ + "WS : (' '|'\\n') {skip();} ;"; + String found = execLexer("L.g", grammar, "L", "1234 56", true); + String expecting = + "[@0,0:3='enum',<4>,1:0]\n" + + "[@1,5:7='abc',<5>,1:5]\n" + + "[@2,8:8='',<-1>,1:8]\n"; // no dfa since preds on left edge + assertEquals(expecting, found); + } + +} diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java new file mode 100644 index 000000000..b4ab9d858 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java @@ -0,0 +1,120 @@ +package org.antlr.v4.test; + +import org.junit.Test; + +public class TestSemPredEvalParser extends BaseTest { + @Test public void testToLeft() throws Exception { + String grammar = + "grammar T;\n" + + "s : a+ ;\n" + + "a : {false}? ID {System.out.println(\"alt 1\");}\n" + + " | {true}? ID {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "alt 2\n" + + "alt 2\n" + + "alt 2\n"; + assertEquals(expecting, found); + } + + @Test public void testToRight() throws Exception { + String grammar = + "grammar T;\n" + + "s : a+ ;\n" + + "a : ID {false}? {System.out.println(\"alt 1\");}\n" + + " | ID {true}? {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "alt 2\n" + + "alt 2\n" + + "alt 2\n"; + assertEquals(expecting, found); + } + + @Test public void testActionHidesPreds() throws Exception { + // can't see preds, resolves to first alt found (1 in this case) + String grammar = + "grammar T;\n" + + "@members {int i;}\n" + + "s : a+ ;\n" + + "a : {i=1;} ID {i==1}? {System.out.println(\"alt 1\");}\n" + + " | {i=2;} ID {i==2}? {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "alt 1\n" + + "alt 1\n" + + "alt 1\n"; + assertEquals(expecting, found); + } + + @Test public void testToLeftWithVaryingPredicate() throws Exception { + // alternate predicted alt to ensure DFA doesn't cache + // must use forced action since i++ must exec; FOLLOW(a) sees + // both preds since it loops around in s. + String grammar = + "grammar T;\n" + + "@members {int i=0;}\n" + + "s : ({i++; System.out.println(\"i=\"+i);} a)+ ;\n" + + "a : {i \\% 2 == 0}? ID {System.out.println(\"alt 1\");}\n" + + " | {i \\% 2 != 0}? ID {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "i=1\n" + + "alt 2\n" + + "i=2\n" + + "alt 1\n" + + "i=3\n" + + "alt 2\n"; + assertEquals(expecting, found); + } + + @Test public void testToRightWithVaryingPredicate() throws Exception { + // alternate predicted alt to ensure DFA doesn't cache + String grammar = + "grammar T;\n" + + "@members {int i=0;}\n" + + "s : ({i++; System.out.println(\"i=\"+i);} a)+ ;\n" + + "a : ID {i \\% 2 == 0}? {System.out.println(\"alt 1\");}\n" + + " | ID {i \\% 2 != 0}? {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "i=1\n" + + "alt 2\n" + + "i=2\n" + + "alt 1\n" + + "i=3\n" + + "alt 2\n"; + assertEquals(expecting, found); + } + +} diff --git a/tool/test/org/antlr/v4/test/TestSymbolIssues.java b/tool/test/org/antlr/v4/test/TestSymbolIssues.java new file mode 100644 index 000000000..1b86366f9 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestSymbolIssues.java @@ -0,0 +1,102 @@ +package org.antlr.v4.test; + +import org.junit.Test; + +/** */ +public class TestSymbolIssues extends BaseTest { + static String[] A = { + // INPUT + "grammar A;\n" + + "options { opt='sss'; k=3; }\n" + + "tokens { X; Y='y'; X='x'; Y='q'; Z; Z; }\n" + + "scope Blort { int x; }\n" + + "\n" + + "@members {foo}\n" + + "@members {bar}\n" + + "@lexer::header {package jj;}\n" + + "@lexer::header {package kk;}\n" + + "\n" + + "scope Blort { int x; }\n" + + "\n" + + "a[int i] returns [foo f] : X ID a[3] b[34] q ;\n" + + "b returns [int g] : Y 'y' 'if' a ;\n" + + "a : FJKD ;\n" + + "\n" + + "ID : 'a'..'z'+ ID ;", + // YIELDS + "error(49): A.g:2:10: illegal option opt\n" + + "error(59): A.g:11:6: scope Blort redefinition\n" + + "error(18): A.g:15:0: rule a redefinition\n" + + "error(58): A.g:7:1: redefinition of members action\n" + + "error(58): A.g:9:1: redefinition of header action\n" + + "error(72): A.g:3:19: cannot alias X; token name already defined\n" + + "error(72): A.g:3:26: cannot alias Y; token name already assigned to 'y'\n" + + "error(72): A.g:3:36: cannot alias Z; token name already defined\n" + + "error(46): A.g:13:37: rule b has no defined parameters\n" + + "error(23): A.g:13:43: reference to undefined rule: q\n" + + "error(45): A.g:14:31: missing parameter(s) on rule reference: a\n" + }; + + static String[] B = { + // INPUT + "parser grammar B;\n" + + "tokens { X='x'; Y; }\n" + + "scope s { int i; }\n" + + "\n" + + "a : s=ID b+=ID X=ID '.' ;\n" + + "\n" + + "b : x=ID x+=ID ;\n" + + "\n" + + "s : FOO ;", + // YIELDS + "error(25): B.g:2:9: can't assign string value to token name X in non-combined grammar\n" + + "error(34): B.g:9:0: symbol s conflicts with global dynamic scope with same name\n" + + "error(35): B.g:5:9: label b conflicts with rule with same name\n" + + "error(34): B.g:5:4: symbol s conflicts with global dynamic scope with same name\n" + + "error(36): B.g:5:15: label X conflicts with token with same name\n" + + "error(41): B.g:7:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n" + }; + + static String[] C = { + // INPUT + "grammar C;\n"+ + "options {output=AST;}\n"+ + "a : A x=b y+=Z 'hi' -> ID A r $foo $x b $y+ 'hi' 'eh?'\n"+ + " | ID -> $x A ID // shouldn't see these refs from other alt ('cept ID)\n"+ + " ;\n"+ + "b : B ;\n"+ + "A : 'a';", + // YIELDS + "error(51): C.g:3:28: reference to rewrite element r not found to left of ->\n" + + "error(51): C.g:3:30: reference to rewrite element foo not found to left of ->\n" + + "error(51): C.g:3:49: reference to rewrite element 'eh?' not found to left of ->\n" + + "error(51): C.g:4:10: reference to rewrite element x not found to left of ->\n" + }; + + static String[] D = { + // INPUT + "parser grammar D;\n" + + "a[int j] \n" + + "scope { int i; }\n" + + " : i=ID j=ID ;\n" + + "\n" + + "b[int i] returns [int i] : ID ;\n" + + "\n" + + "c[int i] returns [String k]\n" + + "scope { int i; int c; int k; }\n" + + " : ID ;", + + // YIELDS + "error(38): D.g:4:21: label j conflicts with rule a's return value or parameter with same name\n" + + "error(37): D.g:4:16: label i conflicts with rule a's dynamically-scoped attribute with same name\n" + + "error(42): D.g:6:0: rule b's argument i conflicts a return value with same name\n" + + "error(40): D.g:9:6: rule c's dynamically-scoped attribute i conflicts with c's return value or parameter with same name\n" + + "error(39): D.g:9:6: rule c's dynamically-scoped attribute c conflicts with the rule name\n" + + "error(40): D.g:9:6: rule c's dynamically-scoped attribute k conflicts with c's return value or parameter with same name\n" + }; + + @Test public void testA() { super.testErrors(A, false); } + @Test public void testB() { super.testErrors(B, false); } + @Test public void testC() { super.testErrors(C, false); } + @Test public void testD() { super.testErrors(D, false); } +} diff --git a/tool/test/org/antlr/v4/test/TestSyntaxErrors.java b/tool/test/org/antlr/v4/test/TestSyntaxErrors.java new file mode 100644 index 000000000..b1845e699 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestSyntaxErrors.java @@ -0,0 +1,143 @@ +package org.antlr.v4.test; + +import org.junit.Test; + +public class TestSyntaxErrors extends BaseTest { + static String[] A = { + // INPUT + "grammar A;\n" + + "", + // YIELDS + "error(63): A.g::: grammar A has no rules\n", + + "A;", + "error(17): :1:0: 'A' came as a complete surprise to me\n", + + "grammar ;", + "error(17): :1:8: ';' came as a complete surprise to me while looking for an identifier\n", + + "grammar A\n" + + "a : ID ;\n", + "error(17): :2:0: missing SEMI at 'a'\n", + + "grammar A;\n" + + "a : ID ;;\n"+ + "b : B ;", + "error(17): A.g:2:8: ';' came as a complete surprise to me\n", + + "grammar A;;\n" + + "a : ID ;\n", + "error(17): A;.g:1:10: ';' came as a complete surprise to me\n", + + "grammar A;\n" + + "a @init : ID ;\n", + "error(17): A.g:2:8: mismatched input ':' expecting ACTION while matching rule preamble\n", + + "grammar A;\n" + + "a ( A | B ) D ;\n" + + "b : B ;", + "error(17): A.g:2:3: '(' came as a complete surprise to me while matching rule preamble\n" + + "error(17): A.g:2:11: mismatched input ')' expecting SEMI while matching a rule\n" + + "error(17): A.g:2:15: ';' came as a complete surprise to me while matching rule preamble\n", + }; + + @Test public void testA() { super.testErrors(A, true); } + + @Test public void testExtraColon() { + String[] pair = new String[] { + "grammar A;\n" + + "a : : A ;\n" + + "b : B ;", + "error(17): A.g:2:4: ':' came as a complete surprise to me while matching alternative\n", + }; + super.testErrors(pair, true); + } + + @Test public void testMissingRuleSemi() { + String[] pair = new String[] { + "grammar A;\n" + + "a : A \n" + + "b : B ;", + "error(17): A.g:3:0: unterminated rule (missing ';') detected at 'b :' while looking for rule element\n", + }; + super.testErrors(pair, true); + } + + @Test public void testMissingRuleSemi2() { + String[] pair = new String[] { + "lexer grammar A;\n" + + "A : 'a' \n" + + "B : 'b' ;", + "error(17): A.g:3:0: unterminated rule (missing ';') detected at 'B :' while looking for rule element\n", + }; + super.testErrors(pair, true); + } + + @Test public void testMissingRuleSemi3() { + String[] pair = new String[] { + "grammar A;\n" + + "a : A \n" + + "b[int i] returns [int y] : B ;", + "error(17): A.g:3:9: unterminated rule (missing ';') detected at 'returns int y' while looking for rule element\n" + }; + super.testErrors(pair, true); + } + + @Test public void testMissingRuleSemi4() { + String[] pair = new String[] { + "grammar A;\n" + + "a : b \n" + + " catch [Exception e] {...}\n" + + "b : B ;\n", + + "error(17): A.g:2:4: unterminated rule (missing ';') detected at 'b catch' while looking for rule element\n" + }; + super.testErrors(pair, true); + } + + @Test public void testMissingRuleSemi5() { + String[] pair = new String[] { + "grammar A;\n" + + "a : A \n" + + " catch [Exception e] {...}\n", + + "error(17): A.g:2:4: unterminated rule (missing ';') detected at 'A catch' while looking for rule element\n" + }; + super.testErrors(pair, true); + } + + @Test public void testBadRulePrequelStart() { + String[] pair = new String[] { + "grammar A;\n" + + "a @ options {k=1;} : A ;\n" + + "b : B ;", + + "error(17): A.g:2:4: 'options {' came as a complete surprise to me while looking for an identifier\n" + }; + super.testErrors(pair, true); + } + + @Test public void testBadRulePrequelStart2() { + String[] pair = new String[] { + "grammar A;\n" + + "a } : A ;\n" + + "b : B ;", + + "error(17): A.g:2:2: '}' came as a complete surprise to me while matching rule preamble\n" + }; + super.testErrors(pair, true); + } + + @Test public void testModeInParser() { + String[] pair = new String[] { + "grammar A;\n" + + "a : A ;\n" + + "mode foo;\n" + + "b : B ;", + + "error(87): A.g:3:5: lexical modes are only allowed in lexer grammars\n" + }; + super.testErrors(pair, true); + } + +} diff --git a/tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java b/tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java new file mode 100644 index 000000000..1dba2d617 --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java @@ -0,0 +1,207 @@ +package org.antlr.v4.test; + +import org.antlr.v4.Tool; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.tool.*; +import org.junit.Test; + +import java.util.*; + +public class TestTokenTypeAssignment extends BaseTest { + + @Test + public void testParserSimpleTokens() throws Exception { + Grammar g = new Grammar( + "parser grammar t;\n"+ + "a : A | B;\n" + + "b : C ;"); + String rules = "a, b"; + String tokenNames = "A, B, C"; + checkSymbols(g, rules, tokenNames); + } + + @Test public void testParserTokensSection() throws Exception { + Grammar g = new Grammar( + "parser grammar t;\n" + + "tokens {\n" + + " C;\n" + + " D;" + + "}\n"+ + "a : A | B;\n" + + "b : C ;"); + String rules = "a, b"; + String tokenNames = "A, B, C, D"; + checkSymbols(g, rules, tokenNames); + } + + @Test public void testLexerTokensSection() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar t;\n" + + "tokens {\n" + + " C;\n" + + " D;" + + "}\n"+ + "A : 'a';\n" + + "C : 'c' ;"); + String rules = "A, C"; + String tokenNames = "A, C, D"; + checkSymbols(g, rules, tokenNames); + } + + @Test public void testTokensSectionWithAssignmentSection() throws Exception { + Grammar g = new Grammar( + "grammar t;\n" + + "tokens {\n" + + " C='c';\n" + + " D;" + + "}\n"+ + "a : A | B;\n" + + "b : C ;"); + String rules = "a, b"; + String tokenNames = "A, B, C, D, 'c'"; + checkSymbols(g, rules, tokenNames); + } + + @Test public void testCombinedGrammarLiterals() throws Exception { + Grammar g = new Grammar( + "grammar t;\n"+ + "a : 'begin' b 'end';\n" + + "b : C ';' ;\n" + + "ID : 'a' ;\n" + + "FOO : 'foo' ;\n" + // "foo" is not a token name + "C : 'c' ;\n"); // nor is 'c' + String rules = "a, b"; + String tokenNames = "C, FOO, ID, 'begin', 'end', ';'"; + checkSymbols(g, rules, tokenNames); + } + + @Test public void testLiteralInParserAndLexer() throws Exception { + // 'x' is token and char in lexer rule + Grammar g = new Grammar( + "grammar t;\n" + + "a : 'x' E ; \n" + + "E: 'x' '0' ;\n"); + + Tool antlr = new Tool(); + antlr.process(g); + + String literals = "['x']"; + String foundLiterals = g.stringLiteralToTypeMap.keySet().toString(); + assertEquals(literals, foundLiterals); + + foundLiterals = g.implicitLexer.stringLiteralToTypeMap.keySet().toString(); + assertEquals("['x']", foundLiterals); // pushed in lexer from parser + + String[] typeToTokenName = g.getTokenNames(); + Set tokens = new HashSet(); + for (String t : typeToTokenName) if ( t!=null ) tokens.add(t); + assertEquals("[E]", tokens.toString()); + } + + @Test public void testCombinedGrammarWithRefToLiteralButNoTokenIDRef() throws Exception { + Grammar g = new Grammar( + "grammar t;\n"+ + "a : 'a' ;\n" + + "A : 'a' ;\n"); + String rules = "a"; + String tokenNames = "A, 'a'"; + checkSymbols(g, rules, tokenNames); + } + + @Test public void testSetDoesNotMissTokenAliases() throws Exception { + Grammar g = new Grammar( + "grammar t;\n"+ + "a : 'a'|'b' ;\n" + + "A : 'a' ;\n" + + "B : 'b' ;\n"); + String rules = "a"; + String tokenNames = "A, 'a', B, 'b'"; + checkSymbols(g, rules, tokenNames); + } + + // T E S T L I T E R A L E S C A P E S + + @Test public void testParserCharLiteralWithEscape() throws Exception { + Grammar g = new Grammar( + "grammar t;\n"+ + "a : '\\n';\n"); + Tool antlr = new Tool(); + antlr.process(g); + Set literals = g.stringLiteralToTypeMap.keySet(); + // must store literals how they appear in the antlr grammar + assertEquals("'\\n'", literals.toArray()[0]); + } + + @Test public void testTokenInTokensSectionAndTokenRuleDef() throws Exception { + // this must return A not I to the parser; calling a nonfragment rule + // from a nonfragment rule does not set the overall token. + String grammar = + "grammar P;\n" + + "tokens { B='}'; }\n"+ + "a : A B {System.out.println(state.input);} ;\n"+ + "A : 'a' ;\n" + + "B : '}' ;\n"+ + "WS : (' '|'\\n') {skip();} ;"; + String found = execParser("P.g", grammar, "PParser", "PLexer", + "a", "a}", false); + assertEquals("a}\n", found); + } + + @Test public void testTokenInTokensSectionAndTokenRuleDef2() throws Exception { + // this must return A not I to the parser; calling a nonfragment rule + // from a nonfragment rule does not set the overall token. + String grammar = + "grammar P;\n" + + "tokens { B='}'; }\n"+ + "a : A '}' {System.out.println(state.input);} ;\n"+ + "A : 'a' ;\n" + + "B : '}' {/* */} ;\n"+ + "WS : (' '|'\\n') {skip();} ;"; + String found = execParser("P.g", grammar, "PParser", "PLexer", + "a", "a}", false); + assertEquals("a}\n", found); + } + + protected void checkSymbols(Grammar g, + String rulesStr, + String tokensStr) + throws Exception + { + Tool antlr = new Tool(); + antlr.process(g); + + String[] typeToTokenName = g.getTokenNames(); + Set tokens = new HashSet(); + for (String t : typeToTokenName) if ( t!=null ) tokens.add(t); + + // make sure expected tokens are there + StringTokenizer st = new StringTokenizer(tokensStr, ", "); + while ( st.hasMoreTokens() ) { + String tokenName = st.nextToken(); + assertTrue("token "+tokenName+" expected, but was undefined", + g.getTokenType(tokenName) != Token.INVALID_TYPE); + tokens.remove(tokenName); + } + // make sure there are not any others (other than etc...) + for (Iterator iter = tokens.iterator(); iter.hasNext();) { + String tokenName = (String) iter.next(); + assertTrue("unexpected token name "+tokenName, + g.getTokenType(tokenName) < Token.MIN_TOKEN_TYPE); + } + + // make sure all expected rules are there + st = new StringTokenizer(rulesStr, ", "); + int n = 0; + while ( st.hasMoreTokens() ) { + String ruleName = st.nextToken(); + assertNotNull("rule "+ruleName+" expected", g.getRule(ruleName)); + n++; + } + //System.out.println("rules="+rules); + // make sure there are no extra rules + assertEquals("number of rules mismatch; expecting "+n+"; found "+g.rules.size(), + n, g.rules.size()); + + } + +}