From b90b073450c71dacda2806bb490196ddf179dfc9 Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jul 2011 14:05:19 -0800 Subject: [PATCH] getting more unit tests in [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8924] --- .../v4/runtime/atn/ParserATNSimulator.java | 7 +- tool/playground/TestU.java | 2 +- .../antlr/v4/test/TestSemPredEvalParser.java | 279 ++++++++++++++++++ 3 files changed, 286 insertions(+), 2 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index b414b67dd..0ae179ec3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -501,7 +501,9 @@ public class ParserATNSimulator extends ATNSimulator { ", using context="+ctx); } int fromRuleIndex = config.state.ruleIndex; - if ( argIndex>=0 ) { + if ( argIndex>=0 && ctx!=null ) { + // we have an argument and we know its context or it's not depedent + /* // we're forced to exec args, even if dependent on _localctx. // If no actual context to use, create dummy context to use // for arg eval only. @@ -509,11 +511,14 @@ public class ParserATNSimulator extends ATNSimulator { // get dummy context for fromRuleIndex ctx = parser.newContext(null, config.state.stateNumber, fromRuleIndex); } + */ newContext = parser.newContext(ctx, t.target.stateNumber, fromRuleIndex, argIndex); } else { + // there is no argument or there is a dependent arg but + // we are unable to identify the proper context newContext = parser.newContext(ctx, t.target.stateNumber, targetRuleIndex); } diff --git a/tool/playground/TestU.java b/tool/playground/TestU.java index 09b09f26b..44809918a 100644 --- a/tool/playground/TestU.java +++ b/tool/playground/TestU.java @@ -34,7 +34,7 @@ public class TestU { ULexer t = new ULexer(new ANTLRFileStream(args[0])); CommonTokenStream tokens = new CommonTokenStream(t); UParser p = new UParser(tokens); - ParserRuleContext ret = p.a(33); + ParserRuleContext ret = p.s(); // System.out.println(((Tree)ret.tree).toStringTree()); } } diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java index b4ab9d858..8140955a4 100644 --- a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java @@ -117,4 +117,283 @@ public class TestSemPredEvalParser extends BaseTest { assertEquals(expecting, found); } + /** + * The ATN simulator ignores the unforced action but sees the forced + * actions at the start of both alternatives. Further, during closure, + * it sees the 2nd forced action s1b in the 1st alternative. Once parsing + * begins, it executes the "parse" action as well as the forced actions + * in the 1st alternative. + */ + @Test public void testForcedAction() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "void f(Object s) {System.out.println(s);}\n" + + "}\n" + + "s : {f(\"parse\");} {{f(\"s1a\");}} ID {{f(\"s1b\");}} | {{f(\"s2\");}} ID ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "s1a\n" + + "s2\n" + + "s1b\n" + + "parse\n" + + "s1a\n" + + "s1b\n"; + assertEquals(expecting, found); + } + + /** To distinguish the alternatives of rule e, we compute FOLLOW(e), + * which includes all tokens that can be matched following all + * references to e. In this case, it sees two forced actions after + * references to e that it must execute. + */ + @Test public void testForcedActionInGlobalFOLLOW() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "void f(Object s) {System.out.println(s);}\n" + + "}\n" + + "s : e {{f(\"s1\");}} {f(\"alt 1\");} '!' ;\n" + + "t : e {{f(\"t1\");}} ID ;\n" + + "e : ID | ;\n" + // non-LL(1) so we use ATN + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "a!", false); + String expecting = + "s1\n" + // do s1, t1 once during s0 computation from epsilon edge in e + "t1\n" + + "s1\n" + // do them again during closure after passing ID in e + "t1\n" + + "s1\n" + // now we are parsing + "alt 1\n"; + assertEquals(expecting, found); + } + + /** + * Actions that depend on local scope information such as local + * variables and arguments are executed if we are sure we have the + * right RuleContext object. Have the correct context for any rule + * that we invoke during ATN simulation (unless we fall off the edge + * of the initial rule into the outer context. see other unit + * tests). In this case, the forced actions are accessing the + * argument of the surrounding rule so everything is okay. + */ + @Test public void testForcedDepedentActionInContext() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "void f(Object s) {System.out.println(s);}\n" + + "}\n" + + "s : a[99] ;\n" + + "a[int i] : {f(\"parse\");} {{f($i);}} ID {{f(\"s1b\");}} | {{f($i+1);}} ID ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "99\n" + + "100\n" + + "s1b\n" + + "parse\n" + + "99\n" + + "s1b\n"; + assertEquals(expecting, found); + } + + /** In this case, we also know what the context is for $i. During + * ATN simulation, rule a invokes rule b which creates the correct context. + */ + @Test public void testForcedDepedentActionInContext2() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "void f(Object s) {System.out.println(s);}\n" + + "}\n" + + "s : a ;\n" + + "a : {f(\"parse\");} b {{f(\"s1b\");}} | b ;\n" + + "b returns [int i=32] : {{f($i);}} ID ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "32\n" + + "32\n" + + "s1b\n" + + "parse\n" + + "32\n" + + "s1b\n"; + assertEquals(expecting, found); + } + + /** We must execute all arguments for rules that we invoked during ATN + * simulation. In this case, the evaluation of the arguments is clearly + * okay because the arguments are not dependent on local context. + */ + @Test public void testArgEval() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "void f(Object s) {System.out.println(s);}\n" + + "}\n" + + "s : {f(\"parse\");} b[1] {{f(\"s1b\");}} | b[2] ;\n" + + "b[int i] : {{f($i);}} ID ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "1\n" + + "2\n" + + "s1b\n" + + "parse\n" + + "1\n" + + "s1b\n"; + assertEquals(expecting, found); + } + + /** The arguments to rule b access $i, which makes it dependent upon + * the local context. It's okay, because we are in the proper spot. + */ + @Test public void testDepedentArgEval() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "void f(Object s) {System.out.println(s);}\n" + + "}\n" + + "s : a[1] ;\n" + + "a[int i] : {f(\"parse\");} b[$i] {{f(\"s1b\");}} | b[$i+1] ;\n" + + "b[int i] : {{f($i);}} ID ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "1\n" + + "2\n" + + "s1b\n" + + "parse\n" + + "1\n" + + "s1b\n"; + assertEquals(expecting, found); + } + + /** The call chain is s a e but since e is optional, we invoke the + * global follow of e. This runs into the foo rule invocations, both of + * which have parameters that are dependent upon the local context. + * Because we can never be sure if we are in the proper context (without + * using full LL(*) context parsing), we can't ever execute dependent + * actions encountered during global follows. We create an uninitialized + * foo_ctx object for use while chasing edges in foo. + */ + @Test public void testDepedentArgEvalInGlobalFOLLOW() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "int f(int s) {System.out.println(s); return s;}\n" + + "}\n" + + "s : a[1] ;\n" + + "a[int i] : e foo[f($i)] ;\n" + + "b[int i] : e foo[f($i+1)] ;\n" + + "e : ID | ;\n" + + "foo[int k] : ID ';' ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x y;", false); + String expecting = + "1\n"; // f($i) executed only during real parse during call to foo. + assertEquals(expecting, found); + } + + @Test public void testForcedDepedentActionInGlobalFOLLOW() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "int f(int s) {System.out.println(s); return s;}\n" + + "}\n" + + "s : a[1] ;\n" + + "a[int i] : e {{f($i);}} ID '!' ;\n" + + "b[int i] : e {{f($i+1);}} ID '?' ;\n" + + "e : ID | ;\n" + + "foo[int k] : ID ';' ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x y !", false); + String expecting = + "1\n"; + assertEquals(expecting, found); + } + + @Test public void testPredsInGlobalFOLLOW() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "void f(Object s) {System.out.println(s);}\n" + + "}\n" + + "s : e {{f(\"s1\");}} {f(\"alt 1\");} '!' ;\n" + + "t : e {{f(\"t1\");}} ID ;\n" + + "e : ID | ;\n" + // non-LL(1) so we use ATN + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "a!", false); + String expecting = + "s1\n" + // do s1, t1 once during s0 computation from epsilon edge in e + "t1\n" + + "s1\n" + // do them again during closure after passing ID in e + "t1\n" + + "s1\n" + // now we are parsing + "alt 1\n"; + assertEquals(expecting, found); + } + + + @Test public void testDepedentPredsInGlobalFOLLOW() throws Exception { + String grammar = + "grammar T;\n" + + "@members {" + + "int f(int s) {System.out.println(s); return s;}\n" + + "}\n" + + "s : a[1] ;\n" + + "a[int i] : e {{f($i);}} ID '!' ;\n" + + "b[int i] : e {{f($i+1);}} ID '?' ;\n" + + "e : ID | ;\n" + + "foo[int k] : ID ';' ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x y !", false); + String expecting = + "1\n"; + assertEquals(expecting, found); + } + + + // "boolean p(boolean v) {System.out.println(\"eval=\"+v); return v;}\n" + }