From e97083fd81d53611caa7d8a72a59e53f26a3b559 Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Fri, 24 Oct 2014 22:15:05 +0800 Subject: [PATCH] validated SemPredEvalLexer --- .../org/antlr/v4/test/rt/gen/Generator.java | 18 +- .../antlr/v4/test/rt/gen/LexerTestMethod.java | 1 + .../gen/grammars/SemPredEvalLexer/Indent.st | 4 +- .../SemPredEvalLexer/PredicatedKeywords.st | 2 +- .../org/antlr/v4/test/rt/java/Java.test.stg | 8 +- .../v4/test/rt/java/TestCompositeLexers.java | 4 +- .../v4/test/rt/java/TestLexerErrors.java | 24 +-- .../antlr/v4/test/rt/java/TestLexerExec.java | 76 ++++---- .../v4/test/rt/java/TestSemPredEvalLexer.java | 28 +-- .../v4/test/tool/TestSemPredEvalLexer.java | 183 ------------------ 10 files changed, 86 insertions(+), 262 deletions(-) delete mode 100644 tool/test/org/antlr/v4/test/tool/TestSemPredEvalLexer.java diff --git a/tool/test/org/antlr/v4/test/rt/gen/Generator.java b/tool/test/org/antlr/v4/test/rt/gen/Generator.java index bb52d49f1..cb3f4c15f 100644 --- a/tool/test/org/antlr/v4/test/rt/gen/Generator.java +++ b/tool/test/org/antlr/v4/test/rt/gen/Generator.java @@ -382,7 +382,7 @@ public class Generator { private TestFile buildSemPredEvalLexer() throws Exception { TestFile file = new TestFile("SemPredEvalLexer"); - file.addLexerTest(input, "DisableRule", "L", + LexerTestMethod tm = file.addLexerTest(input, "DisableRule", "L", "enum abc", "[@0,0:3='enum',<2>,1:0]\n" + "[@1,5:7='abc',<3>,1:5]\n" + @@ -395,7 +395,8 @@ public class Generator { ":s6=>3-'b'->:s6=>3\n" + ":s6=>3-'c'->:s6=>3\n", null); - file.addLexerTest(input, "IDvsEnum", "L", + tm.showDFA = true; + tm = file.addLexerTest(input, "IDvsEnum", "L", "enum abc enum", "[@0,0:3='enum',<2>,1:0]\n" + "[@1,5:7='abc',<2>,1:5]\n" + @@ -409,7 +410,8 @@ public class Generator { ":s4=>2-'b'->:s4=>2\n" + ":s4=>2-'c'->:s4=>2\n", // no 'm'-> transition...conflicts with pred null); - file.addLexerTest(input, "IDnotEnum", "L", + tm.showDFA = true; + tm = file.addLexerTest(input, "IDnotEnum", "L", "enum abc enum", "[@0,0:3='enum',<2>,1:0]\n" + "[@1,5:7='abc',<2>,1:5]\n" + @@ -417,7 +419,8 @@ public class Generator { "[@3,13:12='',<-1>,1:13]\n" + "s0-' '->:s2=>3\n", // no edges in DFA for enum/id. all paths lead to pred. null); - file.addLexerTest(input, "EnumNotID", "L", + tm.showDFA = true; + tm = file.addLexerTest(input, "EnumNotID", "L", "enum abc enum", "[@0,0:3='enum',<1>,1:0]\n" + "[@1,5:7='abc',<2>,1:5]\n" + @@ -425,7 +428,8 @@ public class Generator { "[@3,13:12='',<-1>,1:13]\n" + "s0-' '->:s3=>3\n", // no edges in DFA for enum/id. all paths lead to pred. null); - file.addLexerTest(input, "Indent", "L", + tm.showDFA = true; + tm = file.addLexerTest(input, "Indent", "L", "abc\n def \n", "INDENT\n" + // action output "[@0,0:2='abc',<1>,1:0]\n" + // ID @@ -444,7 +448,8 @@ public class Generator { ":s1=>1-'e'->:s1=>1\n" + ":s1=>1-'f'->:s1=>1\n", null); - file.addLexerTest(input, "LexerInputPositionSensitivePredicates", "L", + tm.showDFA = true; + tm = file.addLexerTest(input, "LexerInputPositionSensitivePredicates", "L", "a cde\nabcde\n", "a\n" + "cde\n" + @@ -456,6 +461,7 @@ public class Generator { "[@3,8:10='cde',<2>,2:2]\n" + "[@4,12:11='',<-1>,3:0]\n", null); + tm.showDFA = true; file.addLexerTest(input, "PredicatedKeywords", "L", "enum enu a", "enum!\n" + diff --git a/tool/test/org/antlr/v4/test/rt/gen/LexerTestMethod.java b/tool/test/org/antlr/v4/test/rt/gen/LexerTestMethod.java index 8d378891d..4de133d58 100644 --- a/tool/test/org/antlr/v4/test/rt/gen/LexerTestMethod.java +++ b/tool/test/org/antlr/v4/test/rt/gen/LexerTestMethod.java @@ -4,6 +4,7 @@ public class LexerTestMethod extends TestMethod { public String[] outputLines; public boolean lexerOnly = true; + public boolean showDFA = false; public LexerTestMethod(String name, String grammarName, String input, String expectedOutput, String expectedErrors, Integer index) { diff --git a/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/Indent.st b/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/Indent.st index 36c452401..9aa936553 100644 --- a/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/Indent.st +++ b/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/Indent.st @@ -1,6 +1,6 @@ lexer grammar ; ID : [a-z]+ ; -INDENT : [ \t]+ { }? \n" + - { } ;"+ +INDENT : [ \t]+ { }? + { } ; NL : '\n'; WS : [ \t]+ ; diff --git a/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/PredicatedKeywords.st b/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/PredicatedKeywords.st index 529601cb6..2f3127cd5 100644 --- a/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/PredicatedKeywords.st +++ b/tool/test/org/antlr/v4/test/rt/gen/grammars/SemPredEvalLexer/PredicatedKeywords.st @@ -1,4 +1,4 @@ lexer grammar ; ENUM : [a-z]+ { }? { } ; -ID : [a-z]+ { } ; +ID : [a-z]+ { } ; WS : [ \n] -> skip ; diff --git a/tool/test/org/antlr/v4/test/rt/java/Java.test.stg b/tool/test/org/antlr/v4/test/rt/java/Java.test.stg index bb3ed5111..ce1a3f775 100644 --- a/tool/test/org/antlr/v4/test/rt/java/Java.test.stg +++ b/tool/test/org/antlr/v4/test/rt/java/Java.test.stg @@ -28,7 +28,7 @@ public void test() throws Exception { };separator="\n", wrap, anchor> String grammar = };separator="\\n\" +\n", wrap, anchor>"; - String found = execLexer(".g4", grammar, "Lexer", ""); + String found = execLexer(".g4", grammar, "Lexer", "", ); assertEquals(\\n"};separator=" + \n", wrap, anchor>, found); assertEquals("", this.stderrDuringParse); @@ -125,13 +125,13 @@ BailErrorStrategy() ::= <%this._errHandler = new antlr4.error.BailErrorStrategy( ToStringTree(s) ::= <%.toStringTree(null, this);%> -Column() ::= "this.column" +Column() ::= "this.getCharPositionInLine()" Text() ::= "this.getText()" ValEquals(a,b) ::= <%===%> -TextEquals(a) ::= <%this.text===""%> +TextEquals(a) ::= <%this.getText().equals("")%> PlusText(a) ::= <%"" + this.getText()%> @@ -141,7 +141,7 @@ LTEquals(i, v) ::= <%this._input.LT().text===%> LANotEquals(i, v) ::= <%this._input.LA()!=%> -TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===%> +TokenStartColumnEquals(i) ::= <%this._tokenStartCharPositionInLine==%> ImportListener(X) ::= <Listener = require('./Listener').Listener;>> diff --git a/tool/test/org/antlr/v4/test/rt/java/TestCompositeLexers.java b/tool/test/org/antlr/v4/test/rt/java/TestCompositeLexers.java index bc0343ea0..375f38aad 100644 --- a/tool/test/org/antlr/v4/test/rt/java/TestCompositeLexers.java +++ b/tool/test/org/antlr/v4/test/rt/java/TestCompositeLexers.java @@ -17,7 +17,7 @@ public class TestCompositeLexers extends BaseTest { "import S;\n" + "B : 'b';\n" + "WS : (' '|'\\n') -> skip ;"; - String found = execLexer("M.g4", grammar, "M", "abc"); + String found = execLexer("M.g4", grammar, "M", "abc", false); assertEquals("S.A\n" + "[@0,0:0='a',<3>,1:0]\n" + "[@1,1:1='b',<1>,1:1]\n" + @@ -38,7 +38,7 @@ public class TestCompositeLexers extends BaseTest { "import S;\n" + "A : 'a' B {document.getElementById('output').value += \"M.A\";};\n" + "WS : (' '|'\\n') -> skip ;"; - String found = execLexer("M.g4", grammar, "M", "ab"); + String found = execLexer("M.g4", grammar, "M", "ab", false); assertEquals("M.A\n" + "[@0,0:1='ab',<1>,1:0]\n" + "[@1,2:1='',<-1>,1:2]\n", found); diff --git a/tool/test/org/antlr/v4/test/rt/java/TestLexerErrors.java b/tool/test/org/antlr/v4/test/rt/java/TestLexerErrors.java index 112ce021b..890719e13 100644 --- a/tool/test/org/antlr/v4/test/rt/java/TestLexerErrors.java +++ b/tool/test/org/antlr/v4/test/rt/java/TestLexerErrors.java @@ -9,7 +9,7 @@ public class TestLexerErrors extends BaseTest { public void testInvalidCharAtStart() throws Exception { String grammar = "lexer grammar L;\n" + "A : 'a' 'b' ;"; - String found = execLexer("L.g4", grammar, "L", "x"); + String found = execLexer("L.g4", grammar, "L", "x", false); assertEquals("[@0,1:0='',<-1>,1:1]\n", found); assertEquals("line 1:0 token recognition error at: 'x'\n", this.stderrDuringParse); } @@ -20,7 +20,7 @@ public class TestLexerErrors extends BaseTest { "ACTION2 : '[' (STRING | ~'\"')*? ']';\n" + "STRING : '\"' ('\\\"' | .)*? '\"';\n" + "WS : [ \\t\\r\\n]+ -> skip;"; - String found = execLexer("L.g4", grammar, "L", "[\"foo\"]"); + String found = execLexer("L.g4", grammar, "L", "[\"foo\"]", false); assertEquals("[@0,0:6='[\"foo\"]',<1>,1:0]\n" + "[@1,7:6='',<-1>,1:7]\n", found); assertNull(this.stderrDuringParse); @@ -32,7 +32,7 @@ public class TestLexerErrors extends BaseTest { "ACTION2 : '[' (STRING | ~'\"')*? ']';\n" + "STRING : '\"' ('\\\"' | .)*? '\"';\n" + "WS : [ \\t\\r\\n]+ -> skip;"; - String found = execLexer("L.g4", grammar, "L", "[\"foo]"); + String found = execLexer("L.g4", grammar, "L", "[\"foo]", false); assertEquals("[@0,6:5='',<-1>,1:6]\n", found); assertEquals("line 1:0 token recognition error at: '[\"foo]'\n", this.stderrDuringParse); } @@ -42,7 +42,7 @@ public class TestLexerErrors extends BaseTest { String grammar = "lexer grammar L;\n" + "ACTION : '{' (ACTION | ~[{}])* '}';\n" + "WS : [ \\r\\n\\t]+ -> skip;"; - String found = execLexer("L.g4", grammar, "L", "{ { } }"); + String found = execLexer("L.g4", grammar, "L", "{ { } }", false); assertEquals("[@0,0:6='{ { } }',<1>,1:0]\n" + "[@1,7:6='',<-1>,1:7]\n", found); assertNull(this.stderrDuringParse); @@ -53,7 +53,7 @@ public class TestLexerErrors extends BaseTest { String grammar = "lexer grammar L;\n" + "ACTION : '{' (ACTION | ~[{}])* '}';\n" + "WS : [ \\r\\n\\t]+ -> skip;"; - String found = execLexer("L.g4", grammar, "L", "{ { }"); + String found = execLexer("L.g4", grammar, "L", "{ { }", false); assertEquals("[@0,5:4='',<-1>,1:5]\n", found); assertEquals("line 1:0 token recognition error at: '{ { }'\n", this.stderrDuringParse); } @@ -62,7 +62,7 @@ public class TestLexerErrors extends BaseTest { public void testInvalidCharAtStartAfterDFACache() throws Exception { String grammar = "lexer grammar L;\n" + "A : 'a' 'b' ;"; - String found = execLexer("L.g4", grammar, "L", "abx"); + String found = execLexer("L.g4", grammar, "L", "abx", false); assertEquals("[@0,0:1='ab',<1>,1:0]\n" + "[@1,3:2='',<-1>,1:3]\n", found); assertEquals("line 1:2 token recognition error at: 'x'\n", this.stderrDuringParse); @@ -72,7 +72,7 @@ public class TestLexerErrors extends BaseTest { public void testInvalidCharInToken() throws Exception { String grammar = "lexer grammar L;\n" + "A : 'a' 'b' ;"; - String found = execLexer("L.g4", grammar, "L", "ax"); + String found = execLexer("L.g4", grammar, "L", "ax", false); assertEquals("[@0,2:1='',<-1>,1:2]\n", found); assertEquals("line 1:0 token recognition error at: 'ax'\n", this.stderrDuringParse); } @@ -81,7 +81,7 @@ public class TestLexerErrors extends BaseTest { public void testInvalidCharInTokenAfterDFACache() throws Exception { String grammar = "lexer grammar L;\n" + "A : 'a' 'b' ;"; - String found = execLexer("L.g4", grammar, "L", "abax"); + String found = execLexer("L.g4", grammar, "L", "abax", false); assertEquals("[@0,0:1='ab',<1>,1:0]\n" + "[@1,4:3='',<-1>,1:4]\n", found); assertEquals("line 1:2 token recognition error at: 'ax'\n", this.stderrDuringParse); @@ -92,7 +92,7 @@ public class TestLexerErrors extends BaseTest { String grammar = "lexer grammar L;\n" + "A : 'ab' ;\n" + "B : 'abc' ;"; - String found = execLexer("L.g4", grammar, "L", "ababx"); + String found = execLexer("L.g4", grammar, "L", "ababx", false); assertEquals("[@0,0:1='ab',<1>,1:0]\n" + "[@1,2:3='ab',<1>,1:2]\n" + "[@2,5:4='',<-1>,1:5]\n", found); @@ -105,7 +105,7 @@ public class TestLexerErrors extends BaseTest { "A : 'ab' ;\n" + "B : 'abc' ;\n" + "C : 'abcd' ;"; - String found = execLexer("L.g4", grammar, "L", "ababcx"); + String found = execLexer("L.g4", grammar, "L", "ababcx", false); assertEquals("[@0,0:1='ab',<1>,1:0]\n" + "[@1,2:4='abc',<2>,1:2]\n" + "[@2,6:5='',<-1>,1:6]\n", found); @@ -116,7 +116,7 @@ public class TestLexerErrors extends BaseTest { public void testErrorInMiddle() throws Exception { String grammar = "lexer grammar L;\n" + "A : 'abc' ;"; - String found = execLexer("L.g4", grammar, "L", "abx"); + String found = execLexer("L.g4", grammar, "L", "abx", false); assertEquals("[@0,3:2='',<-1>,1:3]\n", found); assertEquals("line 1:0 token recognition error at: 'abx'\n", this.stderrDuringParse); } @@ -128,7 +128,7 @@ public class TestLexerErrors extends BaseTest { "expr : primary expr? {} | expr '->' ID;\n" + "primary : ID;\n" + "ID : [a-z]+;"; - String found = execLexer("L.g4", grammar, "LLexer", "x : x"); + String found = execLexer("L.g4", grammar, "LLexer", "x : x", false); assertEquals("[@0,0:0='x',<3>,1:0]\n" + "[@1,2:2=':',<1>,1:2]\n" + "[@2,4:4='x',<3>,1:4]\n" + diff --git a/tool/test/org/antlr/v4/test/rt/java/TestLexerExec.java b/tool/test/org/antlr/v4/test/rt/java/TestLexerExec.java index 9e33810c5..51114b219 100644 --- a/tool/test/org/antlr/v4/test/rt/java/TestLexerExec.java +++ b/tool/test/org/antlr/v4/test/rt/java/TestLexerExec.java @@ -9,7 +9,7 @@ public class TestLexerExec extends BaseTest { public void testQuoteTranslation() throws Exception { String grammar = "lexer grammar L;\n" + "QUOTE : '\"' ; // make sure this compiles"; - String found = execLexer("L.g4", grammar, "L", "\""); + String found = execLexer("L.g4", grammar, "L", "\"", false); assertEquals("[@0,0:0='\"',<1>,1:0]\n" + "[@1,1:0='',<-1>,1:1]\n", found); assertNull(this.stderrDuringParse); @@ -21,7 +21,7 @@ public class TestLexerExec extends BaseTest { "A : '-' I ;\n" + "I : '0'..'9'+ ;\n" + "WS : (' '|'\\n') -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "34 -21 3"); + String found = execLexer("L.g4", grammar, "L", "34 -21 3", false); assertEquals("[@0,0:1='34',<2>,1:0]\n" + "[@1,3:5='-21',<1>,1:3]\n" + "[@2,7:7='3',<2>,1:7]\n" + @@ -37,7 +37,7 @@ public class TestLexerExec extends BaseTest { "Vee : '\\\\/';\n" + "Wedge : '/\\\\';\n" + "WS : [ \\t] -> skip;"; - String found = execLexer("L.g4", grammar, "L", "\\ / \\/ /\\"); + String found = execLexer("L.g4", grammar, "L", "\\ / \\/ /\\", false); assertEquals("[@0,0:0='\\',<1>,1:0]\n" + "[@1,2:2='/',<2>,1:2]\n" + "[@2,4:5='\\/',<3>,1:4]\n" + @@ -55,7 +55,7 @@ public class TestLexerExec extends BaseTest { "fragment L_A: '.-';\n" + "fragment L_K: '-.-';\n" + "SEPARATOR: '!';"; - String found = execLexer("L.g4", grammar, "L", "-.-.-!"); + String found = execLexer("L.g4", grammar, "L", "-.-.-!", false); assertEquals("[@0,0:4='-.-.-',<1>,1:0]\n" + "[@1,5:5='!',<3>,1:5]\n" + "[@2,6:5='',<-1>,1:6]\n", found); @@ -66,7 +66,7 @@ public class TestLexerExec extends BaseTest { public void testNonGreedyTermination1() throws Exception { String grammar = "lexer grammar L;\n" + "STRING : '\"' ('\"\"' | .)*? '\"';"; - String found = execLexer("L.g4", grammar, "L", "\"hi\"\"mom\""); + String found = execLexer("L.g4", grammar, "L", "\"hi\"\"mom\"", false); assertEquals("[@0,0:3='\"hi\"',<1>,1:0]\n" + "[@1,4:8='\"mom\"',<1>,1:4]\n" + "[@2,9:8='',<-1>,1:9]\n", found); @@ -77,7 +77,7 @@ public class TestLexerExec extends BaseTest { public void testNonGreedyTermination2() throws Exception { String grammar = "lexer grammar L;\n" + "STRING : '\"' ('\"\"' | .)+? '\"';"; - String found = execLexer("L.g4", grammar, "L", "\"\"\"mom\""); + String found = execLexer("L.g4", grammar, "L", "\"\"\"mom\"", false); assertEquals("[@0,0:6='\"\"\"mom\"',<1>,1:0]\n" + "[@1,7:6='',<-1>,1:7]\n", found); assertNull(this.stderrDuringParse); @@ -88,7 +88,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '//' .*? '\\n' CMT?;\n" + "WS : (' '|'\\t')+;"; - String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); + String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n", false); assertEquals("[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" + "[@1,14:13='',<-1>,3:14]\n", found); assertNull(this.stderrDuringParse); @@ -99,7 +99,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '//' .*? '\\n' CMT??;\n" + "WS : (' '|'\\t')+;"; - String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); + String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n", false); assertEquals("[@0,0:6='//blah\\n',<1>,1:0]\n" + "[@1,7:13='//blah\\n',<1>,2:0]\n" + "[@2,14:13='',<-1>,3:7]\n", found); @@ -111,7 +111,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '//' .*? '\\n' CMT*;\n" + "WS : (' '|'\\t')+;"; - String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); + String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n", false); assertEquals("[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" + "[@1,14:13='',<-1>,3:14]\n", found); assertNull(this.stderrDuringParse); @@ -122,7 +122,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '//' .*? '\\n' CMT*?;\n" + "WS : (' '|'\\t')+;"; - String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); + String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n", false); assertEquals("[@0,0:6='//blah\\n',<1>,1:0]\n" + "[@1,7:13='//blah\\n',<1>,2:0]\n" + "[@2,14:13='',<-1>,3:7]\n", found); @@ -134,7 +134,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : ('//' .*? '\\n')+;\n" + "WS : (' '|'\\t')+;"; - String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); + String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n", false); assertEquals("[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" + "[@1,14:13='',<-1>,3:14]\n", found); assertNull(this.stderrDuringParse); @@ -145,7 +145,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : ('//' .*? '\\n')+?;\n" + "WS : (' '|'\\t')+;"; - String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); + String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n", false); assertEquals("[@0,0:6='//blah\\n',<1>,1:0]\n" + "[@1,7:13='//blah\\n',<1>,2:0]\n" + "[@2,14:13='',<-1>,3:7]\n", found); @@ -157,7 +157,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '/*' (CMT | .)*? '*/' ;\n" + "WS : (' '|'\\n')+;"; - String found = execLexer("L.g4", grammar, "L", "/* ick */\n/* /* */\n/* /*nested*/ */\n"); + String found = execLexer("L.g4", grammar, "L", "/* ick */\n/* /* */\n/* /*nested*/ */\n", false); assertEquals("[@0,0:8='/* ick */',<1>,1:0]\n" + "[@1,9:9='\\n',<2>,1:9]\n" + "[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" + @@ -171,7 +171,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '/*' (CMT | .)*? '*/' ;\n" + "WS : (' '|'\\n')+;"; - String found = execLexer("L.g4", grammar, "L", "/* ick */x\n/* /* */x\n/* /*nested*/ */x\n"); + String found = execLexer("L.g4", grammar, "L", "/* ick */x\n/* /* */x\n/* /*nested*/ */x\n", false); assertEquals("[@0,0:8='/* ick */',<1>,1:0]\n" + "[@1,10:10='\\n',<2>,1:10]\n" + "[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" + @@ -185,7 +185,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '/*' (CMT | .)+? '*/' ;\n" + "WS : (' '|'\\n')+;"; - String found = execLexer("L.g4", grammar, "L", "/* ick */\n/* /* */\n/* /*nested*/ */\n"); + String found = execLexer("L.g4", grammar, "L", "/* ick */\n/* /* */\n/* /*nested*/ */\n", false); assertEquals("[@0,0:8='/* ick */',<1>,1:0]\n" + "[@1,9:9='\\n',<2>,1:9]\n" + "[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" + @@ -199,7 +199,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "CMT : '/*' (CMT | .)+? '*/' ;\n" + "WS : (' '|'\\n')+;"; - String found = execLexer("L.g4", grammar, "L", "/* ick */x\n/* /* */x\n/* /*nested*/ */x\n"); + String found = execLexer("L.g4", grammar, "L", "/* ick */x\n/* /* */x\n/* /*nested*/ */x\n", false); assertEquals("[@0,0:8='/* ick */',<1>,1:0]\n" + "[@1,10:10='\\n',<2>,1:10]\n" + "[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" + @@ -218,7 +218,7 @@ public class TestLexerExec extends BaseTest { " {System.out.println(this.getText());} ;\n" + "WS : (' '|'\\n') -> skip ;\n" + "J : .;"; - String found = execLexer("L.g4", grammar, "L", "ab"); + String found = execLexer("L.g4", grammar, "L", "ab", false); assertEquals("stuff0: \n" + "stuff1: a\n" + "stuff2: ab\n" + @@ -234,7 +234,7 @@ public class TestLexerExec extends BaseTest { "I : ('a' | 'ab') {System.out.println(this.getText());} ;\n" + "WS : (' '|'\\n') -> skip ;\n" + "J : .;"; - String found = execLexer("L.g4", grammar, "L", "ab"); + String found = execLexer("L.g4", grammar, "L", "ab", false); assertEquals("ab\n" + "[@0,0:1='ab',<1>,1:0]\n" + "[@1,2:1='',<-1>,1:2]\n", found); @@ -247,7 +247,7 @@ public class TestLexerExec extends BaseTest { "I : .*? ('a' | 'ab') {System.out.println(this.getText());} ;\n" + "WS : (' '|'\\n') -> skip ;\n" + "J : . {System.out.println(this.getText());};"; - String found = execLexer("L.g4", grammar, "L", "ab"); + String found = execLexer("L.g4", grammar, "L", "ab", false); assertEquals("a\n" + "b\n" + "[@0,0:0='a',<1>,1:0]\n" + @@ -262,7 +262,7 @@ public class TestLexerExec extends BaseTest { "KEND : 'end' ; // has priority\n" + "ID : 'a'..'z'+ ;\n" + "WS : (' '|'\\n')+;"; - String found = execLexer("L.g4", grammar, "L", "end eend ending a"); + String found = execLexer("L.g4", grammar, "L", "end eend ending a", false); assertEquals("[@0,0:2='end',<1>,1:0]\n" + "[@1,3:3=' ',<3>,1:3]\n" + "[@2,4:7='eend',<2>,1:4]\n" + @@ -284,7 +284,7 @@ public class TestLexerExec extends BaseTest { "ID : 'a'..'z'+ ;\n" + "fragment HexDigit : ('0'..'9'|'a'..'f'|'A'..'F') ;\n" + "WS : (' '|'\\n')+;"; - String found = execLexer("L.g4", grammar, "L", "x 0 1 a.b a.l"); + String found = execLexer("L.g4", grammar, "L", "x 0 1 a.b a.l", false); assertEquals("[@0,0:0='x',<5>,1:0]\n" + "[@1,1:1=' ',<6>,1:1]\n" + "[@2,2:2='0',<2>,1:2]\n" + @@ -307,7 +307,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "DONE : EOF ;\n" + "A : 'a';"; - String found = execLexer("L.g4", grammar, "L", ""); + String found = execLexer("L.g4", grammar, "L", "", false); assertEquals("[@0,0:-1='',<1>,1:0]\n" + "[@1,0:-1='',<-1>,1:0]\n", found); assertNull(this.stderrDuringParse); @@ -319,7 +319,7 @@ public class TestLexerExec extends BaseTest { "A : 'a' EOF ;\n" + "B : 'a';\n" + "C : 'c';"; - String found = execLexer("L.g4", grammar, "L", ""); + String found = execLexer("L.g4", grammar, "L", "", false); assertEquals("[@0,0:-1='',<-1>,1:0]\n", found); assertNull(this.stderrDuringParse); } @@ -330,7 +330,7 @@ public class TestLexerExec extends BaseTest { "A : 'a' EOF ;\n" + "B : 'a';\n" + "C : 'c';"; - String found = execLexer("L.g4", grammar, "L", "a"); + String found = execLexer("L.g4", grammar, "L", "a", false); assertEquals("[@0,0:0='a',<1>,1:0]\n" + "[@1,1:0='',<-1>,1:1]\n", found); assertNull(this.stderrDuringParse); @@ -341,7 +341,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "I : '0'..'9'+ {System.out.println(\"I\");} ;\n" + "WS : [ \\n\\u000D] -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "34\r\n 34"); + String found = execLexer("L.g4", grammar, "L", "34\r\n 34", false); assertEquals("I\n" + "I\n" + "[@0,0:1='34',<1>,1:0]\n" + @@ -355,7 +355,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "I : '0'..'9'+ {System.out.println(\"I\");} ;\n" + "WS : [ \\n\\u000D]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "34\r\n 34"); + String found = execLexer("L.g4", grammar, "L", "34\r\n 34", false); assertEquals("I\n" + "I\n" + "[@0,0:1='34',<1>,1:0]\n" + @@ -369,7 +369,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "I : ~[ab \\n] ~[ \\ncd]* {System.out.println(\"I\");} ;\n" + "WS : [ \\n\\u000D]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "xaf"); + String found = execLexer("L.g4", grammar, "L", "xaf", false); assertEquals("I\n" + "[@0,0:2='xaf',<1>,1:0]\n" + "[@1,3:2='',<-1>,1:3]\n", found); @@ -382,7 +382,7 @@ public class TestLexerExec extends BaseTest { "I : (~[ab \\n]|'a') {System.out.println(\"I\");} ;\n" + "WS : [ \\n\\u000D]+ -> skip ;\n" + " "; - String found = execLexer("L.g4", grammar, "L", "a x"); + String found = execLexer("L.g4", grammar, "L", "a x", false); assertEquals("I\n" + "I\n" + "[@0,0:0='a',<1>,1:0]\n" + @@ -397,7 +397,7 @@ public class TestLexerExec extends BaseTest { "I : [0-9]+ {System.out.println(\"I\");} ;\n" + "ID : [a-zA-Z] [a-zA-Z0-9]* {System.out.println(\"ID\");} ;\n" + "WS : [ \\n\\u0009\\r]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "34\r 34 a2 abc \n "); + String found = execLexer("L.g4", grammar, "L", "34\r 34 a2 abc \n ", false); assertEquals("I\n" + "I\n" + "ID\n" + @@ -415,7 +415,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "I : [0-]+ {System.out.println(\"I\");} ;\n" + "WS : [ \\n\\u000D]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "00\r\n"); + String found = execLexer("L.g4", grammar, "L", "00\r\n", false); assertEquals("I\n" + "[@0,0:1='00',<1>,1:0]\n" + "[@1,4:3='',<-1>,2:0]\n", found); @@ -427,7 +427,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "I : [0-9]+ {System.out.println(\"I\");} ;\n" + "WS : [ \\u]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "34 "); + String found = execLexer("L.g4", grammar, "L", "34 ", false); assertEquals("I\n" + "[@0,0:1='34',<1>,1:0]\n" + "[@1,3:2='',<-1>,1:3]\n", found); @@ -439,7 +439,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "DASHBRACK : [\\-\\]]+ {System.out.println(\"DASHBRACK\");} ;\n" + "WS : [ \\u]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "- ] "); + String found = execLexer("L.g4", grammar, "L", "- ] ", false); assertEquals("DASHBRACK\n" + "DASHBRACK\n" + "[@0,0:0='-',<1>,1:0]\n" + @@ -453,7 +453,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "A : [z-a9]+ {System.out.println(\"A\");} ;\n" + "WS : [ \\u]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "9"); + String found = execLexer("L.g4", grammar, "L", "9", false); assertEquals("A\n" + "[@0,0:0='9',<1>,1:0]\n" + "[@1,1:0='',<-1>,1:1]\n", found); @@ -465,7 +465,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "A : [\"a-z]+ {System.out.println(\"A\");} ;\n" + "WS : [ \\n\\t]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "b\"a"); + String found = execLexer("L.g4", grammar, "L", "b\"a", false); assertEquals("A\n" + "[@0,0:2='b\"a',<1>,1:0]\n" + "[@1,3:2='',<-1>,1:3]\n", found); @@ -477,7 +477,7 @@ public class TestLexerExec extends BaseTest { String grammar = "lexer grammar L;\n" + "A : [\"\\ab]+ {System.out.println(\"A\");} ;\n" + "WS : [ \\n\\t]+ -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "b\"\\a"); + String found = execLexer("L.g4", grammar, "L", "b\"\\a", false); assertEquals("A\n" + "[@0,0:3='b\"\\a',<1>,1:0]\n" + "[@1,4:3='',<-1>,1:4]\n", found); @@ -599,7 +599,7 @@ public class TestLexerExec extends BaseTest { "WS\n" + " : [ \\t]+ -> skip\n" + " ;"; - String found = execLexer("PositionAdjustingLexer.g4", grammar, "PositionAdjustingLexer", "tokens\ntokens {\nnotLabel\nlabel1 =\nlabel2 +=\nnotLabel\n"); + String found = execLexer("PositionAdjustingLexer.g4", grammar, "PositionAdjustingLexer", "tokens\ntokens {\nnotLabel\nlabel1 =\nlabel2 +=\nnotLabel\n", false); assertEquals("[@0,0:5='tokens',<6>,1:0]\n" + "[@1,7:12='tokens',<4>,2:0]\n" + "[@2,14:14='{',<3>,2:7]\n" + @@ -4617,7 +4617,7 @@ public class TestLexerExec extends BaseTest { "KW3997 : 'KW' '3997';\n" + "KW3998 : 'KW' '3998';\n" + "KW3999 : 'KW' '3999';"; - String found = execLexer("L.g4", grammar, "L", "KW400"); + String found = execLexer("L.g4", grammar, "L", "KW400", false); assertEquals("[@0,0:4='KW400',<402>,1:0]\n" + "[@1,5:4='',<-1>,1:5]\n", found); assertNull(this.stderrDuringParse); @@ -4634,7 +4634,7 @@ public class TestLexerExec extends BaseTest { " StringMode_Done : -> more, mode(EndStringMode);\n" + "mode EndStringMode; \n" + " EndString : '\\'' -> popMode;"; - String found = execLexer("L.g4", grammar, "L", "'xxx'"); + String found = execLexer("L.g4", grammar, "L", "'xxx'", false); assertEquals("[@0,0:4=''xxx'',<1>,1:0]\n" + "[@1,5:4='',<-1>,1:5]\n", found); assertNull(this.stderrDuringParse); diff --git a/tool/test/org/antlr/v4/test/rt/java/TestSemPredEvalLexer.java b/tool/test/org/antlr/v4/test/rt/java/TestSemPredEvalLexer.java index 1817a622b..3211cdf6a 100644 --- a/tool/test/org/antlr/v4/test/rt/java/TestSemPredEvalLexer.java +++ b/tool/test/org/antlr/v4/test/rt/java/TestSemPredEvalLexer.java @@ -12,7 +12,7 @@ public class TestSemPredEvalLexer extends BaseTest { "E2 : 'enum' { true }? ; // winner not E1 or ID\n" + "ID : 'a'..'z'+ ;\n" + "WS : (' '|'\\n') -> skip;"; - String found = execLexer("L.g4", grammar, "L", "enum abc"); + String found = execLexer("L.g4", grammar, "L", "enum abc", true); assertEquals("[@0,0:3='enum',<2>,1:0]\n" + "[@1,5:7='abc',<3>,1:5]\n" + "[@2,8:7='',<-1>,1:8]\n" + @@ -32,7 +32,7 @@ public class TestSemPredEvalLexer extends BaseTest { "ENUM : 'enum' { false }? ;\n" + "ID : 'a'..'z'+ ;\n" + "WS : (' '|'\\n') -> skip;"; - String found = execLexer("L.g4", grammar, "L", "enum abc enum"); + String found = execLexer("L.g4", grammar, "L", "enum abc enum", true); assertEquals("[@0,0:3='enum',<2>,1:0]\n" + "[@1,5:7='abc',<2>,1:5]\n" + "[@2,9:12='enum',<2>,1:9]\n" + @@ -53,7 +53,7 @@ public class TestSemPredEvalLexer extends BaseTest { "ENUM : [a-z]+ { false }? ;\n" + "ID : [a-z]+ ;\n" + "WS : (' '|'\\n') -> skip;"; - String found = execLexer("L.g4", grammar, "L", "enum abc enum"); + String found = execLexer("L.g4", grammar, "L", "enum abc enum", true); assertEquals("[@0,0:3='enum',<2>,1:0]\n" + "[@1,5:7='abc',<2>,1:5]\n" + "[@2,9:12='enum',<2>,1:9]\n" + @@ -65,10 +65,10 @@ public class TestSemPredEvalLexer extends BaseTest { @Test public void testEnumNotID() throws Exception { String grammar = "lexer grammar L;\n" + - "ENUM : [a-z]+ { this.text===\"enum\" }? ;\n" + + "ENUM : [a-z]+ { this.getText().equals(\"enum\") }? ;\n" + "ID : [a-z]+ ;\n" + "WS : (' '|'\\n') -> skip;"; - String found = execLexer("L.g4", grammar, "L", "enum abc enum"); + String found = execLexer("L.g4", grammar, "L", "enum abc enum", true); assertEquals("[@0,0:3='enum',<1>,1:0]\n" + "[@1,5:7='abc',<2>,1:5]\n" + "[@2,9:12='enum',<1>,1:9]\n" + @@ -81,11 +81,11 @@ public class TestSemPredEvalLexer extends BaseTest { public void testIndent() throws Exception { String grammar = "lexer grammar L;\n" + "ID : [a-z]+ ;\n" + - "INDENT : [ \\t]+ { this._tokenStartColumn===0 }? \\n\" +\n" + - " { System.out.println(\"INDENT\"); } ;\"+\n" + + "INDENT : [ \\t]+ { this._tokenStartCharPositionInLine==0 }?\n" + + " { System.out.println(\"INDENT\"); } ;\n" + "NL : '\\n';\n" + "WS : [ \\t]+ ;"; - String found = execLexer("L.g4", grammar, "L", "abc\n def \n"); + String found = execLexer("L.g4", grammar, "L", "abc\n def \n", true); assertEquals("INDENT\n" + "[@0,0:2='abc',<1>,1:0]\n" + "[@1,3:3='\\n',<3>,1:3]\n" + @@ -110,10 +110,10 @@ public class TestSemPredEvalLexer extends BaseTest { String grammar = "lexer grammar L;\n" + "WORD1 : ID1+ { System.out.println(this.getText()); } ;\n" + "WORD2 : ID2+ { System.out.println(this.getText()); } ;\n" + - "fragment ID1 : { this.column < 2 }? [a-zA-Z];\n" + - "fragment ID2 : { this.column >= 2 }? [a-zA-Z];\n" + + "fragment ID1 : { this.getCharPositionInLine() < 2 }? [a-zA-Z];\n" + + "fragment ID2 : { this.getCharPositionInLine() >= 2 }? [a-zA-Z];\n" + "WS : (' '|'\\n') -> skip;"; - String found = execLexer("L.g4", grammar, "L", "a cde\nabcde\n"); + String found = execLexer("L.g4", grammar, "L", "a cde\nabcde\n", true); assertEquals("a\n" + "cde\n" + "ab\n" + @@ -129,10 +129,10 @@ public class TestSemPredEvalLexer extends BaseTest { @Test public void testPredicatedKeywords() throws Exception { String grammar = "lexer grammar L;\n" + - "ENUM : [a-z]+ { this.text===\"enum\" }? { System.out.println(\"enum!\"); } ;\n" + - "ID : [a-z]+ { System.out.println(\"ID\" + this.getText()); } ;\n" + + "ENUM : [a-z]+ { this.getText().equals(\"enum\") }? { System.out.println(\"enum!\"); } ;\n" + + "ID : [a-z]+ { System.out.println(\"ID \" + this.getText()); } ;\n" + "WS : [ \\n] -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "enum enu a"); + String found = execLexer("L.g4", grammar, "L", "enum enu a", false); assertEquals("enum!\n" + "ID enu\n" + "ID a\n" + diff --git a/tool/test/org/antlr/v4/test/tool/TestSemPredEvalLexer.java b/tool/test/org/antlr/v4/test/tool/TestSemPredEvalLexer.java deleted file mode 100644 index 862871c3d..000000000 --- a/tool/test/org/antlr/v4/test/tool/TestSemPredEvalLexer.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * [The "BSD license"] - * Copyright (c) 2012 Terence Parr - * Copyright (c) 2012 Sam Harwell - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.antlr.v4.test.tool; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class TestSemPredEvalLexer extends BaseTest { - - @Test public void testDisableRule() throws Exception { - String grammar = - "lexer grammar L;\n"+ - "E1 : 'enum' {false}? ;\n" + - "E2 : 'enum' {true}? ;\n" + // winner not E1 or ID - "ID : 'a'..'z'+ ;\n"+ - "WS : (' '|'\\n') -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "enum abc", true); - String expecting = - "[@0,0:3='enum',<2>,1:0]\n" + - "[@1,5:7='abc',<3>,1:5]\n" + - "[@2,8:7='',<-1>,1:8]\n" + - "s0-' '->:s5=>4\n" + - "s0-'a'->:s6=>3\n" + - "s0-'e'->:s1=>3\n" + - ":s1=>3-'n'->:s2=>3\n" + - ":s2=>3-'u'->:s3=>3\n" + - ":s6=>3-'b'->:s6=>3\n" + - ":s6=>3-'c'->:s6=>3\n"; - assertEquals(expecting, found); - } - - @Test public void testIDvsEnum() throws Exception { - String grammar = - "lexer grammar L;\n"+ - "ENUM : 'enum' {false}? ;\n" + - "ID : 'a'..'z'+ ;\n"+ - "WS : (' '|'\\n') -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "enum abc enum", true); - String expecting = - "[@0,0:3='enum',<2>,1:0]\n" + - "[@1,5:7='abc',<2>,1:5]\n" + - "[@2,9:12='enum',<2>,1:9]\n" + - "[@3,13:12='',<-1>,1:13]\n" + - "s0-' '->:s5=>3\n" + - "s0-'a'->:s4=>2\n" + - "s0-'e'->:s1=>2\n" + - ":s1=>2-'n'->:s2=>2\n" + - ":s2=>2-'u'->:s3=>2\n" + - ":s4=>2-'b'->:s4=>2\n" + - ":s4=>2-'c'->:s4=>2\n"; // no 'm'-> transition...conflicts with pred - assertEquals(expecting, found); - } - - @Test public void testIDnotEnum() throws Exception { - String grammar = - "lexer grammar L;\n"+ - "ENUM : [a-z]+ {false}? ;\n" + - "ID : [a-z]+ ;\n"+ - "WS : (' '|'\\n') -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "enum abc enum", true); - String expecting = - "[@0,0:3='enum',<2>,1:0]\n" + - "[@1,5:7='abc',<2>,1:5]\n" + - "[@2,9:12='enum',<2>,1:9]\n" + - "[@3,13:12='',<-1>,1:13]\n" + - "s0-' '->:s2=>3\n"; // no edges in DFA for enum/id. all paths lead to pred. - assertEquals(expecting, found); - } - - @Test public void testEnumNotID() throws Exception { - String grammar = - "lexer grammar L;\n"+ - "ENUM : [a-z]+ {getText().equals(\"enum\")}? ;\n" + - "ID : [a-z]+ ;\n"+ - "WS : (' '|'\\n') -> skip ;"; - String found = execLexer("L.g4", grammar, "L", "enum abc enum", true); - String expecting = - "[@0,0:3='enum',<1>,1:0]\n" + - "[@1,5:7='abc',<2>,1:5]\n" + - "[@2,9:12='enum',<1>,1:9]\n" + - "[@3,13:12='',<-1>,1:13]\n" + - "s0-' '->:s3=>3\n"; // no edges in DFA for enum/id. all paths lead to pred. - assertEquals(expecting, found); - } - - @Test public void testIndent() throws Exception { - String grammar = - "lexer grammar L;\n"+ - "ID : [a-z]+ ;\n"+ - "INDENT : [ \\t]+ {_tokenStartCharPositionInLine==0}? \n" + - " {System.out.println(\"INDENT\");} ;"+ - "NL : '\\n' ;"+ - "WS : [ \\t]+ ;"; - String found = execLexer("L.g4", grammar, "L", "abc\n def \n", true); - String expecting = - "INDENT\n" + // action output - "[@0,0:2='abc',<1>,1:0]\n" + // ID - "[@1,3:3='\\n',<3>,1:3]\n" + // NL - "[@2,4:5=' ',<2>,2:0]\n" + // INDENT - "[@3,6:8='def',<1>,2:2]\n" + // ID - "[@4,9:10=' ',<4>,2:5]\n" + // WS - "[@5,11:11='\\n',<3>,2:7]\n" + - "[@6,12:11='',<-1>,3:8]\n" + - "s0-'\n" + - "'->:s2=>3\n" + - "s0-'a'->:s1=>1\n" + - "s0-'d'->:s1=>1\n" + - ":s1=>1-'b'->:s1=>1\n" + - ":s1=>1-'c'->:s1=>1\n" + - ":s1=>1-'e'->:s1=>1\n" + - ":s1=>1-'f'->:s1=>1\n"; - assertEquals(expecting, found); - } - - @Test public void testLexerInputPositionSensitivePredicates() throws Exception { - String grammar = - "lexer grammar L;\n"+ - "WORD1 : ID1+ {System.out.println(getText());} ;\n"+ - "WORD2 : ID2+ {System.out.println(getText());} ;\n"+ - "fragment ID1 : {getCharPositionInLine()<2}? [a-zA-Z];\n"+ - "fragment ID2 : {getCharPositionInLine()>=2}? [a-zA-Z];\n"+ - "WS : (' '|'\\n') -> skip;\n"; - String found = execLexer("L.g4", grammar, "L", "a cde\nabcde\n"); - String expecting = - "a\n" + - "cde\n" + - "ab\n" + - "cde\n" + - "[@0,0:0='a',<1>,1:0]\n" + - "[@1,2:4='cde',<2>,1:2]\n" + - "[@2,6:7='ab',<1>,2:0]\n" + - "[@3,8:10='cde',<2>,2:2]\n" + - "[@4,12:11='',<-1>,3:0]\n"; - assertEquals(expecting, found); - } - - @Test public void testPredicatedKeywords() { - String grammar = - "lexer grammar A;" + - "ENUM : [a-z]+ {getText().equals(\"enum\")}? {System.out.println(\"enum!\");} ;\n" + - "ID : [a-z]+ {System.out.println(\"ID \"+getText());} ;\n" + - "WS : [ \\n] -> skip ;"; - String found = execLexer("A.g4", grammar, "A", "enum enu a"); - String expecting = - "enum!\n" + - "ID enu\n" + - "ID a\n" + - "[@0,0:3='enum',<1>,1:0]\n" + - "[@1,5:7='enu',<2>,1:5]\n" + - "[@2,9:9='a',<2>,1:9]\n" + - "[@3,10:9='',<-1>,1:10]\n"; - assertEquals(expecting, found); - } -}