forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

prepare release 

all tests pass
added test for complement set
This commit is contained in:
ericvergnaud 2014-11-23 02:02:01 +08:00
parent fba656e6a2
commit 4b1a7d1274
17 changed files with 4527 additions and 4631 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -2,4 +2,5 @@
*.pyc
__pycache__
src/TestAntLR.py
/dist
/dist
.DS_Store

48
src/antlr4/IntervalSet.py
View File

@ -23,6 +23,7 @@ class IntervalSet(object):
return k
else:
i += 1
return Token.INVALID_TYPE
def addOne(self, v:int):
self.addRange(range(v, v+1))
@ -71,6 +72,13 @@ class IntervalSet(object):
self.intervals[k] = range(l.start, r.stop)
self.intervals.pop(k+1)
def complement(self, start, stop):
result = IntervalSet()
result.addRange(range(start,stop+1))
for i in self.intervals:
result.removeRange(i)
return result
def __contains__(self, item):
if self.intervals is None:
return False
@ -86,7 +94,34 @@ class IntervalSet(object):
xlen += len(i)
return xlen
def remove(self, v:int):
def removeRange(self, v):
if v.start==v.stop-1:
self.removeOne(v.start)
elif self.intervals is not None:
k = 0
for i in self.intervals:
# intervals are ordered
if v.stop<=i.start:
return
# check for including range, split it
elif v.start>i.start and v.stop<i.stop:
self.intervals[k] = range(i.start, v.start)
x = range(v.stop, i.stop)
self.intervals.insert(k, x)
return
# check for included range, remove it
elif v.start<=i.start and v.stop>=i.stop:
self.intervals.pop(k)
k = k - 1 # need another pass
# check for lower boundary
elif v.start<i.stop:
self.intervals[k] = range(i.start, v.start)
# check for upper boundary
elif v.stop<i.stop:
self.intervals[k] = range(v.stop, i.stop)
k += 1
def removeOne(self, v):
if self.intervals is not None:
k = 0
for i in self.intervals:
@ -233,3 +268,14 @@ class TestIntervalSet(unittest.TestCase):
self.assertEquals(1,len(s.intervals))
self.assertTrue(20 in s)
self.assertTrue(60 in s)
def testComplement(self):
s = IntervalSet()
s.addRange(range(10,21))
c = s.complement(1,100)
self.assertTrue(1 in c)
self.assertTrue(100 in c)
self.assertTrue(10 not in c)
self.assertTrue(20 not in c)

8
src/antlr4/Lexer.py
View File

@ -238,13 +238,9 @@ class Lexer(Recognizer, TokenSource):
def emitEOF(self):
cpos = self.column
# The character position for EOF is one beyond the position of
# the previous token's last character
if self._token is not None:
n = self._token.stop - self._token.start + 1
cpos = self._token.column + n
lpos = self.line
eof = self._factory.create(self._tokenFactorySourcePair, Token.EOF, None, Token.DEFAULT_CHANNEL, self._input.index,
self._input.index-1, self.line, cpos)
self._input.index-1, lpos, cpos)
self.emitToken(eof)
return eof

4
src/antlr4/atn/ATN.py
View File

@ -139,13 +139,13 @@ class ATN(object):
return following
expected = IntervalSet()
expected.addSet(following)
expected.remove(Token.EPSILON)
expected.removeOne(Token.EPSILON)
while (ctx != None and ctx.invokingState >= 0 and Token.EPSILON in following):
invokingState = self.states[ctx.invokingState]
rt = invokingState.transitions[0]
following = self.nextTokens(rt.followState)
expected.addSet(following)
expected.remove(Token.EPSILON)
expected.removeOne(Token.EPSILON)
ctx = ctx.parentCtx
if Token.EPSILON in following:
expected.addOne(Token.EOF)

15
src/antlr4/atn/LexerATNSimulator.py
View File

@ -161,8 +161,6 @@ class LexerATNSimulator(ATNSimulator):
if ds0.isAcceptState:
# allow zero-length tokens
self.captureSimState(self.prevAccept, input, ds0)
# adjust index since the current input character was not yet consumed
self.prevAccept.index -= 1
t = input.LA(1)
s = ds0 # s is current/from DFA state
@ -198,14 +196,19 @@ class LexerATNSimulator(ATNSimulator):
if target == self.ERROR:
break
# If this is a consumable input element, make sure to consume before
# capturing the accept state so the input index, line, and char
# position accurately reflect the state of the interpreter at the
# end of the token.
if t != Token.EOF:
self.consume(input)
if target.isAcceptState:
self.captureSimState(self.prevAccept, input, target)
if t == Token.EOF:
break
if t != Token.EOF:
self.consume(input)
t = input.LA(1)
t = input.LA(1)
s = target # flip; current DFA target becomes new src/from state
@ -307,8 +310,6 @@ class LexerATNSimulator(ATNSimulator):
input.seek(index)
self.line = line
self.column = charPos
if input.LA(1) != Token.EOF:
self.consume(input)
if lexerActionExecutor is not None and self.recog is not None:
lexerActionExecutor.execute(self.recog, input, startIndex)

2
src/antlr4/error/ErrorStrategy.py
View File

@ -661,7 +661,7 @@ class DefaultErrorStrategy(ErrorStrategy):
follow = atn.nextTokens(rt.followState)
recoverSet.addSet(follow)
ctx = ctx.parentCtx
recoverSet.remove(Token.EPSILON)
recoverSet.removeOne(Token.EPSILON)
return recoverSet
# Consume tokens until one matches the given token set.#

28
tool/test/org/antlr/v4/py/test/BasePythonTest.java → tool/test/org/antlr/v4/test/rt/py/BasePythonTest.java
View File

@ -27,7 +27,7 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.py.test;
package org.antlr.v4.test.rt.py;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@ -537,21 +537,37 @@ public abstract class BasePythonTest {
return null;
}
private String locateTool(String tool) {
String[] roots = { "/usr/bin/", "/usr/local/bin/" };
for(String root : roots) {
if(new File(root + tool).exists())
return root + tool;
}
throw new RuntimeException("Could not locate " + tool);
}
protected String locatePython() {
// typically "/usr/local/bin/Python2.7"
String propName = getPropertyPrefix() + "-python";
String propName = getPropertyPrefix() + "-python";
String prop = System.getProperty(propName);
if(prop==null || prop.length()==0)
throw new RuntimeException("Missing system property:" + propName);
return prop;
prop = locateTool(getPythonExecutable());
File file = new File(prop);
if(!file.exists())
throw new RuntimeException("Missing system property:" + propName);
return file.getAbsolutePath();
}
protected abstract String getPythonExecutable();
private String locateRuntime() {
String propName = getPropertyPrefix() + "-runtime";
String prop = System.getProperty(propName);
if(prop==null || prop.length()==0)
prop = "../../antlr4-" + getLanguage() + "/src";
File file = new File(prop);
if(!file.exists())
throw new RuntimeException("Missing system property:" + propName);
return prop;
return file.getAbsolutePath();
}
public void testErrors(String[] pairs, boolean printTree) {

7
tool/test/org/antlr/v4/test/rt/py3/BasePython3Test.java
View File

@ -29,7 +29,7 @@
*/
package org.antlr.v4.test.rt.py3;
import org.antlr.v4.py.test.BasePythonTest;
import org.antlr.v4.test.rt.py.BasePythonTest;
import org.stringtemplate.v4.ST;
public abstract class BasePython3Test extends BasePythonTest {
@ -39,6 +39,11 @@ public abstract class BasePython3Test extends BasePythonTest {
return "Python3";
}
@Override
protected String getPythonExecutable() {
return "Python3.4";
}
@Override
protected void writeLexerTestFile(String lexerName, boolean showDFA) {
ST outputFileST = new ST(

4
tool/test/org/antlr/v4/test/rt/py3/Python3.test.stg
View File

@ -26,7 +26,9 @@ public void test<test.name>() throws Exception {
mkdir(tmpdir);
writeFile(tmpdir, "<grammar.grammarName>.g4", slave_<grammar.grammarName>);
};separator="\n", wrap, anchor>
String grammar = <test.grammar.lines:{ line | "<line>};separator="\\n\" +\n", wrap, anchor>";
StringBuilder sb = new StringBuilder();
<test.grammar.lines:{ line | sb.append("<line>\\n");};separator="\n", wrap, anchor>
String grammar = sb.toString();
<test.afterGrammar>
String found = execLexer("<test.grammar.grammarName>.g4", grammar, "<test.grammar.grammarName><if(!test.lexerOnly)>Lexer<endif>", "<test.input>", <test.showDFA>);
assertEquals(<test.outputLines:{ line | "<line>\\n"};separator=" + \n", wrap, anchor>, found);

20
tool/test/org/antlr/v4/test/rt/py3/TestCompositeLexers.java
View File

@ -13,10 +13,12 @@ public class TestCompositeLexers extends BasePython3Test {
mkdir(tmpdir);
writeFile(tmpdir, "S.g4", slave_S);
String grammar = "lexer grammar M;\n" +
"import S;\n" +
"B : 'b';\n" +
"WS : (' '|'\\n') -> skip ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar M;\n");
sb.append("import S;\n");
sb.append("B : 'b';\n");
sb.append("WS : (' '|'\\n') -> skip ;\n");
String grammar = sb.toString();
String found = execLexer("M.g4", grammar, "M", "abc", false);
assertEquals("S.A\n" +
"[@0,0:0='a',<3>,1:0]\n" +
@ -34,10 +36,12 @@ public class TestCompositeLexers extends BasePython3Test {
mkdir(tmpdir);
writeFile(tmpdir, "S.g4", slave_S);
String grammar = "lexer grammar M;\n" +
"import S;\n" +
"A : 'a' B {print(\"M.A\")};\n" +
"WS : (' '|'\\n') -> skip ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar M;\n");
sb.append("import S;\n");
sb.append("A : 'a' B {print(\"M.A\")};\n");
sb.append("WS : (' '|'\\n') -> skip ;\n");
String grammar = sb.toString();
String found = execLexer("M.g4", grammar, "M", "ab", false);
assertEquals("M.A\n" +
"[@0,0:1='ab',<1>,1:0]\n" +

276
tool/test/org/antlr/v4/test/rt/py3/TestLexerActions.java
View File

@ -1,276 +0,0 @@
package org.antlr.v4.test.rt.py3;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class TestLexerActions extends BasePython3Test {
// ----- ACTIONS --------------------------------------------------------
@Test public void testActionExecutedInDFA() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : '0'..'9'+ { print(\"I\") } ;\n"+
"WS : (' '|'\\n') -> skip ;";
String found = execLexer("L.g4", grammar, "L", "34 34");
String expecting =
"I\n" +
"I\n" +
"[@0,0:1='34',<1>,1:0]\n" +
"[@1,3:4='34',<1>,1:3]\n" +
"[@2,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, found);
}
@Test public void testActionEvalsAtCorrectIndex() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : [0-9] { print(\"2nd char: \" + chr(self._input.LA(1))) } [0-9]+ ;\n"+
"WS : (' '|'\\n') -> skip ;";
String found = execLexer("L.g4", grammar, "L", "123 45");
String expecting =
"2nd char: 2\n" +
"2nd char: 5\n" +
"[@0,0:2='123',<1>,1:0]\n" +
"[@1,4:5='45',<1>,1:4]\n" +
"[@2,6:5='<EOF>',<-1>,1:6]\n";
assertEquals(expecting, found);
}
/**
* This is a regressing test for antlr/antlr4#469 "Not all internal lexer
* rule actions are executed".
* https://github.com/antlr/antlr4/issues/469
*/
@Test public void testEvalMultipleActions() throws Exception {
String grammar =
"lexer grammar L;\n" +
"\n" +
"@lexer::members\n" +
"{\n" +
"def getRecordedText (self):\n" +
" return self._input.getText (self.start_index, self.stop_index)\n" +
"\n" +
"def start (self):\n" +
" self.start_index = self._input.index\n" +
" print (\"Start:\" + str(self.start_index))\n" +
"\n" +
"def stop (self):\n" +
" self.stop_index = self._input.index\n" +
" print (\"Stop:\" + str(self.stop_index))\n" +
"\n" +
"}\n" +
"\n" +
"HELLO: 'hello' WS { self.start () } NAME { self.stop () } '\\n' { print (\"Hello: \" + self.getRecordedText ()) };\n" +
"NAME: ('a'..'z' | 'A'..'Z')+ ('\\n')?;\n" +
"\n" +
"fragment WS: [ \\r\\t\\n]+ ;\n";
String found = execLexer("L.g4", grammar, "L", "hello Steve\n");
String expecting =
"Start:6\n" +
"Stop:11\n" +
"Hello: Steve\n" +
"\n" +
"[@0,0:11='hello Steve\\n',<1>,1:0]\n" +
"[@1,12:11='<EOF>',<-1>,2:12]\n";
assertEquals(expecting, found);
}
@Test public void test2ActionsIn1Rule() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : [0-9] { print(\"x\") } [0-9]+ { print(\"y\") } ;\n"+
"WS : (' '|'\\n') -> skip ;";
String found = execLexer("L.g4", grammar, "L", "123 45");
String expecting =
"x\n" +
"y\n" +
"x\n" +
"y\n" +
"[@0,0:2='123',<1>,1:0]\n" +
"[@1,4:5='45',<1>,1:4]\n" +
"[@2,6:5='<EOF>',<-1>,1:6]\n";
assertEquals(expecting, found);
}
@Test public void testAltActionsIn1Rule() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : ( [0-9]+ { print(\"int\", end='') }\n" +
" | [a-z]+ { print(\"id\", end='') }\n" +
" )\n" +
" { print(\" last\") }\n" +
" ;\n"+
"WS : (' '|'\\n') -> skip ;";
String found = execLexer("L.g4", grammar, "L", "123 ab");
String expecting =
"int last\n" +
"id last\n" +
"[@0,0:2='123',<1>,1:0]\n" +
"[@1,4:5='ab',<1>,1:4]\n" +
"[@2,6:5='<EOF>',<-1>,1:6]\n";
assertEquals(expecting, found);
}
@Test public void testActionPlusCommand() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : '0'..'9'+ { print(\"I\") } -> skip ;\n"+
"WS : (' '|'\\n') -> skip ;";
String found = execLexer("L.g4", grammar, "L", "34 34");
String expecting =
"I\n" +
"I\n" +
"[@0,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, found);
}
// ----- COMMANDS --------------------------------------------------------
@Test public void testSkipCommand() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : '0'..'9'+ { print(\"I\") } ;\n"+
"WS : (' '|'\\n') -> skip ;";
String found = execLexer("L.g4", grammar, "L", "34 34");
String expecting =
"I\n" +
"I\n" +
"[@0,0:1='34',<1>,1:0]\n" +
"[@1,3:4='34',<1>,1:3]\n" +
"[@2,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, found);
}
@Test public void testMoreCommand() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : '0'..'9'+ { print(\"I\") } ;\n"+
"WS : '#' -> more ;";
String found = execLexer("L.g4", grammar, "L", "34#10");
String expecting =
"I\n" +
"I\n" +
"[@0,0:1='34',<1>,1:0]\n" +
"[@1,2:4='#10',<1>,1:2]\n" +
"[@2,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, found);
}
@Test public void testTypeCommand() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : '0'..'9'+ { print(\"I\") } ;\n"+
"HASH : '#' -> type(HASH) ;";
String found = execLexer("L.g4", grammar, "L", "34#");
String expecting =
"I\n" +
"[@0,0:1='34',<1>,1:0]\n" +
"[@1,2:2='#',<2>,1:2]\n" +
"[@2,3:2='<EOF>',<-1>,1:3]\n";
assertEquals(expecting, found);
}
@Test public void testCombinedCommand() throws Exception {
String grammar =
"lexer grammar L;\n"+
"I : '0'..'9'+ { print(\"I\") } ;\n"+
"HASH : '#' -> type(100), skip, more ;";
String found = execLexer("L.g4", grammar, "L", "34#11");
String expecting =
"I\n" +
"I\n" +
"[@0,0:1='34',<1>,1:0]\n" +
"[@1,2:4='#11',<1>,1:2]\n" +
"[@2,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, found);
}
@Test public void testLexerMode() throws Exception {
String grammar =
"lexer grammar L;\n" +
"STRING_START : '\"' -> pushMode(STRING_MODE), more;\n" +
"WS : (' '|'\\n') -> skip ;\n"+
"mode STRING_MODE;\n"+
"STRING : '\"' -> popMode;\n"+
"ANY : . -> more;\n";
String found = execLexer("L.g4", grammar, "L", "\"abc\" \"ab\"");
String expecting =
"[@0,0:4='\"abc\"',<2>,1:0]\n" +
"[@1,6:9='\"ab\"',<2>,1:6]\n" +
"[@2,10:9='<EOF>',<-1>,1:10]\n";
assertEquals(expecting, found);
}
@Test public void testLexerPushPopModeAction() throws Exception {
String grammar =
"lexer grammar L;\n" +
"STRING_START : '\"' -> pushMode(STRING_MODE), more ;\n" +
"WS : (' '|'\\n') -> skip ;\n"+
"mode STRING_MODE;\n"+
"STRING : '\"' -> popMode ;\n"+ // token type 2
"ANY : . -> more ;\n";
String found = execLexer("L.g4", grammar, "L", "\"abc\" \"ab\"");
String expecting =
"[@0,0:4='\"abc\"',<2>,1:0]\n" +
"[@1,6:9='\"ab\"',<2>,1:6]\n" +
"[@2,10:9='<EOF>',<-1>,1:10]\n";
assertEquals(expecting, found);
}
@Test public void testLexerModeAction() throws Exception {
String grammar =
"lexer grammar L;\n" +
"STRING_START : '\"' -> mode(STRING_MODE), more ;\n" +
"WS : (' '|'\\n') -> skip ;\n"+
"mode STRING_MODE;\n"+
"STRING : '\"' -> mode(DEFAULT_MODE) ;\n"+ // ttype 2 since '"' ambiguity
"ANY : . -> more ;\n";
String found = execLexer("L.g4", grammar, "L", "\"abc\" \"ab\"");
String expecting =
"[@0,0:4='\"abc\"',<2>,1:0]\n" +
"[@1,6:9='\"ab\"',<2>,1:6]\n" +
"[@2,10:9='<EOF>',<-1>,1:10]\n";
assertEquals(expecting, found);
}
// ----- PREDICATES --------------------------------------------------------
/**
* This is a regression test for antlr/antlr4#398 "Lexer: literal matches
* while negated char set fail to match"
* https://github.com/antlr/antlr4/issues/398
*/
@Test
public void testFailingPredicateEvalIsNotCached() {
String grammar =
"lexer grammar TestLexer;\n" +
"\n" +
"fragment WS: [ \\t]+;\n" +
"fragment EOL: '\\r'? '\\n';\n" +
"\n" +
"LINE: WS? ~[\\r\\n]* EOL { not self.text.strip().startswith(\"Item:\") }?;\n" +
"ITEM: WS? 'Item:' -> pushMode(ITEM_HEADING_MODE);\n" +
"\n" +
"mode ITEM_HEADING_MODE;\n" +
"\n" +
"NAME: ~[\\r\\n]+;\n" +
"SECTION_HEADING_END: EOL -> popMode;\n";
String input =
"A line here.\n" +
"Item: name of item\n" +
"Another line.\n" +
"More line.\n";
String found = execLexer("TestLexer.g4", grammar, "TestLexer", input);
String expecting =
"[@0,0:12='A line here.\\n',<1>,1:0]\n" +
"[@1,13:17='Item:',<2>,2:0]\n" +
"[@2,18:30=' name of item',<3>,2:5]\n" +
"[@3,31:31='\\n',<4>,2:18]\n" +
"[@4,32:45='Another line.\\n',<1>,3:0]\n" +
"[@5,46:56='More line.\\n',<1>,4:0]\n" +
"[@6,57:56='<EOF>',<-1>,5:11]\n";
assertEquals(expecting, found);
}
}

96
tool/test/org/antlr/v4/test/rt/py3/TestLexerErrors.java
View File

@ -7,8 +7,10 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testInvalidCharAtStart() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "x", false);
assertEquals("[@0,1:0='<EOF>',<-1>,1:1]\n", found);
assertEquals("line 1:0 token recognition error at: 'x'\n", this.stderrDuringParse);
@ -16,10 +18,12 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testStringsEmbeddedInActions_1() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION2 : '[' (STRING | ~'\"')*? ']';\n" +
"STRING : '\"' ('\\\"' | .)*? '\"';\n" +
"WS : [ \\t\\r\\n]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION2 : '[' (STRING | ~'\"')*? ']';\n");
sb.append("STRING : '\"' ('\\\"' | .)*? '\"';\n");
sb.append("WS : [ \\t\\r\\n]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "[\"foo\"]", false);
assertEquals("[@0,0:6='[\"foo\"]',<1>,1:0]\n" +
"[@1,7:6='<EOF>',<-1>,1:7]\n", found);
@ -28,10 +32,12 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testStringsEmbeddedInActions_2() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION2 : '[' (STRING | ~'\"')*? ']';\n" +
"STRING : '\"' ('\\\"' | .)*? '\"';\n" +
"WS : [ \\t\\r\\n]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION2 : '[' (STRING | ~'\"')*? ']';\n");
sb.append("STRING : '\"' ('\\\"' | .)*? '\"';\n");
sb.append("WS : [ \\t\\r\\n]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "[\"foo]", false);
assertEquals("[@0,6:5='<EOF>',<-1>,1:6]\n", found);
assertEquals("line 1:0 token recognition error at: '[\"foo]'\n", this.stderrDuringParse);
@ -39,9 +45,11 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testEnforcedGreedyNestedBrances_1() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION : '{' (ACTION | ~[{}])* '}';\n" +
"WS : [ \\r\\n\\t]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION : '{' (ACTION | ~[{}])* '}';\n");
sb.append("WS : [ \\r\\n\\t]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "{ { } }", false);
assertEquals("[@0,0:6='{ { } }',<1>,1:0]\n" +
"[@1,7:6='<EOF>',<-1>,1:7]\n", found);
@ -50,9 +58,11 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testEnforcedGreedyNestedBrances_2() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION : '{' (ACTION | ~[{}])* '}';\n" +
"WS : [ \\r\\n\\t]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION : '{' (ACTION | ~[{}])* '}';\n");
sb.append("WS : [ \\r\\n\\t]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "{ { }", false);
assertEquals("[@0,5:4='<EOF>',<-1>,1:5]\n", found);
assertEquals("line 1:0 token recognition error at: '{ { }'\n", this.stderrDuringParse);
@ -60,8 +70,10 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testInvalidCharAtStartAfterDFACache() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abx", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,3:2='<EOF>',<-1>,1:3]\n", found);
@ -70,8 +82,10 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testInvalidCharInToken() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "ax", false);
assertEquals("[@0,2:1='<EOF>',<-1>,1:2]\n", found);
assertEquals("line 1:0 token recognition error at: 'ax'\n", this.stderrDuringParse);
@ -79,8 +93,10 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testInvalidCharInTokenAfterDFACache() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abax", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,4:3='<EOF>',<-1>,1:4]\n", found);
@ -89,9 +105,11 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testDFAToATNThatFailsBackToDFA() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'ab' ;\n" +
"B : 'abc' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'ab' ;\n");
sb.append("B : 'abc' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "ababx", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,2:3='ab',<1>,1:2]\n" +
@ -101,10 +119,12 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testDFAToATNThatMatchesThenFailsInATN() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'ab' ;\n" +
"B : 'abc' ;\n" +
"C : 'abcd' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'ab' ;\n");
sb.append("B : 'abc' ;\n");
sb.append("C : 'abcd' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "ababcx", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,2:4='abc',<2>,1:2]\n" +
@ -114,8 +134,10 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testErrorInMiddle() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'abc' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'abc' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abx", false);
assertEquals("[@0,3:2='<EOF>',<-1>,1:3]\n", found);
assertEquals("line 1:0 token recognition error at: 'abx'\n", this.stderrDuringParse);
@ -123,11 +145,13 @@ public class TestLexerErrors extends BasePython3Test {
@Test
public void testLexerExecDFA() throws Exception {
String grammar = "grammar L;\n" +
"start : ID ':' expr;\n" +
"expr : primary expr? {} | expr '->' ID;\n" +
"primary : ID;\n" +
"ID : [a-z]+;";
StringBuilder sb = new StringBuilder();
sb.append("grammar L;\n");
sb.append("start : ID ':' expr;\n");
sb.append("expr : primary expr? {} | expr '->' ID;\n");
sb.append("primary : ID;\n");
sb.append("ID : [a-z]+;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "LLexer", "x : x", false);
assertEquals("[@0,0:0='x',<3>,1:0]\n" +
"[@1,2:2=':',<1>,1:2]\n" +

8530
tool/test/org/antlr/v4/test/rt/py3/TestLexerExec.java
View File

File diff suppressed because it is too large Load Diff

23
tool/test/org/antlr/v4/test/rt/py3/TestParserExec.java
View File

@ -416,28 +416,5 @@ public class TestParserExec extends BasePython3Test {
assertNull(this.stderrDuringParse);
}
@Test
public void testAlternateQuotes() throws Exception {
String slave_ModeTagsLexer = "lexer grammar ModeTagsLexer;\n" +
"// Default mode rules (the SEA)\n" +
"OPEN : '«' -> mode(ISLAND) ; // switch to ISLAND mode\n" +
"TEXT : ~'«'+ ; // clump all text together\n" +
"mode ISLAND;\n" +
"CLOSE : '»' -> mode(DEFAULT_MODE) ; // back to SEA mode\n" +
"SLASH : '/' ;\n" +
"ID : [a-zA-Z]+ ; // match/send ID in tag to parser";
rawGenerateAndBuildRecognizer("ModeTagsLexer.g4", slave_ModeTagsLexer, null, "ModeTagsLexer");
String grammar = "parser grammar ModeTagsParser;\n" +
"options { tokenVocab=ModeTagsLexer; } // use tokens from ModeTagsLexer.g4\n" +
"file_: (tag | TEXT)* ;\n" +
"tag : '«' ID '»'\n" +
" | '«' '/' ID '»'\n" +
" ;";
String found = execParser("ModeTagsParser.g4", grammar, "ModeTagsParser", "ModeTagsLexer", "ModeTagsParserListener", "ModeTagsParserVisitor", "file_", "", false);
assertEquals("", found);
assertNull(this.stderrDuringParse);
}
}

82
tool/test/org/antlr/v4/test/rt/py3/TestSemPredEvalLexer.java
View File

@ -7,11 +7,13 @@ public class TestSemPredEvalLexer extends BasePython3Test {
@Test
public void testDisableRule() throws Exception {
String grammar = "lexer grammar L;\n" +
"E1 : 'enum' { False }? ;\n" +
"E2 : 'enum' { True }? ; // winner not E1 or ID\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("E1 : 'enum' { False }? ;\n");
sb.append("E2 : 'enum' { True }? ; // winner not E1 or ID\n");
sb.append("ID : 'a'..'z'+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc", true);
assertEquals("[@0,0:3='enum',<2>,1:0]\n" +
"[@1,5:7='abc',<3>,1:5]\n" +
@ -28,10 +30,12 @@ public class TestSemPredEvalLexer extends BasePython3Test {
@Test
public void testIDvsEnum() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : 'enum' { False }? ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : 'enum' { False }? ;\n");
sb.append("ID : 'a'..'z'+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
assertEquals("[@0,0:3='enum',<2>,1:0]\n" +
"[@1,5:7='abc',<2>,1:5]\n" +
@ -49,10 +53,12 @@ public class TestSemPredEvalLexer extends BasePython3Test {
@Test
public void testIDnotEnum() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : [a-z]+ { False }? ;\n" +
"ID : [a-z]+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : [a-z]+ { False }? ;\n");
sb.append("ID : [a-z]+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
assertEquals("[@0,0:3='enum',<2>,1:0]\n" +
"[@1,5:7='abc',<2>,1:5]\n" +
@ -64,10 +70,12 @@ public class TestSemPredEvalLexer extends BasePython3Test {
@Test
public void testEnumNotID() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : [a-z]+ { self.text==\"enum\" }? ;\n" +
"ID : [a-z]+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : [a-z]+ { self.text==\"enum\" }? ;\n");
sb.append("ID : [a-z]+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
assertEquals("[@0,0:3='enum',<1>,1:0]\n" +
"[@1,5:7='abc',<2>,1:5]\n" +
@ -79,12 +87,14 @@ public class TestSemPredEvalLexer extends BasePython3Test {
@Test
public void testIndent() throws Exception {
String grammar = "lexer grammar L;\n" +
"ID : [a-z]+ ;\n" +
"INDENT : [ \\t]+ { self._tokenStartColumn==0 }?\n" +
" { print(\"INDENT\") } ;\n" +
"NL : '\\n';\n" +
"WS : [ \\t]+ ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ID : [a-z]+ ;\n");
sb.append("INDENT : [ \\t]+ { self._tokenStartColumn==0 }?\n");
sb.append(" { print(\"INDENT\") } ;\n");
sb.append("NL : '\\n';\n");
sb.append("WS : [ \\t]+ ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abc\n def \n", true);
assertEquals("INDENT\n" +
"[@0,0:2='abc',<1>,1:0]\n" +
@ -93,7 +103,7 @@ public class TestSemPredEvalLexer extends BasePython3Test {
"[@3,6:8='def',<1>,2:2]\n" +
"[@4,9:10=' ',<4>,2:5]\n" +
"[@5,11:11='\\n',<3>,2:7]\n" +
"[@6,12:11='<EOF>',<-1>,3:8]\n" +
"[@6,12:11='<EOF>',<-1>,3:0]\n" +
"s0-'\n" +
"'->:s2=>3\n" +
"s0-'a'->:s1=>1\n" +
@ -107,12 +117,14 @@ public class TestSemPredEvalLexer extends BasePython3Test {
@Test
public void testLexerInputPositionSensitivePredicates() throws Exception {
String grammar = "lexer grammar L;\n" +
"WORD1 : ID1+ { print(self.text) } ;\n" +
"WORD2 : ID2+ { print(self.text) } ;\n" +
"fragment ID1 : { self.column < 2 }? [a-zA-Z];\n" +
"fragment ID2 : { self.column >= 2 }? [a-zA-Z];\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("WORD1 : ID1+ { print(self.text) } ;\n");
sb.append("WORD2 : ID2+ { print(self.text) } ;\n");
sb.append("fragment ID1 : { self.column < 2 }? [a-zA-Z];\n");
sb.append("fragment ID2 : { self.column >= 2 }? [a-zA-Z];\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "a cde\nabcde\n", true);
assertEquals("a\n" +
"cde\n" +
@ -128,10 +140,12 @@ public class TestSemPredEvalLexer extends BasePython3Test {
@Test
public void testPredicatedKeywords() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : [a-z]+ { self.text==\"enum\" }? { print(\"enum!\") } ;\n" +
"ID : [a-z]+ { print(\"ID \" + self.text) } ;\n" +
"WS : [ \\n] -> skip ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : [a-z]+ { self.text==\"enum\" }? { print(\"enum!\") } ;\n");
sb.append("ID : [a-z]+ { print(\"ID \" + self.text) } ;\n");
sb.append("WS : [ \\n] -> skip ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum enu a", false);
assertEquals("enum!\n" +
"ID enu\n" +

2
tool/test/org/antlr/v4/test/rt/py3/TestSemPredEvalParser.java
View File

@ -412,7 +412,7 @@ public class TestSemPredEvalParser extends BasePython3Test {
public void testPredFromAltTestedInLoopBack_1() throws Exception {
String found = testPredFromAltTestedInLoopBack("s\n\n\nx\n");
assertEquals("(file_ (para (paraContent s) \\n \\n) (para (paraContent \\n x \\n)) <EOF>)\n", found);
assertEquals("line 5:2 mismatched input '<EOF>' expecting '\n'\n", this.stderrDuringParse);
assertEquals("line 5:0 mismatched input '<EOF>' expecting '\n'\n", this.stderrDuringParse);
}
@Test

10
tool/test/org/antlr/v4/test/rt/py3/TestSets.java
View File

@ -223,5 +223,15 @@ public class TestSets extends BasePython3Test {
assertNull(this.stderrDuringParse);
}
@Test
public void testComplementSet() throws Exception {
String grammar = "grammar T;\n" +
"parse : ~NEW_LINE;\n" +
"NEW_LINE: '\\r'? '\\n';";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "parse", "a", false);
assertEquals("", found);
assertEquals("line 1:0 token recognition error at: 'a'\nline 1:1 missing {} at '<EOF>'\n", this.stderrDuringParse);
}
}