forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Do not allow raw newline characters in lexer string literals

This commit is contained in:
Sam Harwell 2013-02-22 14:06:25 -06:00
parent 1bc4c5e6a4
commit 18508e2209
6 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tool/src/org/antlr/v4/parse/ANTLRLexer.g
View File

@ -621,7 +621,7 @@ STRING_LITERAL
@init { @init {
int len = 0; int len = 0;
} }
: '\'' ( ( ESC_SEQ | ~('\\'|'\'') ) {len++;} )* '\'' : '\'' ( ( ESC_SEQ | ~('\\'|'\''|'\r'|'\n') ) {len++;} )* '\''
; ;
// A valid hex digit specification // A valid hex digit specification

2
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -159,6 +159,7 @@ public abstract class BaseTest {
protected ATN createATN(Grammar g, boolean useSerializer) { protected ATN createATN(Grammar g, boolean useSerializer) {
if ( g.atn==null ) { if ( g.atn==null ) {
semanticProcess(g); semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f; ParserATNFactory f;
if ( g.isLexer() ) { if ( g.isLexer() ) {
@ -169,6 +170,7 @@ public abstract class BaseTest {
} }
g.atn = f.createATN(); g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
} }
ATN atn = g.atn; ATN atn = g.atn;

2
tool/test/org/antlr/v4/test/TestATNDeserialization.java
View File

@ -125,7 +125,7 @@ public class TestATNDeserialization extends BaseTest {
@Test public void testLexerEOFInSet() throws Exception { @Test public void testLexerEOFInSet() throws Exception {
LexerGrammar lg = new LexerGrammar( LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"A : 'a' (EOF|'\n') ;\n"); "A : 'a' (EOF|'\\n') ;\n");
checkDeserializationIsStable(lg); checkDeserializationIsStable(lg);
} }

14
tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java
View File

@ -143,7 +143,7 @@ public class TestATNLexerInterpreter extends BaseTest {
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"KEND : 'end' ;\n" + "KEND : 'end' ;\n" +
"ID : 'a'..'z'+ ;\n" + "ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\n')+ ;"); "WS : (' '|'\\n')+ ;");
String expecting = "ID, EOF"; String expecting = "ID, EOF";
//checkLexerMatches(lg, "e", expecting); //checkLexerMatches(lg, "e", expecting);
expecting = "KEND, EOF"; expecting = "KEND, EOF";
@ -159,7 +159,7 @@ public class TestATNLexerInterpreter extends BaseTest {
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"INT : DIGIT+ ;\n" + "INT : DIGIT+ ;\n" +
"fragment DIGIT : '0'..'9' ;\n" + "fragment DIGIT : '0'..'9' ;\n" +
"WS : (' '|'\n')+ ;"); "WS : (' '|'\\n')+ ;");
String expecting = "INT, WS, INT, EOF"; String expecting = "INT, WS, INT, EOF";
checkLexerMatches(lg, "32 99", expecting); checkLexerMatches(lg, "32 99", expecting);
} }
@ -168,7 +168,7 @@ public class TestATNLexerInterpreter extends BaseTest {
LexerGrammar lg = new LexerGrammar( LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"CMT : '/*' (CMT | ~'*')+ '*/' ;\n" + "CMT : '/*' (CMT | ~'*')+ '*/' ;\n" +
"WS : (' '|'\n')+ ;"); "WS : (' '|'\\n')+ ;");
String expecting = "CMT, WS, CMT, EOF"; String expecting = "CMT, WS, CMT, EOF";
checkLexerMatches(lg, "/* ick */\n/* /*nested*/ */", expecting); checkLexerMatches(lg, "/* ick */\n/* /*nested*/ */", expecting);
} }
@ -177,7 +177,7 @@ public class TestATNLexerInterpreter extends BaseTest {
LexerGrammar lg = new LexerGrammar( LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"CMT : '/*' (CMT | .)*? '*/' ;\n" + "CMT : '/*' (CMT | .)*? '*/' ;\n" +
"WS : (' '|'\n')+ ;"); "WS : (' '|'\\n')+ ;");
String expecting = "CMT, WS, CMT, WS, EOF"; String expecting = "CMT, WS, CMT, WS, EOF";
checkLexerMatches(lg, checkLexerMatches(lg,
@ -257,7 +257,7 @@ public class TestATNLexerInterpreter extends BaseTest {
@Test public void testEOFAtEndOfLineComment() throws Exception { @Test public void testEOFAtEndOfLineComment() throws Exception {
LexerGrammar lg = new LexerGrammar( LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"CMT : '//' ~('\n')* ;\n"); "CMT : '//' ~('\\n')* ;\n");
String expecting = "CMT, EOF"; String expecting = "CMT, EOF";
checkLexerMatches(lg, "//x", expecting); checkLexerMatches(lg, "//x", expecting);
} }
@ -265,7 +265,7 @@ public class TestATNLexerInterpreter extends BaseTest {
@Test public void testEOFAtEndOfLineComment2() throws Exception { @Test public void testEOFAtEndOfLineComment2() throws Exception {
LexerGrammar lg = new LexerGrammar( LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"CMT : '//' ~('\n'|'\r')* ;\n"); "CMT : '//' ~('\\n'|'\\r')* ;\n");
String expecting = "CMT, EOF"; String expecting = "CMT, EOF";
checkLexerMatches(lg, "//x", expecting); checkLexerMatches(lg, "//x", expecting);
} }
@ -276,7 +276,7 @@ public class TestATNLexerInterpreter extends BaseTest {
@Test public void testEOFInSetAtEndOfLineComment() throws Exception { @Test public void testEOFInSetAtEndOfLineComment() throws Exception {
LexerGrammar lg = new LexerGrammar( LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"CMT : '//' .* (EOF|'\n') ;\n"); "CMT : '//' .* (EOF|'\\n') ;\n");
String expecting = "CMT, EOF"; String expecting = "CMT, EOF";
checkLexerMatches(lg, "//", expecting); checkLexerMatches(lg, "//", expecting);
} }

2
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -358,7 +358,7 @@ public class TestATNSerialization extends BaseTest {
@Test public void testLexerEOFInSet() throws Exception { @Test public void testLexerEOFInSet() throws Exception {
LexerGrammar lg = new LexerGrammar( LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"INT : 'a' (EOF|'\n') ;\n"); "INT : 'a' (EOF|'\\n') ;\n");
String expecting = String expecting =
"max type 1\n" + "max type 1\n" +
"0:TOKEN_START -1\n" + "0:TOKEN_START -1\n" +

14
tool/test/org/antlr/v4/test/TestLexerExec.java
View File

@ -202,7 +202,7 @@ public class TestLexerExec extends BaseTest {
String grammar = String grammar =
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"CMT : '/*' (CMT | .)*? '*/' ;\n" + "CMT : '/*' (CMT | .)*? '*/' ;\n" +
"WS : (' '|'\n')+ ;\n" "WS : (' '|'\\n')+ ;\n"
/*+ "ANY : .;"*/; /*+ "ANY : .;"*/;
String expecting = String expecting =
@ -240,7 +240,7 @@ public class TestLexerExec extends BaseTest {
String grammar = String grammar =
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"CMT : '/*' (CMT | .)+? '*/' ;\n" + "CMT : '/*' (CMT | .)+? '*/' ;\n" +
"WS : (' '|'\n')+ ;\n" "WS : (' '|'\\n')+ ;\n"
/*+ "ANY : .;"*/; /*+ "ANY : .;"*/;
String expecting = String expecting =
@ -382,7 +382,7 @@ public class TestLexerExec extends BaseTest {
String grammar = String grammar =
"lexer grammar L;\n" + "lexer grammar L;\n" +
"STRING_START : '\"' {pushMode(STRING_MODE); more();} ;\n" + "STRING_START : '\"' {pushMode(STRING_MODE); more();} ;\n" +
"WS : (' '|'\n') {skip();} ;\n"+ "WS : (' '|'\\n') {skip();} ;\n"+
"mode STRING_MODE;\n"+ "mode STRING_MODE;\n"+
"STRING : '\"' {popMode();} ;\n"+ "STRING : '\"' {popMode();} ;\n"+
"ANY : . {more();} ;\n"; "ANY : . {more();} ;\n";
@ -398,7 +398,7 @@ public class TestLexerExec extends BaseTest {
String grammar = String grammar =
"lexer grammar L;\n" + "lexer grammar L;\n" +
"STRING_START : '\"' -> pushMode(STRING_MODE), more ;\n" + "STRING_START : '\"' -> pushMode(STRING_MODE), more ;\n" +
"WS : (' '|'\n') -> skip ;\n"+ "WS : (' '|'\\n') -> skip ;\n"+
"mode STRING_MODE;\n"+ "mode STRING_MODE;\n"+
"STRING : '\"' -> popMode ;\n"+ // token type 2 "STRING : '\"' -> popMode ;\n"+ // token type 2
"ANY : . -> more ;\n"; "ANY : . -> more ;\n";
@ -414,7 +414,7 @@ public class TestLexerExec extends BaseTest {
String grammar = String grammar =
"lexer grammar L;\n" + "lexer grammar L;\n" +
"STRING_START : '\"' -> mode(STRING_MODE), more ;\n" + "STRING_START : '\"' -> mode(STRING_MODE), more ;\n" +
"WS : (' '|'\n') -> skip ;\n"+ "WS : (' '|'\\n') -> skip ;\n"+
"mode STRING_MODE;\n"+ "mode STRING_MODE;\n"+
"STRING : '\"' -> mode(DEFAULT_MODE) ;\n"+ // ttype 2 since '"' ambiguity "STRING : '\"' -> mode(DEFAULT_MODE) ;\n"+ // ttype 2 since '"' ambiguity
"ANY : . -> more ;\n"; "ANY : . -> more ;\n";
@ -431,7 +431,7 @@ public class TestLexerExec extends BaseTest {
"lexer grammar L;\n"+ "lexer grammar L;\n"+
"KEND : 'end' ;\n" + // has priority "KEND : 'end' ;\n" + // has priority
"ID : 'a'..'z'+ ;\n" + "ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\n')+ ;"; "WS : (' '|'\\n')+ ;";
String found = execLexer("L.g4", grammar, "L", "end eend ending a"); String found = execLexer("L.g4", grammar, "L", "end eend ending a");
String expecting = String expecting =
"[@0,0:2='end',<1>,1:0]\n" + "[@0,0:2='end',<1>,1:0]\n" +
@ -454,7 +454,7 @@ public class TestLexerExec extends BaseTest {
"DOT : '.' ;\n" + "DOT : '.' ;\n" +
"ID : 'a'..'z'+ ;\n" + "ID : 'a'..'z'+ ;\n" +
"fragment HexDigit : ('0'..'9'|'a'..'f'|'A'..'F') ;\n" + "fragment HexDigit : ('0'..'9'|'a'..'f'|'A'..'F') ;\n" +
"WS : (' '|'\n')+ ;"; "WS : (' '|'\\n')+ ;";
String found = execLexer("L.g4", grammar, "L", "x 0 1 a.b a.l"); String found = execLexer("L.g4", grammar, "L", "x 0 1 a.b a.l");
String expecting = String expecting =
"[@0,0:0='x',<5>,1:0]\n" + "[@0,0:0='x',<5>,1:0]\n" +