forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #1417 from KvanTTT/command_wrong_id_1411 

ANTLR generates a wrong lexer code if type, channel or mode commands use inproper identifier
This commit is contained in:
Terence Parr 2016-12-03 16:55:23 -08:00 committed by GitHub
parent 2c63e95359 4e2f50aef8
commit f11a380609
4 changed files with 112 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java
View File

@ -229,4 +229,22 @@ public class TestSymbolIssues extends BaseJavaToolTest {
testErrors(test, false);
}
// https://github.com/antlr/antlr4/issues/1411
@Test public void testWrongIdForTypeChannelModeCommand() throws Exception {
String[] test = {
"lexer grammar L;\n" +
"tokens { TOKEN1 }\n" +
"channels { CHANNEL1 }\n" +
"TOKEN: 'asdf' -> type(CHANNEL1), channel(MODE1), mode(TOKEN1);\n" +
"mode MODE1;\n" +
"MODE1_TOKEN: 'qwer';",
"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_TOKEN_NAME.code + "): L.g4:4:22: CHANNEL1 is not a recognized token name\n" +
"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): L.g4:4:41: MODE1 is not a recognized channel name\n" +
"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_MODE_NAME.code + "): L.g4:4:54: TOKEN1 is not a recognized mode name\n"
};
testErrors(test, false);
}
}

6
tool-testsuite/test/org/antlr/v4/test/tool/TestToolSyntaxErrors.java
View File

@ -657,8 +657,8 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest {
"WHITESPACE: [ \\t]+ -> channel(WHITESPACE_CHANNEL);\n";
String expected =
"warning(" + ErrorType.UNKNOWN_LEXER_CONSTANT.code + "): T.g4:10:35: rule COMMENT contains a lexer command with an unrecognized constant value; lexer interpreters may produce incorrect output\n" +
"warning(" + ErrorType.UNKNOWN_LEXER_CONSTANT.code + "): T.g4:11:35: rule WHITESPACE contains a lexer command with an unrecognized constant value; lexer interpreters may produce incorrect output\n" +
"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): T.g4:10:35: COMMENT_CHANNEL is not a recognized channel name\n" +
"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): T.g4:11:35: WHITESPACE_CHANNEL is not a recognized channel name\n" +
"error(" + ErrorType.CHANNELS_BLOCK_IN_COMBINED_GRAMMAR.code + "): T.g4:3:0: custom channels are not supported in combined grammars\n";
String[] pair = { grammar, expected };
@ -686,7 +686,7 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest {
// WHITESPACE_CHANNEL and COMMENT_CHANNEL are defined, but NEWLINE_CHANNEL is not
String expected =
"warning(" + ErrorType.UNKNOWN_LEXER_CONSTANT.code + "): T.g4:10:34: rule NEWLINE contains a lexer command with an unrecognized constant value; lexer interpreters may produce incorrect output\n";
"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): T.g4:10:34: NEWLINE_CHANNEL is not a recognized channel name\n";
String[] pair = { grammar, expected };
super.testErrors(pair, true);

108
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -412,8 +412,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("mode".equals(command) && arg != null) {
String modeName = arg.getText();
checkMode(modeName, arg.token);
Integer mode = getConstantValue(modeName, arg.getToken());
Integer mode = getModeConstantValue(modeName, arg.getToken());
if (mode == null) {
return null;
}
@ -422,8 +421,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("pushMode".equals(command) && arg != null) {
String modeName = arg.getText();
checkMode(modeName, arg.token);
Integer mode = getConstantValue(modeName, arg.getToken());
Integer mode = getModeConstantValue(modeName, arg.getToken());
if (mode == null) {
return null;
}
@ -432,8 +430,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("type".equals(command) && arg != null) {
String typeName = arg.getText();
checkToken(typeName, arg.token);
Integer type = getConstantValue(typeName, arg.getToken());
Integer type = getTokenConstantValue(typeName, arg.getToken());
if (type == null) {
return null;
}
@ -442,8 +439,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("channel".equals(command) && arg != null) {
String channelName = arg.getText();
checkChannel(channelName, arg.token);
Integer channel = getConstantValue(channelName, arg.getToken());
Integer channel = getChannelConstantValue(channelName, arg.getToken());
if (channel == null) {
return null;
}
@ -455,54 +451,84 @@ public class LexerATNFactory extends ParserATNFactory {
}
}
protected void checkMode(String modeName, Token token) {
if (!modeName.equals("DEFAULT_MODE") && COMMON_CONSTANTS.containsKey(modeName)) {
g.tool.errMgr.grammarError(ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
}
}
protected void checkToken(String tokenName, Token token) {
if (!tokenName.equals("EOF") && COMMON_CONSTANTS.containsKey(tokenName)) {
g.tool.errMgr.grammarError(ErrorType.TOKEN_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
}
}
protected void checkChannel(String channelName, Token token) {
if (!channelName.equals("HIDDEN") && !channelName.equals("DEFAULT_TOKEN_CHANNEL") && COMMON_CONSTANTS.containsKey(channelName)) {
g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
}
}
protected Integer getConstantValue(String name, Token token) {
if (name == null) {
private Integer getModeConstantValue(String modeName, Token token) {
if (modeName == null) {
return null;
}
Integer commonConstant = COMMON_CONSTANTS.get(name);
if (commonConstant != null) {
return commonConstant;
if (modeName.equals("DEFAULT_MODE")) {
return Lexer.DEFAULT_MODE;
}
int tokenType = g.getTokenType(name);
List<String> modeNames = new ArrayList<String>(((LexerGrammar)g).modes.keySet());
int mode = modeNames.indexOf(modeName);
if (mode >= 0) {
return mode;
}
else if (COMMON_CONSTANTS.containsKey(modeName)) {
g.tool.errMgr.grammarError(ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
return null;
}
try {
return Integer.parseInt(modeName);
} catch (NumberFormatException ex) {
g.tool.errMgr.grammarError(ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_MODE_NAME, g.fileName, token, token.getText());
return null;
}
}
private Integer getTokenConstantValue(String tokenName, Token token) {
if (tokenName == null) {
return null;
}
if (tokenName.equals("EOF")) {
return Lexer.EOF;
}
int tokenType = g.getTokenType(tokenName);
if (tokenType != org.antlr.v4.runtime.Token.INVALID_TYPE) {
return tokenType;
}
else if (COMMON_CONSTANTS.containsKey(tokenName)) {
g.tool.errMgr.grammarError(ErrorType.TOKEN_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
return null;
}
int channelValue = g.getChannelValue(name);
try {
return Integer.parseInt(tokenName);
} catch (NumberFormatException ex) {
g.tool.errMgr.grammarError(ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_TOKEN_NAME, g.fileName, token, token.getText());
return null;
}
}
private Integer getChannelConstantValue(String channelName, Token token) {
if (channelName == null) {
return null;
}
if (channelName.equals("HIDDEN")) {
return Lexer.HIDDEN;
}
else if (channelName.equals("DEFAULT_TOKEN_CHANNEL")) {
return Lexer.DEFAULT_TOKEN_CHANNEL;
}
else if (COMMON_CONSTANTS.containsKey(channelName)) {
g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
return null;
}
int channelValue = g.getChannelValue(channelName);
if (channelValue >= org.antlr.v4.runtime.Token.MIN_USER_CHANNEL_VALUE) {
return channelValue;
}
List<String> modeNames = new ArrayList<String>(((LexerGrammar)g).modes.keySet());
int mode = modeNames.indexOf(name);
if (mode >= 0) {
return mode;
}
try {
return Integer.parseInt(name);
return Integer.parseInt(channelName);
} catch (NumberFormatException ex) {
g.tool.errMgr.grammarError(ErrorType.UNKNOWN_LEXER_CONSTANT, g.fileName, token, currentRule.name, token != null ? token.getText() : null);
g.tool.errMgr.grammarError(ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME, g.fileName, token, token.getText());
return null;
}
}

24
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -1011,6 +1011,30 @@ public enum ErrorType {
* <pre>C: 'test' '';</pre>
*/
EMPTY_STRINGS_NOT_ALLOWED(174, "string literals cannot be empty", ErrorSeverity.ERROR),
/**
* Compiler Error 175.
*
* <p><em>name</em> is not a recognized token name</p>
*
* <pre>TOKEN: 'a' -> type(CHANNEL1); // error 175</pre>
*/
CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_TOKEN_NAME(175, "<arg> is not a recognized token name", ErrorSeverity.ERROR),
/**
* Compiler Error 176.
*
* <p><em>name</em>is not a recognized mode name</p>
*
* <pre>TOKEN: 'a' -> channel(MODE1); // error 176</pre>
*/
CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_MODE_NAME(176, "<arg> is not a recognized mode name", ErrorSeverity.ERROR),
/**
* Compiler Error 177.
*
* <p><em>name</em> is not a recognized channel name</p>
*
* <pre>TOKEN: 'a' -> mode(TOKEN1); // error 177</pre>
*/
CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME(177, "<arg> is not a recognized channel name", ErrorSeverity.ERROR),
/*
* Backward incompatibility errors