Merge pull request #1417 from KvanTTT/command_wrong_id_1411

ANTLR generates a wrong lexer code if type, channel or mode commands use inproper identifier

tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java

@@ -229,4 +229,22 @@ public class TestSymbolIssues extends BaseJavaToolTest {
 
 		testErrors(test, false);
 	}
+
+	// https://github.com/antlr/antlr4/issues/1411
+	@Test public void testWrongIdForTypeChannelModeCommand() throws Exception {
+		String[] test = {
+			"lexer grammar L;\n" +
+			"tokens { TOKEN1 }\n" +
+			"channels { CHANNEL1 }\n" +
+			"TOKEN: 'asdf' -> type(CHANNEL1), channel(MODE1), mode(TOKEN1);\n" +
+			"mode MODE1;\n" +
+			"MODE1_TOKEN: 'qwer';",
+
+			"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_TOKEN_NAME.code + "): L.g4:4:22: CHANNEL1 is not a recognized token name\n" +
+			"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): L.g4:4:41: MODE1 is not a recognized channel name\n" +
+			"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_MODE_NAME.code + "): L.g4:4:54: TOKEN1 is not a recognized mode name\n"
+		};
+
+		testErrors(test, false);
+	}
 }

tool-testsuite/test/org/antlr/v4/test/tool/TestToolSyntaxErrors.java

@@ -657,8 +657,8 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest {
 			"WHITESPACE: [ \\t]+      -> channel(WHITESPACE_CHANNEL);\n";
 
 		String expected =
-			"warning(" + ErrorType.UNKNOWN_LEXER_CONSTANT.code + "): T.g4:10:35: rule COMMENT contains a lexer command with an unrecognized constant value; lexer interpreters may produce incorrect output\n" +
+			"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): T.g4:10:35: COMMENT_CHANNEL is not a recognized channel name\n" +
-			"warning(" + ErrorType.UNKNOWN_LEXER_CONSTANT.code + "): T.g4:11:35: rule WHITESPACE contains a lexer command with an unrecognized constant value; lexer interpreters may produce incorrect output\n" +
+			"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): T.g4:11:35: WHITESPACE_CHANNEL is not a recognized channel name\n" +
 			"error(" + ErrorType.CHANNELS_BLOCK_IN_COMBINED_GRAMMAR.code + "): T.g4:3:0: custom channels are not supported in combined grammars\n";
 
 		String[] pair = { grammar, expected };
@@ -686,7 +686,7 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest {
 
 		// WHITESPACE_CHANNEL and COMMENT_CHANNEL are defined, but NEWLINE_CHANNEL is not
 		String expected =
-			"warning(" + ErrorType.UNKNOWN_LEXER_CONSTANT.code + "): T.g4:10:34: rule NEWLINE contains a lexer command with an unrecognized constant value; lexer interpreters may produce incorrect output\n";
+			"error(" + ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME.code + "): T.g4:10:34: NEWLINE_CHANNEL is not a recognized channel name\n";
 
 		String[] pair = { grammar, expected };
 		super.testErrors(pair, true);

tool/src/org/antlr/v4/automata/LexerATNFactory.java

@@ -412,8 +412,7 @@ public class LexerATNFactory extends ParserATNFactory {
 		}
 		else if ("mode".equals(command) && arg != null) {
 			String modeName = arg.getText();
-			checkMode(modeName, arg.token);
+			Integer mode = getModeConstantValue(modeName, arg.getToken());
-			Integer mode = getConstantValue(modeName, arg.getToken());
 			if (mode == null) {
 				return null;
 			}
@@ -422,8 +421,7 @@ public class LexerATNFactory extends ParserATNFactory {
 		}
 		else if ("pushMode".equals(command) && arg != null) {
 			String modeName = arg.getText();
-			checkMode(modeName, arg.token);
+			Integer mode = getModeConstantValue(modeName, arg.getToken());
-			Integer mode = getConstantValue(modeName, arg.getToken());
 			if (mode == null) {
 				return null;
 			}
@@ -432,8 +430,7 @@ public class LexerATNFactory extends ParserATNFactory {
 		}
 		else if ("type".equals(command) && arg != null) {
 			String typeName = arg.getText();
-			checkToken(typeName, arg.token);
+			Integer type = getTokenConstantValue(typeName, arg.getToken());
-			Integer type = getConstantValue(typeName, arg.getToken());
 			if (type == null) {
 				return null;
 			}
@@ -442,8 +439,7 @@ public class LexerATNFactory extends ParserATNFactory {
 		}
 		else if ("channel".equals(command) && arg != null) {
 			String channelName = arg.getText();
-			checkChannel(channelName, arg.token);
+			Integer channel = getChannelConstantValue(channelName, arg.getToken());
-			Integer channel = getConstantValue(channelName, arg.getToken());
 			if (channel == null) {
 				return null;
 			}
@@ -455,54 +451,84 @@ public class LexerATNFactory extends ParserATNFactory {
 		}
 	}
 
-	protected void checkMode(String modeName, Token token) {
+	private Integer getModeConstantValue(String modeName, Token token) {
-		if (!modeName.equals("DEFAULT_MODE") && COMMON_CONSTANTS.containsKey(modeName)) {
+		if (modeName == null) {
-			g.tool.errMgr.grammarError(ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
-		}
-	}
-
-	protected void checkToken(String tokenName, Token token) {
-		if (!tokenName.equals("EOF") && COMMON_CONSTANTS.containsKey(tokenName)) {
-			g.tool.errMgr.grammarError(ErrorType.TOKEN_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
-		}
-	}
-
-	protected void checkChannel(String channelName, Token token) {
-		if (!channelName.equals("HIDDEN") && !channelName.equals("DEFAULT_TOKEN_CHANNEL") && COMMON_CONSTANTS.containsKey(channelName)) {
-			g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
-		}
-	}
-
-	protected Integer getConstantValue(String name, Token token) {
-		if (name == null) {
 			return null;
 		}
 
-		Integer commonConstant = COMMON_CONSTANTS.get(name);
+		if (modeName.equals("DEFAULT_MODE")) {
-		if (commonConstant != null) {
+			return Lexer.DEFAULT_MODE;
-			return commonConstant;
 		}
 
-		int tokenType = g.getTokenType(name);
+		List<String> modeNames = new ArrayList<String>(((LexerGrammar)g).modes.keySet());
+		int mode = modeNames.indexOf(modeName);
+		if (mode >= 0) {
+			return mode;
+		}
+		else if (COMMON_CONSTANTS.containsKey(modeName)) {
+			g.tool.errMgr.grammarError(ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
+			return null;
+		}
+
+		try {
+			return Integer.parseInt(modeName);
+		} catch (NumberFormatException ex) {
+			g.tool.errMgr.grammarError(ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_MODE_NAME, g.fileName, token, token.getText());
+			return null;
+		}
+	}
+
+	private Integer getTokenConstantValue(String tokenName, Token token) {
+		if (tokenName == null) {
+			return null;
+		}
+
+		if (tokenName.equals("EOF")) {
+			return Lexer.EOF;
+		}
+
+		int tokenType = g.getTokenType(tokenName);
 		if (tokenType != org.antlr.v4.runtime.Token.INVALID_TYPE) {
 			return tokenType;
 		}
+		else if (COMMON_CONSTANTS.containsKey(tokenName)) {
+			g.tool.errMgr.grammarError(ErrorType.TOKEN_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
+			return null;
+		}
 
-		int channelValue = g.getChannelValue(name);
+		try {
+			return Integer.parseInt(tokenName);
+		} catch (NumberFormatException ex) {
+			g.tool.errMgr.grammarError(ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_TOKEN_NAME, g.fileName, token, token.getText());
+			return null;
+		}
+	}
+
+	private Integer getChannelConstantValue(String channelName, Token token) {
+		if (channelName == null) {
+			return null;
+		}
+
+		if (channelName.equals("HIDDEN")) {
+			return Lexer.HIDDEN;
+		}
+		else if (channelName.equals("DEFAULT_TOKEN_CHANNEL")) {
+			return Lexer.DEFAULT_TOKEN_CHANNEL;
+		}
+		else if (COMMON_CONSTANTS.containsKey(channelName)) {
+			g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
+			return null;
+		}
+
+		int channelValue = g.getChannelValue(channelName);
 		if (channelValue >= org.antlr.v4.runtime.Token.MIN_USER_CHANNEL_VALUE) {
 			return channelValue;
 		}
 
-		List<String> modeNames = new ArrayList<String>(((LexerGrammar)g).modes.keySet());
-		int mode = modeNames.indexOf(name);
-		if (mode >= 0) {
-			return mode;
-		}
-
 		try {
-			return Integer.parseInt(name);
+			return Integer.parseInt(channelName);
 		} catch (NumberFormatException ex) {
-			g.tool.errMgr.grammarError(ErrorType.UNKNOWN_LEXER_CONSTANT, g.fileName, token, currentRule.name, token != null ? token.getText() : null);
+			g.tool.errMgr.grammarError(ErrorType.CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME, g.fileName, token, token.getText());
 			return null;
 		}
 	}

tool/src/org/antlr/v4/tool/ErrorType.java

@@ -1011,6 +1011,30 @@ public enum ErrorType {
 	 * <pre>C: 'test' '';</pre>
 	 */
 	EMPTY_STRINGS_NOT_ALLOWED(174, "string literals cannot be empty", ErrorSeverity.ERROR),
+	/**
+	 * Compiler Error 175.
+	 *
+	 * <p><em>name</em> is not a recognized token name</p>
+	 *
+	 * <pre>TOKEN: 'a' -> type(CHANNEL1); // error 175</pre>
+	 */
+	CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_TOKEN_NAME(175, "<arg> is not a recognized token name", ErrorSeverity.ERROR),
+	/**
+	 * Compiler Error 176.
+	 *
+	 * <p><em>name</em>is not a recognized mode name</p>
+	 *
+	 * <pre>TOKEN: 'a' -> channel(MODE1); // error 176</pre>
+	 */
+	CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_MODE_NAME(176, "<arg> is not a recognized mode name", ErrorSeverity.ERROR),
+	/**
+	 * Compiler Error 177.
+	 *
+	 * <p><em>name</em> is not a recognized channel name</p>
+	 *
+	 * <pre>TOKEN: 'a' -> mode(TOKEN1); // error 177</pre>
+	 */
+	CONSTANT_VALUE_IS_NOT_A_RECOGNIZED_CHANNEL_NAME(177, "<arg> is not a recognized channel name", ErrorSeverity.ERROR),
 
 	/*
 	 * Backward incompatibility errors