forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #1059 from parrt/KvanTTT-master 

Kvan ttt master
This commit is contained in:
Terence Parr 2015-12-08 11:39:09 -08:00
parent 6068a4a2c7 1438ab817f
commit 01e6923dbd
10 changed files with 204 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
contributors.txt
View File

@ -80,6 +80,7 @@ YYYY/MM/DD, github id, Full name, email
2015/09/24, HSorensen, Henrik Sorensen, henrik.b.sorensen@gmail.com
2015/10/06, brwml, Bryan Wilhelm, bryan.wilhelm@microsoft.com
2015/10/08, fedotovalex, Alex Fedotov, me@alexfedotov.com
2015/10/12, KvanTTT, Ivan Kochurkin, ivan.kochurkin@gmail.com
2015/10/21, martin-probst, Martin Probst, martin-probst@web.de
2015/10/21, hkff, Walid Benghabrit, walid.benghabrit@mines-nantes.fr
2015/11/25, abego, Udo Borkowski, ub@abego.org
2015/11/25, abego, Udo Borkowski, ub@abego.org

2
doc/index.md
View File

@ -4,7 +4,7 @@ Please check [Frequently asked questions (FAQ)](faq/index.md) before asking ques
Notes:
<ul>
<li>To add to or improve this FAQ, <a href=https://help.github.com/articles/fork-a-repo>fork</a> the <a href=https://github.com/antlr/antlr4>antlr/antlr4 repo</a> then update this `doc/index.md` or file(s) in that directory. Submit a <a href=https://help.github.com/articles/creating-a-pull-request>pull request</a> to get your changes incorporated into the main repository. Do not mix code and FAQ updates in the sample pull request. <b>You must sign the contributors.txt certificate of origin with your pull request if you've not done so before.</b></li>
<li>To add to or improve this documentation, <a href=https://help.github.com/articles/fork-a-repo>fork</a> the <a href=https://github.com/antlr/antlr4>antlr/antlr4 repo</a> then update this `doc/index.md` or file(s) in that directory. Submit a <a href=https://help.github.com/articles/creating-a-pull-request>pull request</a> to get your changes incorporated into the main repository. Do not mix code and documentation updates in the sample pull request. <b>You must sign the contributors.txt certificate of origin with your pull request if you've not done so before.</b></li>
<li>Copyright © 2012, The Pragmatic Bookshelf. Pragmatic Bookshelf grants a nonexclusive, irrevocable, royalty-free, worldwide license to reproduce, distribute, prepare derivative works, and otherwise use this contribution as part of the ANTLR project and associated documentation.</li>

2
tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java
View File

@ -632,7 +632,7 @@ public class TestATNSerialization extends BaseTest {
"lexer grammar L;\n"+
"A : 'a'\n ;\n" +
"B : 'b';\n" +
"mode A;\n" +
"mode M;\n" +
"C : 'c';\n"+
"D : 'd';\n");
String expecting =

54
tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java
View File

@ -116,10 +116,25 @@ public class TestSymbolIssues extends BaseTest {
"warning(" + ErrorType.TOKEN_NAME_REASSIGNMENT.code + "): E.g4:3:4: token name A is already defined\n"
};
static String[] F = {
// INPUT
"lexer grammar F;\n" +
"A: 'a';\n" +
"mode M1;\n" +
"A1: 'a';\n" +
"mode M2;\n" +
"A2: 'a';\n" +
"M1: 'b';\n",
// YIELDS
"error(" + ErrorType.MODE_CONFLICTS_WITH_TOKEN.code + "): F.g4:3:0: mode M1 conflicts with token with same name\n"
};
@Test public void testA() { super.testErrors(A, false); }
@Test public void testB() { super.testErrors(B, false); }
@Test public void testD() { super.testErrors(D, false); }
@Test public void testE() { super.testErrors(E, false); }
@Test public void testF() { super.testErrors(F, false); }
@Test public void testStringLiteralRedefs() throws Exception {
String grammar =
@ -169,4 +184,43 @@ public class TestSymbolIssues extends BaseTest {
testErrors(test, false);
}
@Test public void testTokensModesChannelsDeclarationConflictsWithReserved() throws Exception {
String[] test = {
"lexer grammar L;\n" +
"channels { SKIP, HIDDEN, channel0 }\n" +
"A: 'a';\n" +
"mode MAX_CHAR_VALUE;\n" +
"MIN_CHAR_VALUE: 'a';\n" +
"mode DEFAULT_MODE;\n" +
"B: 'b';\n" +
"mode M;\n" +
"C: 'c';",
"error(" + ErrorType.RESERVED_RULE_NAME.code + "): L.g4:5:0: cannot declare a rule with reserved name MIN_CHAR_VALUE\n" +
"error(" + ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS.code + "): L.g4:4:0: cannot use or declare mode with reserved name MAX_CHAR_VALUE\n" +
"error(" + ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS.code + "): L.g4:2:11: cannot use or declare channel with reserved name SKIP\n" +
"error(" + ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS.code + "): L.g4:2:17: cannot use or declare channel with reserved name HIDDEN\n"
};
testErrors(test, false);
}
@Test public void testTokensModesChannelsUsingConflictsWithReserved() throws Exception {
String[] test = {
"lexer grammar L;\n" +
"A: 'a' -> channel(SKIP);\n" +
"B: 'b' -> type(MORE);\n" +
"C: 'c' -> mode(SKIP);\n" +
"D: 'd' -> channel(HIDDEN);\n" +
"E: 'e' -> type(EOF);\n" +
"F: 'f' -> pushMode(DEFAULT_MODE);",
"error(" + ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS.code + "): L.g4:2:18: cannot use or declare channel with reserved name SKIP\n" +
"error(" + ErrorType.TOKEN_CONFLICTS_WITH_COMMON_CONSTANTS.code + "): L.g4:3:15: cannot use or declare token with reserved name MORE\n" +
"error(" + ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS.code + "): L.g4:4:15: cannot use or declare mode with reserved name SKIP\n"
};
testErrors(test, false);
}
}

27
tool-testsuite/test/org/antlr/v4/test/tool/TestToolSyntaxErrors.java
View File

@ -454,6 +454,33 @@ public class TestToolSyntaxErrors extends BaseTest {
super.testErrors(pair, true);
}
/**
* This is a regression test for antlr/antlr4#959 "NullPointerException".
* https://github.com/antlr/antlr4/issues/959
*/
@Test public void testNotAllowedEmptyStrings() {
String grammar =
"lexer grammar T;\n" +
"Error0: '''test''';\n" +
"Error1: '' 'test';\n" +
"Error2: 'test' '';\n" +
"Error3: '';\n" +
"NotError: ' ';";
String expected =
"error(" + ErrorType.EMPTY_STRINGS_NOT_ALLOWED.code + "): T.g4:2:8: string literals cannot be empty\n" +
"error(" + ErrorType.EMPTY_STRINGS_NOT_ALLOWED.code + "): T.g4:2:16: string literals cannot be empty\n" +
"error(" + ErrorType.EMPTY_STRINGS_NOT_ALLOWED.code + "): T.g4:3:8: string literals cannot be empty\n" +
"error(" + ErrorType.EMPTY_STRINGS_NOT_ALLOWED.code + "): T.g4:4:15: string literals cannot be empty\n" +
"error(" + ErrorType.EMPTY_STRINGS_NOT_ALLOWED.code + "): T.g4:5:8: string literals cannot be empty\n";
String[] pair = new String[] {
grammar,
expected
};
super.testErrors(pair, true);
}
/**
* This test ensures the {@link ErrorType#UNRECOGNIZED_ASSOC_OPTION} warning
* is produced as described in the documentation.

31
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -83,7 +83,7 @@ public class LexerATNFactory extends ParserATNFactory {
* actions, but are required during code generation for creating
* {@link LexerAction} instances that are usable by a lexer interpreter.
*/
protected static final Map<String, Integer> COMMON_CONSTANTS = new HashMap<String, Integer>();
public static final Map<String, Integer> COMMON_CONSTANTS = new HashMap<String, Integer>();
static {
COMMON_CONSTANTS.put("HIDDEN", Lexer.HIDDEN);
COMMON_CONSTANTS.put("DEFAULT_TOKEN_CHANNEL", Lexer.DEFAULT_TOKEN_CHANNEL);
@ -112,6 +112,10 @@ public class LexerATNFactory extends ParserATNFactory {
codegenTemplates = gen.getTemplates();
}
public static Set<String> getCommonConstants() {
return COMMON_CONSTANTS.keySet();
}
@Override
public ATN createATN() {
// BUILD ALL START STATES (ONE PER MODE)
@ -243,8 +247,8 @@ public class LexerATNFactory extends ParserATNFactory {
// fall back to standard action generation for the command
ST cmdST = codegenTemplates.getInstanceOf("Lexer" +
CharSupport.capitalize(ID.getText())+
"Command");
CharSupport.capitalize(ID.getText()) +
"Command");
if (cmdST == null) {
g.tool.errMgr.grammarError(ErrorType.INVALID_LEXER_COMMAND, g.fileName, ID.token, ID.getText());
return epsilon(ID);
@ -407,6 +411,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("mode".equals(command) && arg != null) {
String modeName = arg.getText();
checkMode(modeName, arg.token);
Integer mode = getConstantValue(modeName, arg.getToken());
if (mode == null) {
return null;
@ -416,6 +421,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("pushMode".equals(command) && arg != null) {
String modeName = arg.getText();
checkMode(modeName, arg.token);
Integer mode = getConstantValue(modeName, arg.getToken());
if (mode == null) {
return null;
@ -425,6 +431,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("type".equals(command) && arg != null) {
String typeName = arg.getText();
checkToken(typeName, arg.token);
Integer type = getConstantValue(typeName, arg.getToken());
if (type == null) {
return null;
@ -434,6 +441,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
else if ("channel".equals(command) && arg != null) {
String channelName = arg.getText();
checkChannel(channelName, arg.token);
Integer channel = getConstantValue(channelName, arg.getToken());
if (channel == null) {
return null;
@ -446,6 +454,23 @@ public class LexerATNFactory extends ParserATNFactory {
}
}
protected void checkMode(String modeName, Token token) {
if (!modeName.equals("DEFAULT_MODE") && COMMON_CONSTANTS.containsKey(modeName)) {
g.tool.errMgr.grammarError(ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
}
}
protected void checkToken(String tokenName, Token token) {
if (!tokenName.equals("EOF") && COMMON_CONSTANTS.containsKey(tokenName)) {
g.tool.errMgr.grammarError(ErrorType.TOKEN_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
}
}
protected void checkChannel(String channelName, Token token) {
if (!channelName.equals("HIDDEN") && !channelName.equals("DEFAULT_TOKEN_CHANNEL") && COMMON_CONSTANTS.containsKey(channelName)) {
g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, token, token.getText());
}
}
protected Integer getConstantValue(String name, Token token) {
if (name == null) {

10
tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java
View File

@ -349,7 +349,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
GrammarAST root = (GrammarAST)rulesNode.getParent();
GrammarAST IDNode = (GrammarAST)root.getChild(0);
g.tool.errMgr.grammarError(ErrorType.NO_RULES, g.fileName,
null, IDNode.getText(), g);
null, IDNode.getText(), g);
}
}
@ -491,6 +491,14 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
label.getText());
}
@Override
protected void enterTerminal(GrammarAST tree) {
String text = tree.getText();
if (text.equals("''")) {
g.tool.errMgr.grammarError(ErrorType.EMPTY_STRINGS_NOT_ALLOWED, g.fileName, tree.token);
}
}
/** Check option is appropriate for grammar, rule, subrule */
boolean checkOptions(GrammarAST parent,
Token optionID,

7
tool/src/org/antlr/v4/semantics/SemanticPipeline.java
View File

@ -31,6 +31,7 @@
package org.antlr.v4.semantics;
import org.antlr.v4.analysis.LeftRecursiveRuleTransformer;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.Pair;
@ -128,6 +129,8 @@ public class SemanticPipeline {
collector.tokenIDRefs, collector.terminals);
}
symcheck.checkForModeConflicts(g);
assignChannelTypes(g, collector.channelDefs);
// CHECK RULE REFS NOW (that we've defined rules in grammar)
@ -287,6 +290,10 @@ public class SemanticPipeline {
g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_TOKEN, g.fileName, channel.token, channelName);
}
if (LexerATNFactory.COMMON_CONSTANTS.containsKey(channelName)) {
g.tool.errMgr.grammarError(ErrorType.CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, channel.token, channelName);
}
if (outermost instanceof LexerGrammar) {
LexerGrammar lexerGrammar = (LexerGrammar)outermost;
if (lexerGrammar.modes.containsKey(channelName)) {

22
tool/src/org/antlr/v4/semantics/SymbolChecks.java
View File

@ -30,7 +30,9 @@
package org.antlr.v4.semantics;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.tool.Alternative;
import org.antlr.v4.tool.Attribute;
import org.antlr.v4.tool.AttributeDict;
@ -38,6 +40,7 @@ import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.GrammarAST;
@ -67,7 +70,7 @@ public class SymbolChecks {
protected final Set<String> reservedNames = new HashSet<String>();
{
reservedNames.add("EOF");
reservedNames.addAll(LexerATNFactory.getCommonConstants());
}
public SymbolChecks(Grammar g, SymbolCollector collector) {
@ -269,6 +272,23 @@ public class SymbolChecks {
}
}
public void checkForModeConflicts(Grammar g) {
if (g.isLexer()) {
LexerGrammar lexerGrammar = (LexerGrammar)g;
for (String modeName : lexerGrammar.modes.keySet()) {
if (!modeName.equals("DEFAULT_MODE") && reservedNames.contains(modeName)) {
Rule rule = lexerGrammar.modes.get(modeName).iterator().next();
g.tool.errMgr.grammarError(ErrorType.MODE_CONFLICTS_WITH_COMMON_CONSTANTS, g.fileName, rule.ast.parent.getToken(), modeName);
}
if (g.getTokenType(modeName) != Token.INVALID_TYPE) {
Rule rule = lexerGrammar.modes.get(modeName).iterator().next();
g.tool.errMgr.grammarError(ErrorType.MODE_CONFLICTS_WITH_TOKEN, g.fileName, rule.ast.parent.getToken(), modeName);
}
}
}
}
// CAN ONLY CALL THE TWO NEXT METHODS AFTER GRAMMAR HAS RULE DEFS (see semanticpipeline)
public void checkRuleArgs(Grammar g, List<GrammarAST> rulerefs) {

54
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -957,6 +957,60 @@ public enum ErrorType {
CHANNELS_BLOCK_IN_COMBINED_GRAMMAR(164, "custom channels are not supported in combined grammars", ErrorSeverity.ERROR),
NONCONFORMING_LR_RULE(169, "rule <arg> is left recursive but doesn't conform to a pattern ANTLR can handle", ErrorSeverity.ERROR),
/**
* Compiler Error 170.
*
* <pre>
* mode M1;
* A1: 'a'; // ok
* mode M2;
* A2: 'a'; // ok
* M1: 'b'; // error 170
* </pre>
*
* <p>mode <em>name</em> conflicts with token with same name</p>
*/
MODE_CONFLICTS_WITH_TOKEN(170, "mode <arg> conflicts with token with same name", ErrorSeverity.ERROR),
/**
* Compiler Error 171.
*
* <p>can not use or declare token with reserved name</p>
*
* <p>Reserved names: HIDDEN, DEFAULT_TOKEN_CHANNEL, SKIP, MORE, MAX_CHAR_VALUE, MIN_CHAR_VALUE.
*
* <p>Can be used but cannot be declared: EOF</p>
*/
TOKEN_CONFLICTS_WITH_COMMON_CONSTANTS(171, "cannot use or declare token with reserved name <arg>", ErrorSeverity.ERROR),
/**
* Compiler Error 172.
*
* <p>can not use or declare channel with reserved name</p>
*
* <p>Reserved names: DEFAULT_MODE, SKIP, MORE, EOF, MAX_CHAR_VALUE, MIN_CHAR_VALUE.
*
* <p>Can be used but cannot be declared: HIDDEN, DEFAULT_TOKEN_CHANNEL</p>
*/
CHANNEL_CONFLICTS_WITH_COMMON_CONSTANTS(172, "cannot use or declare channel with reserved name <arg>", ErrorSeverity.ERROR),
/**
* Compiler Error 173.
*
* <p>can not use or declare mode with reserved name</p>
*
* <p>Reserved names: HIDDEN, DEFAULT_TOKEN_CHANNEL, SKIP, MORE, MAX_CHAR_VALUE, MIN_CHAR_VALUE.
*
* <p>Can be used and cannot declared: DEFAULT_MODE</p>
*/
MODE_CONFLICTS_WITH_COMMON_CONSTANTS(173, "cannot use or declare mode with reserved name <arg>", ErrorSeverity.ERROR),
/**
* Compiler Error 174.
*
* <p>empty strings not allowed</p>
*
* <pre>A: '''test''';</pre>
* <pre>B: '';</pre>
* <pre>C: 'test' '';</pre>
*/
EMPTY_STRINGS_NOT_ALLOWED(174, "string literals cannot be empty", ErrorSeverity.ERROR),
/*
* Backward incompatibility errors