jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge branch 'implicit-tokens' of git://github.com/sharwell/antlr4

This commit is contained in:
Terence Parr 2012-04-04 13:11:02 -07:00
parent 8658be69ba 251490cfc9
commit 8155b1138e
7 changed files with 85 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
tool/src/org/antlr/v4/semantics/SemanticPipeline.java
View File

@ -31,6 +31,7 @@ package org.antlr.v4.semantics;
import org.antlr.v4.analysis.LeftRecursiveRuleTransformer;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.GrammarAST;
@ -116,7 +117,7 @@ public class SemanticPipeline {
}
else {
assignTokenTypes(g, collector.tokensDefs,
collector.tokenIDRefs, collector.strings);
collector.tokenIDRefs, collector.terminals);
}
// CHECK RULE REFS NOW (that we've defined rules in grammar)
@ -163,7 +164,7 @@ public class SemanticPipeline {
}
void assignTokenTypes(Grammar g, List<GrammarAST> tokensDefs,
List<GrammarAST> tokenIDs, Set<String> strings)
List<GrammarAST> tokenIDs, List<GrammarAST> terminals)
{
//Grammar G = g.getOutermostGrammar(); // put in root, even if imported
@ -174,6 +175,9 @@ public class SemanticPipeline {
String lit = alias.getChild(1).getText();
g.defineTokenAlias(name, lit);
}
else {
g.defineTokenName(alias.getText());
}
}
// DEFINE TOKEN TYPES FOR X : 'x' ; RULES
@ -187,10 +191,25 @@ public class SemanticPipeline {
*/
// DEFINE TOKEN TYPES FOR TOKEN REFS LIKE ID, INT
for (GrammarAST idAST : tokenIDs) { g.defineTokenName(idAST.getText()); }
for (GrammarAST idAST : tokenIDs) {
if (g.getTokenType(idAST.getText()) == Token.INVALID_TYPE) {
g.tool.errMgr.grammarError(ErrorType.IMPLICIT_TOKEN_DEFINITION, g.fileName, idAST.token, idAST.getText());
}
g.defineTokenName(idAST.getText());
}
// VERIFY TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
for (GrammarAST termAST : terminals) {
if (termAST.getType() != ANTLRParser.STRING_LITERAL) {
continue;
}
if (g.getTokenType(termAST.getText()) == Token.INVALID_TYPE) {
g.tool.errMgr.grammarError(ErrorType.IMPLICIT_STRING_DEFINITION, g.fileName, termAST.token, termAST.getText());
}
}
// DEFINE TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
for (String s : strings) { g.defineStringLiteral(s); }
g.tool.log("semantics", "tokens="+g.tokenNameToTypeMap);
g.tool.log("semantics", "strings="+g.stringLiteralToTypeMap);
}

3
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -144,6 +144,9 @@ public enum ErrorType {
RULE_WITH_TOO_FEW_ALT_LABELS(122, "rule <arg>: must label all alternatives or none", ErrorSeverity.ERROR),
ALT_LABEL_REDEF(123, "rule alt label <arg> redefined in rule <arg2>, originally in <arg3>", ErrorSeverity.ERROR),
ALT_LABEL_CONFLICTS_WITH_RULE(124, "rule alt label <arg> conflicts with rule <arg2>", ErrorSeverity.ERROR),
IMPLICIT_TOKEN_DEFINITION(125, "implicit definition of token <arg> in parser", ErrorSeverity.WARNING),
IMPLICIT_STRING_DEFINITION(126, "cannot create implicit token for string literal <arg> in non-combined grammar", ErrorSeverity.ERROR),
/** Documentation comment is unterminated */
//UNTERMINATED_DOC_COMMENT(, "", ErrorSeverity.ERROR),

4
tool/test/org/antlr/v4/test/TestATNConstruction.java
View File

@ -360,14 +360,14 @@ public class TestATNConstruction extends BaseTest {
@Test public void testNestedAstar() throws Exception {
Grammar g = new Grammar(
"parser grammar P;\n"+
"a : (',' ID*)*;");
"a : (COMMA ID*)*;");
String expecting =
"RuleStart_a_0->StarLoopEntry_13\n" +
"StarLoopEntry_13->StarBlockStart_11\n" +
"StarLoopEntry_13->s14\n" +
"StarBlockStart_11->s2\n" +
"s14->RuleStop_a_1\n" +
"s2-','->StarLoopEntry_8\n" +
"s2-COMMA->StarLoopEntry_8\n" +
"RuleStop_a_1-EOF->s16\n" +
"StarLoopEntry_8->StarBlockStart_6\n" +
"StarLoopEntry_8->s9\n" +

60
tool/test/org/antlr/v4/test/TestAttributeChecks.java
View File

@ -9,6 +9,7 @@ public class TestAttributeChecks extends BaseTest {
String attributeTemplate =
"parser grammar A;\n"+
"@members {<members>}\n" +
"tokens{ID;}\n" +
"a[int x] returns [int y]\n" +
"@init {<init>}\n" +
" : id=ID ids+=ID lab=b[34] {\n" +
@ -36,8 +37,8 @@ public class TestAttributeChecks extends BaseTest {
"$lab.e", "",
"$ids", "",
"$c", "error(63): A.g4:4:8: unknown attribute reference c in $c\n",
"$a.q", "error(65): A.g4:4:10: unknown attribute q for rule a in $a.q\n",
"$c", "error(63): A.g4:5:8: unknown attribute reference c in $c\n",
"$a.q", "error(65): A.g4:5:10: unknown attribute q for rule a in $a.q\n",
};
String[] inlineChecks = {
@ -58,19 +59,19 @@ public class TestAttributeChecks extends BaseTest {
};
String[] bad_inlineChecks = {
"$lab", "error(67): A.g4:6:4: missing attribute access on rule reference lab in $lab\n",
"$q", "error(63): A.g4:6:4: unknown attribute reference q in $q\n",
"$q.y", "error(63): A.g4:6:4: unknown attribute reference q in $q.y\n",
"$q = 3", "error(63): A.g4:6:4: unknown attribute reference q in $q\n",
"$q = 3;", "error(63): A.g4:6:4: unknown attribute reference q in $q = 3;\n",
"$q.y = 3;", "error(63): A.g4:6:4: unknown attribute reference q in $q.y = 3;\n",
"$q = $blort;", "error(63): A.g4:6:4: unknown attribute reference q in $q = $blort;\n" +
"error(63): A.g4:6:9: unknown attribute reference blort in $blort\n",
"$a.ick", "error(65): A.g4:6:6: unknown attribute ick for rule a in $a.ick\n",
"$a.ick = 3;", "error(65): A.g4:6:6: unknown attribute ick for rule a in $a.ick = 3;\n",
"$b.d", "error(64): A.g4:6:6: cannot access rule d's parameter: $b.d\n", // can't see rule ref's arg
"$d.text", "error(63): A.g4:6:4: unknown attribute reference d in $d.text\n", // valid rule, but no ref
"$lab.d", "error(64): A.g4:6:8: cannot access rule d's parameter: $lab.d\n",
"$lab", "error(67): A.g4:7:4: missing attribute access on rule reference lab in $lab\n",
"$q", "error(63): A.g4:7:4: unknown attribute reference q in $q\n",
"$q.y", "error(63): A.g4:7:4: unknown attribute reference q in $q.y\n",
"$q = 3", "error(63): A.g4:7:4: unknown attribute reference q in $q\n",
"$q = 3;", "error(63): A.g4:7:4: unknown attribute reference q in $q = 3;\n",
"$q.y = 3;", "error(63): A.g4:7:4: unknown attribute reference q in $q.y = 3;\n",
"$q = $blort;", "error(63): A.g4:7:4: unknown attribute reference q in $q = $blort;\n" +
"error(63): A.g4:7:9: unknown attribute reference blort in $blort\n",
"$a.ick", "error(65): A.g4:7:6: unknown attribute ick for rule a in $a.ick\n",
"$a.ick = 3;", "error(65): A.g4:7:6: unknown attribute ick for rule a in $a.ick = 3;\n",
"$b.d", "error(64): A.g4:7:6: cannot access rule d's parameter: $b.d\n", // can't see rule ref's arg
"$d.text", "error(63): A.g4:7:4: unknown attribute reference d in $d.text\n", // valid rule, but no ref
"$lab.d", "error(64): A.g4:7:8: cannot access rule d's parameter: $lab.d\n",
};
String[] finallyChecks = {
@ -84,20 +85,20 @@ public class TestAttributeChecks extends BaseTest {
"$id.text", "",
"$ids", "",
"$lab", "error(67): A.g4:9:14: missing attribute access on rule reference lab in $lab\n",
"$q", "error(63): A.g4:9:14: unknown attribute reference q in $q\n",
"$q.y", "error(63): A.g4:9:14: unknown attribute reference q in $q.y\n",
"$q = 3", "error(63): A.g4:9:14: unknown attribute reference q in $q\n",
"$q = 3;", "error(63): A.g4:9:14: unknown attribute reference q in $q = 3;\n",
"$q.y = 3;", "error(63): A.g4:9:14: unknown attribute reference q in $q.y = 3;\n",
"$q = $blort;", "error(63): A.g4:9:14: unknown attribute reference q in $q = $blort;\n" +
"error(63): A.g4:9:19: unknown attribute reference blort in $blort\n",
"$a.ick", "error(65): A.g4:9:16: unknown attribute ick for rule a in $a.ick\n",
"$a.ick = 3;", "error(65): A.g4:9:16: unknown attribute ick for rule a in $a.ick = 3;\n",
"$b.e", "error(63): A.g4:9:14: unknown attribute reference b in $b.e\n", // can't see rule refs outside alts
"$b.d", "error(63): A.g4:9:14: unknown attribute reference b in $b.d\n",
"$c.text", "error(63): A.g4:9:14: unknown attribute reference c in $c.text\n",
"$lab.d", "error(64): A.g4:9:18: cannot access rule d's parameter: $lab.d\n",
"$lab", "error(67): A.g4:10:14: missing attribute access on rule reference lab in $lab\n",
"$q", "error(63): A.g4:10:14: unknown attribute reference q in $q\n",
"$q.y", "error(63): A.g4:10:14: unknown attribute reference q in $q.y\n",
"$q = 3", "error(63): A.g4:10:14: unknown attribute reference q in $q\n",
"$q = 3;", "error(63): A.g4:10:14: unknown attribute reference q in $q = 3;\n",
"$q.y = 3;", "error(63): A.g4:10:14: unknown attribute reference q in $q.y = 3;\n",
"$q = $blort;", "error(63): A.g4:10:14: unknown attribute reference q in $q = $blort;\n" +
"error(63): A.g4:10:19: unknown attribute reference blort in $blort\n",
"$a.ick", "error(65): A.g4:10:16: unknown attribute ick for rule a in $a.ick\n",
"$a.ick = 3;", "error(65): A.g4:10:16: unknown attribute ick for rule a in $a.ick = 3;\n",
"$b.e", "error(63): A.g4:10:14: unknown attribute reference b in $b.e\n", // can't see rule refs outside alts
"$b.d", "error(63): A.g4:10:14: unknown attribute reference b in $b.d\n",
"$c.text", "error(63): A.g4:10:14: unknown attribute reference c in $c.text\n",
"$lab.d", "error(64): A.g4:10:18: cannot access rule d's parameter: $lab.d\n",
};
String[] dynMembersChecks = {
@ -200,6 +201,7 @@ public class TestAttributeChecks extends BaseTest {
@Test public void testTokenRef() throws RecognitionException {
String grammar =
"parser grammar S;\n" +
"tokens{ID;}\n" +
"a : x=ID {Token t = $x; t = $ID;} ;\n";
String expected =
"";

19
tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
View File

@ -37,6 +37,7 @@ public class TestBasicSemanticErrors extends BaseTest {
"parser grammar U;\n" +
"options { foo=bar; k=\"3\";}\n" +
"tokens {\n" +
" ID;\n" +
" f='fkj';\n" +
" S = 'a';\n" +
"}\n" +
@ -52,16 +53,16 @@ public class TestBasicSemanticErrors extends BaseTest {
// YIELDS
"warning(83): U.g4:2:10: illegal option foo\n" +
"warning(83): U.g4:2:19: illegal option k\n" +
"error(60): U.g4:4:8: token names must start with an uppercase letter: f\n" +
"error(59): U.g4:4:8: can't assign string value to token name f in non-combined grammar\n" +
"error(59): U.g4:5:8: can't assign string value to token name S in non-combined grammar\n" +
"warning(83): U.g4:8:10: illegal option x\n" +
"error(60): U.g4:5:8: token names must start with an uppercase letter: f\n" +
"error(59): U.g4:5:8: can't assign string value to token name f in non-combined grammar\n" +
"error(59): U.g4:6:8: can't assign string value to token name S in non-combined grammar\n" +
"warning(83): U.g4:9:10: illegal option x\n" +
"error(54): U.g4:9:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
"error(54): U.g4:8:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
"error(54): U.g4:7:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
"warning(83): U.g4:11:10: illegal option blech\n" +
"warning(83): U.g4:11:21: illegal option greedy\n" +
"warning(83): U.g4:14:16: illegal option ick\n" +
"warning(83): U.g4:15:16: illegal option x\n",
"warning(83): U.g4:12:10: illegal option blech\n" +
"warning(83): U.g4:12:21: illegal option greedy\n" +
"warning(83): U.g4:15:16: illegal option ick\n" +
"warning(83): U.g4:16:16: illegal option x\n",
};
@Test public void testU() { super.testErrors(U, false); }

10
tool/test/org/antlr/v4/test/TestCompositeGrammars.java
View File

@ -501,29 +501,34 @@ public class TestCompositeGrammars extends BaseTest {
ErrorQueue equeue = new ErrorQueue();
String slave =
"parser grammar T;\n" +
"tokens{T;}\n" +
"x : T ;\n" ;
mkdir(tmpdir);
writeFile(tmpdir, "T.g4", slave);
slave =
"parser grammar S;\n" +
"import T;\n" +
"tokens{S;}\n" +
"y : S ;\n" ;
mkdir(tmpdir);
writeFile(tmpdir, "S.g4", slave);
slave =
"parser grammar C;\n" +
"tokens{C;}\n" +
"i : C ;\n" ;
mkdir(tmpdir);
writeFile(tmpdir, "C.g4", slave);
slave =
"parser grammar B;\n" +
"tokens{B;}\n" +
"j : B ;\n" ;
mkdir(tmpdir);
writeFile(tmpdir, "B.g4", slave);
slave =
"parser grammar A;\n" +
"import B,C;\n" +
"tokens{A;}\n" +
"k : A ;\n" ;
mkdir(tmpdir);
writeFile(tmpdir, "A.g4", slave);
@ -531,12 +536,13 @@ public class TestCompositeGrammars extends BaseTest {
String master =
"grammar M;\n" +
"import S,A;\n" +
"tokens{M;}\n" +
"a : M ;\n" ;
writeFile(tmpdir, "M.g4", master);
Grammar g = new Grammar(tmpdir+"/M.g4", master, equeue);
assertEquals(equeue.errors.toString(), "[]");
assertEquals(equeue.warnings.toString(), "[]");
assertEquals("[]", equeue.errors.toString());
assertEquals("[]", equeue.warnings.toString());
String expectedTokenIDToTypeMap = "{EOF=-1, M=3, S=4, T=5, A=6, B=7, C=8}";
String expectedStringLiteralToTypeMap = "{}";
String expectedTypeToTokenList = "[M, S, T, A, B, C]";

12
tool/test/org/antlr/v4/test/TestSymbolIssues.java
View File

@ -34,7 +34,7 @@ public class TestSymbolIssues extends BaseTest {
static String[] B = {
// INPUT
"parser grammar B;\n" +
"tokens { X='x'; Y; }\n" +
"tokens { ID; FOO; X='x'; Y; }\n" +
"\n" +
"a : s=ID b+=ID X=ID '.' ;\n" +
"\n" +
@ -42,16 +42,18 @@ public class TestSymbolIssues extends BaseTest {
"\n" +
"s : FOO ;",
// YIELDS
"error(59): B.g4:2:9: can't assign string value to token name X in non-combined grammar\n" +
"error(59): B.g4:2:18: can't assign string value to token name X in non-combined grammar\n" +
"error(69): B.g4:4:4: label s conflicts with rule with same name\n" +
"error(69): B.g4:4:9: label b conflicts with rule with same name\n" +
"error(70): B.g4:4:15: label X conflicts with token with same name\n" +
"error(75): B.g4:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n"
"error(75): B.g4:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n" +
"error(126): B.g4:4:20: cannot create implicit token for string literal '.' in non-combined grammar\n"
};
static String[] D = {
// INPUT
"parser grammar D;\n" +
"tokens{ID;}\n" +
"a[int j] \n" +
" : i=ID j=ID ;\n" +
"\n" +
@ -61,8 +63,8 @@ public class TestSymbolIssues extends BaseTest {
" : ID ;",
// YIELDS
"error(72): D.g4:3:21: label j conflicts with rule a's return value or parameter with same name\n" +
"error(76): D.g4:5:0: rule b's argument i conflicts a return value with same name\n"
"error(72): D.g4:4:21: label j conflicts with rule a's return value or parameter with same name\n" +
"error(76): D.g4:6:0: rule b's argument i conflicts a return value with same name\n"
};
static String[] E = {