diff --git a/tool/playground/T.g b/tool/playground/T.g index 777c6ec38..899167684 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,3 +1,5 @@ grammar T; +tokens { A,B } + s : A ; diff --git a/tool/src/org/antlr/v4/parse/ANTLRParser.g b/tool/src/org/antlr/v4/parse/ANTLRParser.g index 2df173861..39e429551 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRParser.g +++ b/tool/src/org/antlr/v4/parse/ANTLRParser.g @@ -283,16 +283,7 @@ delegateGrammar * {tree} parser. */ tokensSpec - : TOKENS_SPEC tokenSpec+ RBRACE -> ^(TOKENS_SPEC tokenSpec+) - ; - -tokenSpec - : id - ( ASSIGN STRING_LITERAL -> ^(ASSIGN id STRING_LITERAL) - | -> id - ) - SEMI - | RULE_REF // INVALID! (an error alt) + : TOKENS_SPEC id (COMMA id)* RBRACE -> ^(TOKENS_SPEC id+) ; // A declaration of a language target specifc section, diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index 12a712f2f..ba6f0982e 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -121,7 +121,7 @@ public void finishGrammar(GrammarRootAST root, GrammarAST ID) { } public void grammarOption(GrammarAST ID, GrammarAST valueAST) { } public void ruleOption(GrammarAST ID, GrammarAST valueAST) { } public void blockOption(GrammarAST ID, GrammarAST valueAST) { } -public void tokenAlias(GrammarAST ID, GrammarAST literal) { } +public void defineToken(GrammarAST ID) { } public void globalNamedAction(GrammarAST scope, GrammarAST ID, ActionAST action) { } public void importGrammar(GrammarAST label, GrammarAST ID) { } @@ -225,8 +225,7 @@ tokensSpec ; tokenSpec - : ^(ASSIGN ID STRING_LITERAL) {tokenAlias($ID, $STRING_LITERAL);} - | ID {tokenAlias($ID, null);} + : ID {defineToken($ID);} ; action diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index 2b5f8b5a2..1289a25bb 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -125,11 +125,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { checkNumPrequels(options, imports, tokens); } - @Override - public void tokenAlias(GrammarAST ID, GrammarAST literal) { - if ( literal!=null ) checkTokenAlias(ID.token); - } - @Override public void importGrammar(GrammarAST label, GrammarAST ID) { checkImport(ID.token); diff --git a/tool/src/org/antlr/v4/semantics/SymbolChecks.java b/tool/src/org/antlr/v4/semantics/SymbolChecks.java index 26412b9bc..54294f763 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolChecks.java +++ b/tool/src/org/antlr/v4/semantics/SymbolChecks.java @@ -90,7 +90,6 @@ public class SymbolChecks { // done in sem pipe for now checkForRuleConflicts(g.rules.values()); // sets nameToRuleMap checkActionRedefinitions(collector.namedActions); - checkTokenAliasRedefinitions(collector.tokensDefs); //checkRuleArgs(collector.rulerefs); checkForTokenConflicts(collector.tokenIDRefs); // sets tokenIDs checkForLabelConflicts(g.rules.values()); @@ -140,57 +139,6 @@ public class SymbolChecks { } } - /** Catch: - tokens { A='a'; A; } can't redefine token type if has alias - tokens { A; A='a'; } can't redefine token type if has alias - tokens { A='a'; A='b'; } can't have two aliases for single token type - tokens { A='a'; B='a'; } can't have to token types for same string alias - */ - public void checkTokenAliasRedefinitions(List aliases) { - if ( aliases==null ) return; - - // map names, strings to root of A or (= A 'a') - Map aliasTokenNames = new HashMap(); - Map aliasStringValues = new HashMap(); - for (int i=0; i0 ) idNode = (GrammarAST)a.getChild(0); - GrammarAST prevToken = aliasTokenNames.get(idNode.getText()); - GrammarAST stringNode = null; - if ( a.getChildCount()>0 ) stringNode = (GrammarAST)a.getChild(1); - GrammarAST prevString = null; - if ( stringNode!=null ) prevString = aliasStringValues.get(stringNode.getText()); - if ( a.getType() == ANTLRParser.ASSIGN ) { // A='a' - idNode = (GrammarAST)a.getChild(0); - if ( prevString==null ) { // not seen string before - if ( stringNode!=null ) aliasStringValues.put(stringNode.getText(), a); - } - } - if ( prevToken==null ) { // not seen before, define it - aliasTokenNames.put(idNode.getText(), a); - } - - // we've defined token names and strings at this point if not seen previously. - // now, look for trouble. - if ( prevToken!=null ) { - if ( !(prevToken.getChildCount()==0 && a.getChildCount()==0) ) { - // one or both have strings; disallow - errMgr.grammarError(ErrorType.TOKEN_NAME_REASSIGNMENT, - a.g.fileName, idNode.token, idNode.getText()); - } - } - if ( prevString!=null ) { - // A='a' and A='a' are ok but not B='a' and A='a' are ok - if ( !prevString.getChild(0).getText().equals(idNode.getText()) ) { - errMgr.grammarError(ErrorType.TOKEN_STRING_REASSIGNMENT, - a.g.fileName, idNode.token, idNode.getText()+"="+stringNode.getText(), - prevString.getChild(0).getText()); - } - } - } - } - public void checkForTokenConflicts(List tokenIDRefs) { // for (GrammarAST a : tokenIDRefs) { // Token t = a.token; diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index 2de0f26bf..d9a737419 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -81,18 +81,10 @@ public class SymbolCollector extends GrammarTreeVisitor { } @Override - public void tokenAlias(GrammarAST ID, GrammarAST literal) { - if ( literal==null ) { - terminals.add(ID); - tokenIDRefs.add(ID); - tokensDefs.add(ID); - } - else { - terminals.add(ID); - tokenIDRefs.add(ID); - tokensDefs.add((GrammarAST)ID.getParent()); - strings.add(literal.getText()); - } + public void defineToken(GrammarAST ID) { + terminals.add(ID); + tokenIDRefs.add(ID); + tokensDefs.add(ID); } @Override diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index 907c5abc6..11bd9808d 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -123,8 +123,8 @@ public enum ErrorType { HETERO_ILLEGAL_IN_REWRITE_ALT(104, "", ErrorSeverity.ERROR), NO_SUCH_GRAMMAR_SCOPE(105, "reference to undefined grammar in rule reference: .", ErrorSeverity.ERROR), NO_SUCH_RULE_IN_SCOPE(106, "rule is not defined in grammar ", ErrorSeverity.ERROR), - TOKEN_STRING_REASSIGNMENT(107, "cannot alias in tokens {}; string already assigned to ", ErrorSeverity.ERROR), - TOKEN_NAME_REASSIGNMENT(108, "cannot redefine ; token name already assigned to defined", ErrorSeverity.ERROR), +// TOKEN_STRING_REASSIGNMENT(107, "cannot alias in tokens {}; string already assigned to ", ErrorSeverity.ERROR), +// TOKEN_NAME_REASSIGNMENT(108, "cannot redefine ; token name already assigned to defined", ErrorSeverity.ERROR), //TOKEN_VOCAB_IN_DELEGATE(, "tokenVocab option ignored in imported grammar ", ErrorSeverity.ERROR), OPTIONS_IN_DELEGATE(109, "options ignored in imported grammar ", ErrorSeverity.WARNING), // TOKEN_ALIAS_IN_DELEGATE(, "can't assign string to token name to string in imported grammar ", ErrorSeverity.ERROR), diff --git a/tool/test/org/antlr/v4/test/TestASTStructure.java b/tool/test/org/antlr/v4/test/TestASTStructure.java index e1042d00e..1abe74bf7 100644 --- a/tool/test/org/antlr/v4/test/TestASTStructure.java +++ b/tool/test/org/antlr/v4/test/TestASTStructure.java @@ -69,7 +69,7 @@ public class TestASTStructure { @Test public void test_grammarSpec2() throws Exception { // gunit test on line 18 - RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n tokens { A; B='33'; }\n @header {foo}\n a : A;\n ", 18); + RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n tokens { A, B }\n @header {foo}\n a : A;\n ", 18); Object actual = ((Tree)rstruct.getTree()).toStringTree(); Object expecting = "(PARSER_GRAMMAR P (tokens { A (= B '33')) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))"; assertEquals("testing rule grammarSpec", expecting, actual); @@ -77,7 +77,7 @@ public class TestASTStructure { @Test public void test_grammarSpec3() throws Exception { // gunit test on line 30 - RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n @header {foo}\n tokens { A; B='33'; }\n a : A;\n ", 30); + RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n @header {foo}\n tokens { A,B }\n a : A;\n ", 30); Object actual = ((Tree)rstruct.getTree()).toStringTree(); Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A (= B '33')) (RULES (RULE a (BLOCK (ALT A)))))"; assertEquals("testing rule grammarSpec", expecting, actual); @@ -378,4 +378,4 @@ public class TestASTStructure { Object expecting = "(* (BLOCK (ALT {blort} 'x')))"; assertEquals("testing rule element", expecting, actual); } -} \ No newline at end of file +} diff --git a/tool/test/org/antlr/v4/test/TestATNDeserialization.java b/tool/test/org/antlr/v4/test/TestATNDeserialization.java index f3816f6ee..b622db934 100644 --- a/tool/test/org/antlr/v4/test/TestATNDeserialization.java +++ b/tool/test/org/antlr/v4/test/TestATNDeserialization.java @@ -33,7 +33,7 @@ public class TestATNDeserialization extends BaseTest { @Test public void testNot() throws Exception { Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {A; B; C;}\n" + + "tokens {A, B, C}\n" + "a : ~A ;"); checkDeserializationIsStable(g); } @@ -41,7 +41,7 @@ public class TestATNDeserialization extends BaseTest { @Test public void testWildcard() throws Exception { Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {A; B; C;}\n" + + "tokens {A, B, C}\n" + "a : . ;"); checkDeserializationIsStable(g); } diff --git a/tool/test/org/antlr/v4/test/TestATNInterpreter.java b/tool/test/org/antlr/v4/test/TestATNInterpreter.java index e6b36e42f..758769fe3 100644 --- a/tool/test/org/antlr/v4/test/TestATNInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNInterpreter.java @@ -40,7 +40,7 @@ public class TestATNInterpreter extends BaseTest { "C : 'c' ;\n"); Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {A; B; C;}\n" + + "tokens {A,B,C}\n" + "a : ~A ;"); checkMatchedAlt(lg, g, "b", 1); } @@ -254,7 +254,7 @@ public class TestATNInterpreter extends BaseTest { ); Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {A;B;C;LP;RP;INT;}\n" + + "tokens {A,B,C,LP,RP,INT}\n" + "a : e B | e C ;\n" + "e : LP e RP\n" + " | INT\n" + diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 179b3e702..abc0b9ef0 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -186,7 +186,7 @@ public class TestATNParserPrediction extends BaseTest { "C : 'c' ;\n"); Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {A;B;C;}\n" + + "tokens {A,B,C}\n" + "a : x B ;\n" + "b : x C ;\n" + "x : A | ;\n"); @@ -302,7 +302,7 @@ public class TestATNParserPrediction extends BaseTest { ); Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {A;B;C;LP;RP;INT;}\n" + + "tokens {A,B,C,LP,RP,INT}\n" + "a : e B | e C ;\n" + "e : LP e RP\n" + " | INT\n" + @@ -361,7 +361,7 @@ public class TestATNParserPrediction extends BaseTest { ); Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {A;B;C;LP;RP;INT;}\n" + + "tokens {A,B,C,LP,RP,INT}\n" + "a : e A | e A B ;\n" + "e : LP e RP\n" + " | INT\n" + @@ -425,7 +425,7 @@ public class TestATNParserPrediction extends BaseTest { ); Grammar g = new Grammar( "parser grammar T;\n"+ - "tokens {ID;SEMI;INT;}\n" + + "tokens {ID,SEMI,INT}\n" + "a : (ID | ID ID?) SEMI ;"); int decision = 1; checkPredictedAlt(lg, g, decision, "a;", 1); diff --git a/tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java b/tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java index cfba8d155..8361a726a 100644 --- a/tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java +++ b/tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java @@ -27,8 +27,8 @@ public class TestTokenTypeAssignment extends BaseTest { Grammar g = new Grammar( "parser grammar t;\n" + "tokens {\n" + - " C;\n" + - " D;" + + " C,\n" + + " D" + "}\n"+ "a : A | B;\n" + "b : C ;"); @@ -41,8 +41,8 @@ public class TestTokenTypeAssignment extends BaseTest { LexerGrammar g = new LexerGrammar( "lexer grammar t;\n" + "tokens {\n" + - " C;\n" + - " D;" + + " C,\n" + + " D" + "}\n"+ "A : 'a';\n" + "C : 'c' ;"); @@ -51,20 +51,6 @@ public class TestTokenTypeAssignment extends BaseTest { checkSymbols(g, rules, tokenNames); } - @Test public void testTokensSectionWithAssignmentSection() throws Exception { - Grammar g = new Grammar( - "grammar t;\n" + - "tokens {\n" + - " C='c';\n" + - " D;" + - "}\n"+ - "a : A | B;\n" + - "b : C ;"); - String rules = "a, b"; - String tokenNames = "A, B, C, D, 'c'"; - checkSymbols(g, rules, tokenNames); - } - @Test public void testCombinedGrammarLiterals() throws Exception { Grammar g = new Grammar( "grammar t;\n"+ @@ -145,32 +131,6 @@ public class TestTokenTypeAssignment extends BaseTest { assertEquals("'\\n'", literals.toArray()[0]); } - @Test public void testTokenInTokensSectionAndTokenRuleDef() throws Exception { - String grammar = - "grammar P;\n" + - "tokens { B='}'; }\n"+ - "a : A B {System.out.println(_input.getText());} ;\n"+ - "A : 'a' ;\n" + - "B : '}' ;\n"+ - "WS : (' '|'\\n') {skip();} ;"; - String found = execParser("P.g4", grammar, "PParser", "PLexer", - "a", "a}", false); - assertEquals("a}\n", found); - } - - @Test public void testTokenInTokensSectionAndTokenRuleDef2() throws Exception { - String grammar = - "grammar P;\n" + - "tokens { B='}'; }\n"+ - "a : A '}' {System.out.println(_input.getText());} ;\n"+ - "A : 'a' ;\n" + - "B : '}' ;\n"+ - "WS : (' '|'\\n') {skip();} ;"; - String found = execParser("P.g4", grammar, "PParser", "PLexer", - "a", "a}", false); - assertEquals("a}\n", found); - } - protected void checkSymbols(Grammar g, String rulesStr, String allValidTokensStr)