rm T='literal' in tokens { }. Also it's comma-separated not ';' terminated now. tokens { A,B }

tool/playground/T.g

@@ -1,3 +1,5 @@
 grammar T;
 
+tokens { A,B }
+
 s : A ;

tool/src/org/antlr/v4/parse/ANTLRParser.g

@@ -283,16 +283,7 @@ delegateGrammar
  *  {tree} parser.
  */
 tokensSpec
-	: TOKENS_SPEC tokenSpec+ RBRACE -> ^(TOKENS_SPEC tokenSpec+)
+	: TOKENS_SPEC id (COMMA id)* RBRACE -> ^(TOKENS_SPEC id+)
-	;
-
-tokenSpec
-	:	id
-		(	ASSIGN STRING_LITERAL	-> ^(ASSIGN id STRING_LITERAL<TerminalAST>)
-		|							-> id
-		)
-		SEMI
-	|	RULE_REF // INVALID! (an error alt)
 	;
 
 // A declaration of a language target specifc section,

tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g

@@ -121,7 +121,7 @@ public void finishGrammar(GrammarRootAST root, GrammarAST ID) { }
 public void grammarOption(GrammarAST ID, GrammarAST valueAST) { }
 public void ruleOption(GrammarAST ID, GrammarAST valueAST) { }
 public void blockOption(GrammarAST ID, GrammarAST valueAST) { }
-public void tokenAlias(GrammarAST ID, GrammarAST literal) { }
+public void defineToken(GrammarAST ID) { }
 public void globalNamedAction(GrammarAST scope, GrammarAST ID, ActionAST action) { }
 public void importGrammar(GrammarAST label, GrammarAST ID) { }
 
@@ -225,8 +225,7 @@ tokensSpec
 	;
 
 tokenSpec
-	:	^(ASSIGN ID STRING_LITERAL)	{tokenAlias($ID, $STRING_LITERAL);}
+	:	ID							{defineToken($ID);}
-	|	ID							{tokenAlias($ID, null);}
 	;
 
 action

tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java

@@ -125,11 +125,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
 		checkNumPrequels(options, imports, tokens);
 	}
 
-	@Override
-	public void tokenAlias(GrammarAST ID, GrammarAST literal) {
-		if ( literal!=null ) checkTokenAlias(ID.token);
-	}
-
 	@Override
 	public void importGrammar(GrammarAST label, GrammarAST ID) {
 		checkImport(ID.token);

tool/src/org/antlr/v4/semantics/SymbolChecks.java

@@ -90,7 +90,6 @@ public class SymbolChecks {
 		// done in sem pipe for now
         checkForRuleConflicts(g.rules.values());         // sets nameToRuleMap
 		checkActionRedefinitions(collector.namedActions);
-        checkTokenAliasRedefinitions(collector.tokensDefs);
         //checkRuleArgs(collector.rulerefs);
         checkForTokenConflicts(collector.tokenIDRefs);  // sets tokenIDs
         checkForLabelConflicts(g.rules.values());
@@ -140,57 +139,6 @@ public class SymbolChecks {
         }
     }
 
-	/** Catch:
-		tokens { A='a'; A; }		can't redefine token type if has alias
-	 	tokens { A; A='a'; }		can't redefine token type if has alias
-	 	tokens { A='a'; A='b'; }	can't have two aliases for single token type
-		tokens { A='a'; B='a'; }	can't have to token types for same string alias
-	 */
-    public void checkTokenAliasRedefinitions(List<GrammarAST> aliases) {
-        if ( aliases==null ) return;
-
-		// map names, strings to root of A or (= A 'a')
-        Map<String, GrammarAST> aliasTokenNames = new HashMap<String, GrammarAST>();
-		Map<String, GrammarAST> aliasStringValues = new HashMap<String, GrammarAST>();
-        for (int i=0; i<aliases.size(); i++) {
-            GrammarAST a = aliases.get(i);
-			GrammarAST idNode = a;
-			if ( a.getChildCount()>0 ) idNode = (GrammarAST)a.getChild(0);
-			GrammarAST prevToken = aliasTokenNames.get(idNode.getText());
-			GrammarAST stringNode = null;
-			if ( a.getChildCount()>0 ) stringNode = (GrammarAST)a.getChild(1);
-			GrammarAST prevString = null;
-			if ( stringNode!=null ) prevString = aliasStringValues.get(stringNode.getText());
-            if ( a.getType() == ANTLRParser.ASSIGN ) { // A='a'
-                idNode = (GrammarAST)a.getChild(0);
-				if ( prevString==null ) { // not seen string before
-					if ( stringNode!=null ) aliasStringValues.put(stringNode.getText(), a);
-				}
-            }
-			if ( prevToken==null ) { // not seen before, define it
-				aliasTokenNames.put(idNode.getText(), a);
-			}
-
-			// we've defined token names and strings at this point if not seen previously.
-			// now, look for trouble.
-			if ( prevToken!=null ) {
-				if ( !(prevToken.getChildCount()==0 && a.getChildCount()==0) ) {
-					// one or both have strings; disallow
-					errMgr.grammarError(ErrorType.TOKEN_NAME_REASSIGNMENT,
-										a.g.fileName, idNode.token, idNode.getText());
-				}
-			}
-			if ( prevString!=null ) {
-				// A='a' and A='a' are ok but not B='a' and A='a' are ok
-				if ( !prevString.getChild(0).getText().equals(idNode.getText()) ) {
-					errMgr.grammarError(ErrorType.TOKEN_STRING_REASSIGNMENT,
-										a.g.fileName, idNode.token, idNode.getText()+"="+stringNode.getText(),
-										prevString.getChild(0).getText());
-				}
-			}
-		}
-	}
-
     public void checkForTokenConflicts(List<GrammarAST> tokenIDRefs) {
 //        for (GrammarAST a : tokenIDRefs) {
 //            Token t = a.token;

tool/src/org/antlr/v4/semantics/SymbolCollector.java

@@ -81,18 +81,10 @@ public class SymbolCollector extends GrammarTreeVisitor {
 	}
 
 	@Override
-	public void tokenAlias(GrammarAST ID, GrammarAST literal) {
+	public void defineToken(GrammarAST ID) {
-		if ( literal==null ) {
+		terminals.add(ID);
-			terminals.add(ID);
+		tokenIDRefs.add(ID);
-			tokenIDRefs.add(ID);
+		tokensDefs.add(ID);
-			tokensDefs.add(ID);
-		}
-		else {
-			terminals.add(ID);
-			tokenIDRefs.add(ID);
-			tokensDefs.add((GrammarAST)ID.getParent());
-			strings.add(literal.getText());
-		}
 	}
 
 	@Override

tool/src/org/antlr/v4/tool/ErrorType.java

@@ -123,8 +123,8 @@ public enum ErrorType {
 	HETERO_ILLEGAL_IN_REWRITE_ALT(104, "", ErrorSeverity.ERROR),
 	NO_SUCH_GRAMMAR_SCOPE(105, "reference to undefined grammar in rule reference: <arg>.<arg2>", ErrorSeverity.ERROR),
 	NO_SUCH_RULE_IN_SCOPE(106, "rule <arg2> is not defined in grammar <arg>", ErrorSeverity.ERROR),
-	TOKEN_STRING_REASSIGNMENT(107, "cannot alias <arg> in tokens {}; string already assigned to <arg2>", ErrorSeverity.ERROR),
+//	TOKEN_STRING_REASSIGNMENT(107, "cannot alias <arg> in tokens {}; string already assigned to <arg2>", ErrorSeverity.ERROR),
-	TOKEN_NAME_REASSIGNMENT(108, "cannot redefine <arg>; token name already <if(arg2)>assigned to <arg2><else>defined<endif>", ErrorSeverity.ERROR),
+//	TOKEN_NAME_REASSIGNMENT(108, "cannot redefine <arg>; token name already <if(arg2)>assigned to <arg2><else>defined<endif>", ErrorSeverity.ERROR),
 	//TOKEN_VOCAB_IN_DELEGATE(, "tokenVocab option ignored in imported grammar <arg>", ErrorSeverity.ERROR),
 	OPTIONS_IN_DELEGATE(109, "options ignored in imported grammar <arg>", ErrorSeverity.WARNING),
 //	TOKEN_ALIAS_IN_DELEGATE(, "can't assign string to token name <arg> to string in imported grammar <arg2>", ErrorSeverity.ERROR),

tool/test/org/antlr/v4/test/TestASTStructure.java

@@ -69,7 +69,7 @@ public class TestASTStructure {
 
 	@Test public void test_grammarSpec2() throws Exception {
 		// gunit test on line 18
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n    parser grammar P;\n    tokens { A; B='33'; }\n    @header {foo}\n    a : A;\n    ", 18);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n    parser grammar P;\n    tokens { A, B }\n    @header {foo}\n    a : A;\n    ", 18);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(PARSER_GRAMMAR P (tokens { A (= B '33')) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))";
 		assertEquals("testing rule grammarSpec", expecting, actual);
@@ -77,7 +77,7 @@ public class TestASTStructure {
 
 	@Test public void test_grammarSpec3() throws Exception {
 		// gunit test on line 30
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n    parser grammar P;\n    @header {foo}\n    tokens { A; B='33'; }\n    a : A;\n    ", 30);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n    parser grammar P;\n    @header {foo}\n    tokens { A,B }\n    a : A;\n    ", 30);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A (= B '33')) (RULES (RULE a (BLOCK (ALT A)))))";
 		assertEquals("testing rule grammarSpec", expecting, actual);
@@ -378,4 +378,4 @@ public class TestASTStructure {
 		Object expecting = "(* (BLOCK (ALT {blort} 'x')))";
 		assertEquals("testing rule element", expecting, actual);
 	}
-}
+}

tool/test/org/antlr/v4/test/TestATNDeserialization.java

@@ -33,7 +33,7 @@ public class TestATNDeserialization extends BaseTest {
 	@Test public void testNot() throws Exception {
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {A; B; C;}\n" +
+			"tokens {A, B, C}\n" +
 			"a : ~A ;");
 		checkDeserializationIsStable(g);
 	}
@@ -41,7 +41,7 @@ public class TestATNDeserialization extends BaseTest {
 	@Test public void testWildcard() throws Exception {
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {A; B; C;}\n" +
+			"tokens {A, B, C}\n" +
 			"a : . ;");
 		checkDeserializationIsStable(g);
 	}

tool/test/org/antlr/v4/test/TestATNInterpreter.java

@@ -40,7 +40,7 @@ public class TestATNInterpreter extends BaseTest {
 			"C : 'c' ;\n");
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {A; B; C;}\n" +
+			"tokens {A,B,C}\n" +
 			"a : ~A ;");
 		checkMatchedAlt(lg, g, "b", 1);
 	}
@@ -254,7 +254,7 @@ public class TestATNInterpreter extends BaseTest {
 		);
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {A;B;C;LP;RP;INT;}\n" +
+			"tokens {A,B,C,LP,RP,INT}\n" +
 			"a : e B | e C ;\n" +
 			"e : LP e RP\n" +
 			"  | INT\n" +

tool/test/org/antlr/v4/test/TestATNParserPrediction.java

@@ -186,7 +186,7 @@ public class TestATNParserPrediction extends BaseTest {
 			"C : 'c' ;\n");
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {A;B;C;}\n" +
+			"tokens {A,B,C}\n" +
 			"a : x B ;\n" +
 			"b : x C ;\n" +
 			"x : A | ;\n");
@@ -302,7 +302,7 @@ public class TestATNParserPrediction extends BaseTest {
 		);
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {A;B;C;LP;RP;INT;}\n" +
+			"tokens {A,B,C,LP,RP,INT}\n" +
 			"a : e B | e C ;\n" +
 			"e : LP e RP\n" +
 			"  | INT\n" +
@@ -361,7 +361,7 @@ public class TestATNParserPrediction extends BaseTest {
 		);
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {A;B;C;LP;RP;INT;}\n" +
+			"tokens {A,B,C,LP,RP,INT}\n" +
 			"a : e A | e A B ;\n" +
 			"e : LP e RP\n" +
 			"  | INT\n" +
@@ -425,7 +425,7 @@ public class TestATNParserPrediction extends BaseTest {
 		);
 		Grammar g = new Grammar(
 			"parser grammar T;\n"+
-			"tokens {ID;SEMI;INT;}\n" +
+			"tokens {ID,SEMI,INT}\n" +
 			"a : (ID | ID ID?) SEMI ;");
 		int decision = 1;
 		checkPredictedAlt(lg, g, decision, "a;", 1);

tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java

@@ -27,8 +27,8 @@ public class TestTokenTypeAssignment extends BaseTest {
 		Grammar g = new Grammar(
 				"parser grammar t;\n" +
 				"tokens {\n" +
-				"  C;\n" +
+				"  C,\n" +
-				"  D;" +
+				"  D" +
 				"}\n"+
 				"a : A | B;\n" +
 				"b : C ;");
@@ -41,8 +41,8 @@ public class TestTokenTypeAssignment extends BaseTest {
 		LexerGrammar g = new LexerGrammar(
 				"lexer grammar t;\n" +
 				"tokens {\n" +
-				"  C;\n" +
+				"  C,\n" +
-				"  D;" +
+				"  D" +
 				"}\n"+
 				"A : 'a';\n" +
 				"C : 'c' ;");
@@ -51,20 +51,6 @@ public class TestTokenTypeAssignment extends BaseTest {
 		checkSymbols(g, rules, tokenNames);
 	}
 
-	@Test public void testTokensSectionWithAssignmentSection() throws Exception {
-		Grammar g = new Grammar(
-				"grammar t;\n" +
-				"tokens {\n" +
-				"  C='c';\n" +
-				"  D;" +
-				"}\n"+
-				"a : A | B;\n" +
-				"b : C ;");
-		String rules = "a, b";
-		String tokenNames = "A, B, C, D, 'c'";
-		checkSymbols(g, rules, tokenNames);
-	}
-
 	@Test public void testCombinedGrammarLiterals() throws Exception {
 		Grammar g = new Grammar(
 				"grammar t;\n"+
@@ -145,32 +131,6 @@ public class TestTokenTypeAssignment extends BaseTest {
 		assertEquals("'\\n'", literals.toArray()[0]);
 	}
 
-	@Test public void testTokenInTokensSectionAndTokenRuleDef() throws Exception {
-		String grammar =
-			"grammar P;\n" +
-			"tokens { B='}'; }\n"+
-			"a : A B {System.out.println(_input.getText());} ;\n"+
-			"A : 'a' ;\n" +
-			"B : '}' ;\n"+
-			"WS : (' '|'\\n') {skip();} ;";
-		String found = execParser("P.g4", grammar, "PParser", "PLexer",
-								  "a", "a}", false);
-		assertEquals("a}\n", found);
-	}
-
-	@Test public void testTokenInTokensSectionAndTokenRuleDef2() throws Exception {
-		String grammar =
-			"grammar P;\n" +
-			"tokens { B='}'; }\n"+
-			"a : A '}' {System.out.println(_input.getText());} ;\n"+
-			"A : 'a' ;\n" +
-			"B : '}' ;\n"+
-			"WS : (' '|'\\n') {skip();} ;";
-		String found = execParser("P.g4", grammar, "PParser", "PLexer",
-								  "a", "a}", false);
-		assertEquals("a}\n", found);
-	}
-
 	protected void checkSymbols(Grammar g,
 								String rulesStr,
 								String allValidTokensStr)