* added check for v3 backward incompatibilities:

** tokens {A;B;} syntax
** tokens {A='C';} syntax

CHANGES.txt

@@ -8,6 +8,8 @@ November 18, 2012
 * added check for v3 backward incompatibilities:
 ** tree grammars
 ** labels in lexer rules
+** tokens {A;B;} syntax
+** tokens {A='C';} syntax
 
 November 17, 2012
 

tool/src/org/antlr/v4/parse/ANTLRLexer.g

@@ -115,6 +115,7 @@ tokens { SEMPRED; TOKEN_REF; RULE_REF; LEXER_CHAR_SET; ARG_ACTION; }
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 package org.antlr.v4.parse;
+import org.antlr.v4.tool.*;
 }
 
 
@@ -122,6 +123,8 @@ package org.antlr.v4.parse;
     public CommonTokenStream tokens; // track stream we push to; need for context info
     public boolean isLexerRule = false;
 
+	public void grammarError(ErrorType etype, org.antlr.runtime.Token token, Object... args) { }
+
 	/** scan backwards from current point in this.tokens list
 	 *  looking for the start of the rule or subrule.
 	 *  Return token or null if for some reason we can't find the start.

tool/src/org/antlr/v4/parse/ANTLRParser.g

@@ -132,10 +132,7 @@ import org.antlr.v4.tool.ast.*;
 
 @members {
 Stack paraphrases = new Stack();
-/** Affects tree construction; no SET collapsing if AST (ID|INT) would hide them from rewrite.
- *  Could use for just AST ops, but we can't see -> until after building sets.
-boolean buildAST;
- */
+public void grammarError(ErrorType etype, org.antlr.runtime.Token token, Object... args) { }
 }
 
 // The main entry point for parsing a V3 grammar from top to toe. This is
@@ -279,15 +276,24 @@ delegateGrammar
     |   id
     ;
 
-/** The declaration of any token types we need that are not already
- *  specified by a preceeding grammar, such as when a parser declares
- *  imaginary tokens with which to construct the AST, or a rewriting
- *  tree parser adds further imaginary tokens to ones defined in a prior
- *  {tree} parser.
- */
 tokensSpec
-	: TOKENS_SPEC id (COMMA id)* RBRACE -> ^(TOKENS_SPEC id+)
+	: TOKENS_SPEC TOKEN_REF (COMMA TOKEN_REF)* RBRACE -> ^(TOKENS_SPEC TOKEN_REF+)
     | TOKENS_SPEC RBRACE ->
+    | TOKENS_SPEC^ v3tokenSpec+ RBRACE!
+      {grammarError(ErrorType.V3_TOKENS_SYNTAX, $TOKENS_SPEC);}
+	;
+	
+v3tokenSpec
+	:	TOKEN_REF
+		(	ASSIGN lit=STRING_LITERAL
+            {
+            grammarError(ErrorType.V3_ASSIGN_IN_TOKENS, $TOKEN_REF,
+                         $TOKEN_REF.getText(), $lit.getText());
+            }
+						            	-> TOKEN_REF // ignore assignment
+		|								-> TOKEN_REF
+		)
+		SEMI
 	;
 
 // A declaration of a language target specifc section,

tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g

@@ -442,7 +442,7 @@ tokenSpec
 @after {
 	exitTokenSpec($start);
 }
-	:	ID							{defineToken($ID);}
+	:	TOKEN_REF					{defineToken($TOKEN_REF);}
 	;
 
 action

tool/src/org/antlr/v4/parse/ToolANTLRParser.java

@@ -76,5 +76,8 @@ public class ToolANTLRParser extends ANTLRParser {
 		return msg;
 	}
 
-
+	@Override
+	public void grammarError(ErrorType etype, org.antlr.runtime.Token token, Object... args) {
+		tool.errMgr.grammarError(etype, getSourceName(), token, args);
+	}
 }

tool/src/org/antlr/v4/tool/ErrorManager.java

@@ -244,7 +244,11 @@ public class ErrorManager {
 
 	public void emit(ErrorType etype, ANTLRMessage msg) {
 		switch ( etype.severity ) {
-			case WARNING: warnings++;
+			case WARNING_ONE_OFF:
+				if ( errorTypes.contains(etype) ) break;
+				// fall thru
+			case WARNING:
+				warnings++;
 				tool.warning(msg);
 				break;
 			case ERROR_ONE_OFF:

tool/src/org/antlr/v4/tool/ErrorSeverity.java

@@ -37,7 +37,8 @@ package org.antlr.v4.tool;
  */
 public enum ErrorSeverity {
     INFO    ("info"),
-    WARNING ("warning"),
+	WARNING ("warning"),
+	WARNING_ONE_OFF ("warning"),
 	ERROR   ("error"),
 	ERROR_ONE_OFF   ("error"),
     FATAL   ("fatal"),  // TODO: add fatal for which phase? sync with ErrorManager

tool/src/org/antlr/v4/tool/ErrorType.java

@@ -117,10 +117,12 @@ public enum ErrorType {
 	USE_OF_BAD_WORD(134, "symbol <arg> conflicts with generated code in target language or runtime", ErrorSeverity.ERROR),
 
 	// Backward incompatibility errors
-	V3_TREE_GRAMMAR(200, "tree grammars are not supported in ANTLR v4", ErrorSeverity.ERROR_ONE_OFF),
+	V3_TREE_GRAMMAR(200, "tree grammars are not supported in ANTLR v4", ErrorSeverity.ERROR),
 	V3_LEXER_LABEL(201, "labels in lexer rules are not supported in ANTLR v4; " +
 		"actions cannot reference elements of lexical rules but you can use " +
-		"getText() to get the entire text matched for the rule", ErrorSeverity.WARNING),
+		"getText() to get the entire text matched for the rule", ErrorSeverity.WARNING_ONE_OFF),
+	V3_TOKENS_SYNTAX(202, "'tokens {A; B;}' syntax is now 'tokens {A, B}' in ANTLR v4", ErrorSeverity.WARNING),
+	V3_ASSIGN_IN_TOKENS(203, "assignments in tokens{} are not supported in ANTLR v4; use lexical rule '<arg> : <arg2>;' instead", ErrorSeverity.WARNING_ONE_OFF),
 
     // Dependency sorting errors