From 4e84ae28a22f9ceb6c5e82313fcf886c7b619923 Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Thu, 22 Nov 2012 11:32:45 -0800
Subject: [PATCH] fix so tokens { A, B } Actually defines them.

---
 tool/src/org/antlr/v4/semantics/SemanticPipeline.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
index 550b33a4c..0b8969b47 100644
--- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
+++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
@@ -151,7 +151,10 @@ public class SemanticPipeline {
 	void assignLexerTokenTypes(Grammar g, List<GrammarAST> tokensDefs) {
 		Grammar G = g.getOutermostGrammar(); // put in root, even if imported
 		for (GrammarAST def : tokensDefs) {
-			if ( def.getType()== ANTLRParser.TOKEN_REF ) G.defineTokenName(def.getText());
+			// tokens { id (',' id)* } so must check IDs not TOKEN_REF
+			if ( Grammar.isTokenName(def.getText()) ) {
+				G.defineTokenName(def.getText());
+			}
 		}
 
 		/* Define token types for nonfragment rules which do not include a 'type(...)'