diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java index 550b33a4c..0b8969b47 100644 --- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java +++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java @@ -151,7 +151,10 @@ public class SemanticPipeline { void assignLexerTokenTypes(Grammar g, List tokensDefs) { Grammar G = g.getOutermostGrammar(); // put in root, even if imported for (GrammarAST def : tokensDefs) { - if ( def.getType()== ANTLRParser.TOKEN_REF ) G.defineTokenName(def.getText()); + // tokens { id (',' id)* } so must check IDs not TOKEN_REF + if ( Grammar.isTokenName(def.getText()) ) { + G.defineTokenName(def.getText()); + } } /* Define token types for nonfragment rules which do not include a 'type(...)'