got lots of ast rewrites working; off to do labels

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8845]

tool/playground/T.g

@@ -1,9 +1,8 @@
 grammar T;
 options {output=AST;}
-tokens {I;D;}
+tokens {I;}
 
-a : A B+ -> B["foo"]
+a : x=A -> $a ;
-  ;
 
 b : B | C ;
 

tool/src/org/antlr/v4/Tool.java

@@ -215,26 +215,30 @@ public class Tool {
 	}
 
 	public void process(Grammar g) {
+		LexerGrammar lexerg = null;
 		GrammarRootAST lexerAST = null;
 		if ( g.ast!=null && g.ast.grammarType== ANTLRParser.COMBINED &&
-		!g.ast.hasErrors )
+			 !g.ast.hasErrors )
-		{
-			lexerAST = extractImplicitLexer(g); // alters ast
-		}
-		processNonCombinedGrammar(g);
-		if ( g.ast!=null && g.ast.grammarType== ANTLRParser.COMBINED &&
-		!g.ast.hasErrors )
 		{
+			lexerAST = extractImplicitLexer(g); // alters g.ast
 			if ( lexerAST!=null ) {
-				LexerGrammar lexerg = new LexerGrammar(this, lexerAST);
+				lexerg = new LexerGrammar(this, lexerAST);
 				lexerg.fileName = g.fileName;
 				g.implicitLexer = lexerg;
 				lexerg.implicitLexerOwner = g;
-				lexerg.importVocab(g);
+
+//			// copy vocab from combined to implicit lexer
+//			g.importVocab(g.implicitLexerOwner); // TODO: don't need i don't think; done in tool process()
+
 				processNonCombinedGrammar(lexerg);
-				g.importVocab(lexerg);
+				System.out.println("lexer tokens="+lexerg.tokenNameToTypeMap);
+				System.out.println("lexer strings="+lexerg.stringLiteralToTypeMap);
 			}
 		}
+		if ( g.implicitLexer!=null ) g.importVocab(g.implicitLexer);
+		System.out.println("tokens="+g.tokenNameToTypeMap);
+		System.out.println("strings="+g.stringLiteralToTypeMap);
+		processNonCombinedGrammar(g);
 	}
 
 	public void processNonCombinedGrammar(Grammar g) {
@@ -337,24 +341,24 @@ public class Tool {
 		GrammarRootAST combinedAST = combinedGrammar.ast;
 		//System.out.println("before="+combinedAST.toStringTree());
 		GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
-		List<org.antlr.v4.tool.GrammarAST> elements = combinedAST.getChildren();
+		List<GrammarAST> elements = combinedAST.getChildren();
 
 		// MAKE A GRAMMAR ROOT and ID
 		String lexerName = combinedAST.getChild(0).getText()+"Lexer";
 		GrammarRootAST lexerAST =
-		new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR,"LEXER_GRAMMAR"));
+		    new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR,"LEXER_GRAMMAR"));
 		lexerAST.grammarType = ANTLRParser.LEXER;
 		lexerAST.token.setInputStream(combinedAST.token.getInputStream());
-		lexerAST.addChild((org.antlr.v4.tool.GrammarAST)adaptor.create(ANTLRParser.ID, lexerName));
+		lexerAST.addChild((GrammarAST)adaptor.create(ANTLRParser.ID, lexerName));
 
 		// MOVE OPTIONS
-		org.antlr.v4.tool.GrammarAST optionsRoot =
+		GrammarAST optionsRoot =
-		(org.antlr.v4.tool.GrammarAST)combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
+		(GrammarAST)combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
 		if ( optionsRoot!=null ) {
-			org.antlr.v4.tool.GrammarAST lexerOptionsRoot = (org.antlr.v4.tool.GrammarAST)adaptor.dupNode(optionsRoot);
+			GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.dupNode(optionsRoot);
 			lexerAST.addChild(lexerOptionsRoot);
-			List<org.antlr.v4.tool.GrammarAST> options = optionsRoot.getChildren();
+			List<GrammarAST> options = optionsRoot.getChildren();
-			for (org.antlr.v4.tool.GrammarAST o : options) {
+			for (GrammarAST o : options) {
 				String optionName = o.getChild(0).getText();
 				if ( !Grammar.doNotCopyOptionsToLexer.contains(optionName) ) {
 					lexerOptionsRoot.addChild(o);
@@ -363,8 +367,8 @@ public class Tool {
 		}
 
 		// MOVE lexer:: actions
-		List<org.antlr.v4.tool.GrammarAST> actionsWeMoved = new ArrayList<org.antlr.v4.tool.GrammarAST>();
+		List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
-		for (org.antlr.v4.tool.GrammarAST e : elements) {
+		for (GrammarAST e : elements) {
 			if ( e.getType()==ANTLRParser.AT ) {
 				if ( e.getChild(0).getText().equals("lexer") ) {
 					lexerAST.addChild(e);
@@ -373,16 +377,16 @@ public class Tool {
 			}
 		}
 		elements.removeAll(actionsWeMoved);
-		org.antlr.v4.tool.GrammarAST combinedRulesRoot =
+		GrammarAST combinedRulesRoot =
-		(org.antlr.v4.tool.GrammarAST)combinedAST.getFirstChildWithType(ANTLRParser.RULES);
+			(GrammarAST)combinedAST.getFirstChildWithType(ANTLRParser.RULES);
 		if ( combinedRulesRoot==null ) return lexerAST;
 
 		// MOVE lexer rules
 
-		org.antlr.v4.tool.GrammarAST lexerRulesRoot =
+		GrammarAST lexerRulesRoot =
-		(org.antlr.v4.tool.GrammarAST)adaptor.create(ANTLRParser.RULES, "RULES");
+			(GrammarAST)adaptor.create(ANTLRParser.RULES, "RULES");
 		lexerAST.addChild(lexerRulesRoot);
-		List<org.antlr.v4.tool.GrammarAST> rulesWeMoved = new ArrayList<org.antlr.v4.tool.GrammarAST>();
+		List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
 		List<GrammarASTWithOptions> rules = combinedRulesRoot.getChildren();
 		for (GrammarASTWithOptions r : rules) {
 			String ruleName = r.getChild(0).getText();
@@ -396,22 +400,24 @@ public class Tool {
 
 		// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
 		Map<String,String> litAliases =
-		Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
+			Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
 
 		if ( nLexicalRules==0 && (litAliases==null||litAliases.size()==0) &&
-		combinedGrammar.stringLiteralToTypeMap.size()==0 )
+			 combinedGrammar.stringLiteralToTypeMap.size()==0 )
 		{
 			// no rules, tokens{}, or 'literals' in grammar
 			return null;
 		}
 
-		// add strings from combined grammar (and imported grammars) into to lexer
+		Set<String> stringLiterals = combinedGrammar.getStringLiterals();
-		for (String lit : combinedGrammar.stringLiteralToTypeMap.keySet()) {
+		// add strings from combined grammar (and imported grammars) into lexer
+		// put them first as they are keywords; must resolve ambigs to these rules
+		for (String lit : stringLiterals) {
 			if ( litAliases!=null && litAliases.containsKey(lit) ) continue; // already has rule
 			// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
 			String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
 			// can't use wizard; need special node types
-			org.antlr.v4.tool.GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
+			GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
 			BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
 			AltAST alt = new AltAST(ANTLRParser.ALT);
 			TerminalAST slit = new TerminalAST(new org.antlr.runtime.CommonToken(ANTLRParser.STRING_LITERAL, lit));
@@ -420,10 +426,7 @@ public class Tool {
 			CommonToken idToken = new CommonToken(ANTLRParser.ID, rname);
 			litRule.addChild(new TerminalAST(idToken));
 			litRule.addChild(blk);
-			lexerRulesRoot.addChild(litRule);
+			lexerRulesRoot.getChildren().add(0, litRule); // add first
-
-//			(GrammarAST)
-//				wiz.create("(RULE ID["+rname+"] (BLOCK (ALT STRING_LITERAL["+lit+"])))");
 		}
 
 		System.out.println("after ="+combinedAST.toStringTree());

tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java

@@ -156,7 +156,8 @@ public class BasicSemanticChecks {
 		String fullyQualifiedName = nameToken.getInputStream().getSourceName();
 		File f = new File(fullyQualifiedName);
 		String fileName = f.getName();
-		if ( !Utils.stripFileExtension(fileName).equals(nameToken.getText()) ) {
+		if ( !Utils.stripFileExtension(fileName).equals(nameToken.getText()) &&
+		     !fileName.equals(Grammar.GRAMMAR_FROM_STRING_NAME)) {
 			g.tool.errMgr.grammarError(ErrorType.FILE_AND_GRAMMAR_NAME_DIFFER,
 									   fileName, nameToken, nameToken.getText(), fileName);
 		}

tool/src/org/antlr/v4/semantics/SemanticPipeline.java

@@ -166,39 +166,33 @@ public class SemanticPipeline {
 	}
 
 	void assignTokenTypes(Grammar g, CollectSymbols collector, SymbolChecks symcheck) {
-		if ( g.implicitLexerOwner!=null ) {
+		Grammar G = g.getOutermostGrammar(); // put in root, even if imported
-			// copy vocab from combined to implicit lexer
-			g.importVocab(g.implicitLexerOwner);
-			System.out.println("tokens="+g.tokenNameToTypeMap);
-			System.out.println("strings="+g.stringLiteralToTypeMap);
-		}
-		else {
-			Grammar G = g.getOutermostGrammar(); // put in root, even if imported
 
-			// DEFINE tokens { X='x'; } ALIASES
+		// DEFINE tokens { X='x'; } ALIASES
-			for (GrammarAST alias : collector.tokensDefs) {
+		for (GrammarAST alias : collector.tokensDefs) {
-				if ( alias.getType()== ANTLRParser.ASSIGN ) {
+			if ( alias.getType()== ANTLRParser.ASSIGN ) {
-					String name = alias.getChild(0).getText();
+				String name = alias.getChild(0).getText();
-					String lit = alias.getChild(1).getText();
+				String lit = alias.getChild(1).getText();
-					G.defineTokenAlias(name, lit);
+				G.defineTokenAlias(name, lit);
-				}
 			}
-
-			// DEFINE TOKEN TYPES FOR X : 'x' ; RULES
-			Map<String,String> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(g.ast);
-			if ( litAliases!=null ) {
-				for (String lit : litAliases.keySet()) {
-					G.defineTokenAlias(litAliases.get(lit), lit);
-				}
-			}
-
-			// DEFINE TOKEN TYPES FOR TOKEN REFS LIKE ID, INT
-			for (String id : symcheck.tokenIDs) { G.defineTokenName(id); }
-
-			// DEFINE TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
-			for (String s : collector.strings) { G.defineStringLiteral(s); }
-//			System.out.println("tokens="+G.tokenNameToTypeMap);
-//			System.out.println("strings="+G.stringLiteralToTypeMap);
 		}
+
+		// DEFINE TOKEN TYPES FOR X : 'x' ; RULES
+		/* done by previous import
+		   Map<String,String> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(g.ast);
+		   if ( litAliases!=null ) {
+			   for (String lit : litAliases.keySet()) {
+				   G.defineTokenAlias(litAliases.get(lit), lit);
+			   }
+		   }
+		   */
+
+		// DEFINE TOKEN TYPES FOR TOKEN REFS LIKE ID, INT
+		for (String id : symcheck.tokenIDs) { G.defineTokenName(id); }
+
+		// DEFINE TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
+		for (String s : collector.strings) { G.defineStringLiteral(s); }
+		System.out.println("tokens="+G.tokenNameToTypeMap);
+		System.out.println("strings="+G.stringLiteralToTypeMap);
 	}
 }

tool/src/org/antlr/v4/tool/ErrorType.java

@@ -50,8 +50,8 @@ public enum ErrorType {
 	DIR_NOT_FOUND("directory not found: <arg>", ErrorSeverity.ERROR),
 	OUTPUT_DIR_IS_FILE("output directory is a file: <arg>", ErrorSeverity.ERROR),
 	CANNOT_OPEN_FILE("cannot find or open file: <arg><if(arg2)>; reason: <arg2><endif>", ErrorSeverity.ERROR),
-	FILE_AND_GRAMMAR_NAME_DIFFER("", ErrorSeverity.ERROR),
+	FILE_AND_GRAMMAR_NAME_DIFFER("grammar name <arg> and file name <arg2> differ", ErrorSeverity.ERROR),
-	FILENAME_EXTENSION_ERROR("", ErrorSeverity.ERROR),
+//	FILENAME_EXTENSION_ERROR("", ErrorSeverity.ERROR),
 
 	INTERNAL_ERROR("internal error: <arg> <arg2><if(exception)>: <exception><endif>\n" +
 				   "<stackTrace; separator=\"\\n\">", ErrorSeverity.ERROR),

tool/src/org/antlr/v4/tool/Grammar.java

@@ -31,7 +31,7 @@ package org.antlr.v4.tool;
 
 import org.antlr.runtime.CommonTokenStream;
 import org.antlr.runtime.*;
-import org.antlr.runtime.tree.TreeWizard;
+import org.antlr.runtime.tree.*;
 import org.antlr.v4.Tool;
 import org.antlr.v4.misc.*;
 import org.antlr.v4.parse.*;
@@ -40,10 +40,13 @@ import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.atn.ATN;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.*;
+import org.antlr.v4.semantics.CollectSymbols;
 
 import java.util.*;
 
 public class Grammar implements AttributeResolver {
+	public static final String GRAMMAR_FROM_STRING_NAME = "<string>";
+
 	public static final Set doNotCopyOptionsToLexer =
         new HashSet() {
             {
@@ -71,9 +74,10 @@ public class Grammar implements AttributeResolver {
     public String text; // testing only
     public String fileName;
 
-    /** Was this created from a COMBINED grammar? */
+    /** Was this parser grammar created from a COMBINED grammar?  If so,
-    public Grammar implicitLexer;
+	 *  this is what we derived.
-    public Grammar implicitLexerOwner;
+	 */
+    public LexerGrammar implicitLexer;
 
     /** If we're imported, who imported us? If null, implies grammar is root */
     public Grammar parent;
@@ -86,6 +90,7 @@ public class Grammar implements AttributeResolver {
 	public List<Rule> indexToRule = new ArrayList<Rule>();
 
 	int ruleNumber = 0; // used to get rule indexes (0..n-1)
+	int stringLiteralRuleNumber = 0; // used to invent rule names for 'keyword', ';', ... (0..n-1)
 
 	/** The ATN that represents the grammar with edges labelled with tokens
 	 *  or epsilon.  It is more suitable to analysis than an AST representation.
@@ -156,14 +161,14 @@ public class Grammar implements AttributeResolver {
 
 	/** For testing */
 	public Grammar(String grammarText) throws org.antlr.runtime.RecognitionException {
-		this("<string>", grammarText, null);
+		this(GRAMMAR_FROM_STRING_NAME, grammarText, null);
 	}
 
 	/** For testing */
 	public Grammar(String grammarText, ANTLRToolListener listener)
 		throws org.antlr.runtime.RecognitionException
 	{
-		this("<string>", grammarText, listener);
+		this(GRAMMAR_FROM_STRING_NAME, grammarText, listener);
 	}
 
 	/** For testing; only builds trees; no sem anal */
@@ -365,8 +370,7 @@ public class Grammar implements AttributeResolver {
     }
 
 	public String getStringLiteralLexerRuleName(String lit) {
-		int ttype = getTokenType(lit);
+		return AUTO_GENERATED_TOKEN_NAME_PREFIX + stringLiteralRuleNumber++;
-		return AUTO_GENERATED_TOKEN_NAME_PREFIX +ttype;
 	}
 
     /** Return grammar directly imported by this grammar */
@@ -396,7 +400,6 @@ public class Grammar implements AttributeResolver {
 	 */
 	public String getTokenDisplayName(int ttype) {
 		String tokenName = null;
-		int index=0;
 		// inside any target's char range and is lexer grammar?
 		if ( isLexer() &&
 			 ttype >= Lexer.MIN_CHAR_VALUE && ttype <= Lexer.MAX_CHAR_VALUE )
@@ -411,6 +414,7 @@ public class Grammar implements AttributeResolver {
 				tokenName = typeToTokenList.get(ttype);
 				if ( tokenName!=null &&
 					 tokenName.startsWith(AUTO_GENERATED_TOKEN_NAME_PREFIX) &&
+					 ttype < typeToStringLiteralList.size() &&
 				     typeToStringLiteralList.get(ttype)!=null)
 				{
 					tokenName = typeToStringLiteralList.get(ttype);
@@ -420,7 +424,7 @@ public class Grammar implements AttributeResolver {
 				tokenName = String.valueOf(ttype);
 			}
 		}
-		//System.out.println("getTokenDisplayName ttype="+ttype+", index="+index+", name="+tokenName);
+//		System.out.println("getTokenDisplayName ttype="+ttype+", name="+tokenName);
 		return tokenName;
 	}
 
@@ -433,9 +437,12 @@ public class Grammar implements AttributeResolver {
 	public String[] getTokenNames() {
 		int numTokens = getMaxTokenType();
 		String[] tokenNames = new String[numTokens+1];
-		for (String t : tokenNameToTypeMap.keySet()) {
+		for (String tokenName : tokenNameToTypeMap.keySet()) {
-			Integer ttype = tokenNameToTypeMap.get(t);
+			Integer ttype = tokenNameToTypeMap.get(tokenName);
-			if ( ttype>0 ) tokenNames[ttype] = t;
+			if ( tokenName!=null && tokenName.startsWith(AUTO_GENERATED_TOKEN_NAME_PREFIX) ) {
+				tokenName = typeToStringLiteralList.get(ttype);
+			}
+			if ( ttype>0 ) tokenNames[ttype] = tokenName;
 		}
 		return tokenNames;
 	}
@@ -493,14 +500,20 @@ public class Grammar implements AttributeResolver {
 		return maxTokenType;
 	}
 
-	public void importVocab(Grammar g) {
+	public void importVocab(Grammar importG) {
-		this.tokenNameToTypeMap.putAll( g.tokenNameToTypeMap );
+		for (String tokenName: importG.tokenNameToTypeMap.keySet()) {
-		this.stringLiteralToTypeMap.putAll( g.stringLiteralToTypeMap );
+			defineTokenName(tokenName, importG.tokenNameToTypeMap.get(tokenName));
-		int max = Math.max(this.typeToTokenList.size(), g.typeToTokenList.size());
+		}
+		for (String tokenName: importG.stringLiteralToTypeMap.keySet()) {
+			defineStringLiteral(tokenName, importG.stringLiteralToTypeMap.get(tokenName));
+		}
+//		this.tokenNameToTypeMap.putAll( importG.tokenNameToTypeMap );
+//		this.stringLiteralToTypeMap.putAll( importG.stringLiteralToTypeMap );
+		int max = Math.max(this.typeToTokenList.size(), importG.typeToTokenList.size());
 		this.typeToTokenList.setSize(max);
-		for (int ttype=0; ttype<g.typeToTokenList.size(); ttype++) {
+		for (int ttype=0; ttype<importG.typeToTokenList.size(); ttype++) {
 			maxTokenType = Math.max(maxTokenType, ttype);
-			this.typeToTokenList.set(ttype, g.typeToTokenList.get(ttype));
+			this.typeToTokenList.set(ttype, importG.typeToTokenList.get(ttype));
 		}
 	}
 
@@ -518,7 +531,11 @@ public class Grammar implements AttributeResolver {
 	}
 
 	public int defineStringLiteral(String lit) {
+		if ( stringLiteralToTypeMap.containsKey(lit) ) {
+			return stringLiteralToTypeMap.get(lit);
+		}
 		return defineStringLiteral(lit, getNewTokenType());
+
 	}
 
 	public int defineStringLiteral(String lit, int ttype) {
@@ -528,7 +545,7 @@ public class Grammar implements AttributeResolver {
 			if ( ttype>=typeToStringLiteralList.size() ) {
 				typeToStringLiteralList.setSize(ttype+1);
 			}
-			typeToStringLiteralList.set(ttype, text);
+			typeToStringLiteralList.set(ttype, lit);
 
 			setTokenForType(ttype, lit);
 			return ttype;
@@ -652,6 +669,7 @@ public class Grammar implements AttributeResolver {
 		Map<String,String> lexerRuleToStringLiteral = new HashMap<String,String>();
 
         for (GrammarASTWithOptions r : ruleNodes) {
+			//System.out.println(r.toStringTree());
             String ruleName = r.getChild(0).getText();
             if ( Character.isUpperCase(ruleName.charAt(0)) ) {
 				Map nodes = new HashMap();
@@ -667,6 +685,15 @@ public class Grammar implements AttributeResolver {
 		return lexerRuleToStringLiteral;
 	}
 
+	public Set<String> getStringLiterals() {
+		GrammarASTAdaptor adaptor = new GrammarASTAdaptor();
+		BufferedTreeNodeStream nodes = new BufferedTreeNodeStream(adaptor,ast);
+		CollectSymbols collector = new CollectSymbols(nodes,this);
+		collector.downup(ast); // no side-effects; compute lists
+		return collector.strings;
+	}
+
+
 	public void setLookaheadDFA(int decision, DFA lookaheadDFA) {
 		decisionDFAs.put(decision, lookaheadDFA);
 	}

tool/src/org/antlr/v4/tool/LexerGrammar.java

@@ -37,6 +37,9 @@ import org.stringtemplate.v4.misc.MultiMap;
 public class LexerGrammar extends Grammar {
 	public static final String DEFAULT_MODE_NAME = "DEFAULT_MODE";
 
+	/** The grammar from which this lexer grammar was derived (if implicit) */
+    public Grammar implicitLexerOwner;
+
 	/** DEFAULT_MODE rules are added first due to grammar syntax order */
 	public MultiMap<String, Rule> modes = new MultiMap<String, Rule>();
 

tool/test/org/antlr/v4/test/TestRewriteAST.java

@@ -60,6 +60,19 @@ public class TestRewriteAST extends BaseTest {
 		assertEquals("abc\n", found);
 	}
 
+	@Test public void testSingleLabeledToken() throws Exception {
+		String grammar =
+			"grammar T;\n" +
+			"options {output=AST;}\n" +
+			"a : x=ID -> $x;\n" +
+			"ID : 'a'..'z'+ ;\n" +
+			"INT : '0'..'9'+;\n" +
+			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
+		String found = execParser("T.g", grammar, "TParser", "TLexer",
+				    "a", "abc", debug);
+		assertEquals("abc\n", found);
+	}
+
 	@Test public void testSingleTokenToNewNode() throws Exception {
 		String grammar =
 			"grammar T;\n" +
@@ -86,20 +99,6 @@ public class TestRewriteAST extends BaseTest {
 		assertEquals("(x INT)\n", found);
 	}
 
-	@Test public void testSingleTokenToNewNode2() throws Exception {
-		// Allow creation of new nodes w/o args.
-		String grammar =
-			"grammar TT;\n" +
-			"options {output=AST;}\n" +
-			"a : ID -> ID[ ];\n" +
-			"ID : 'a'..'z'+ ;\n" +
-			"INT : '0'..'9'+;\n" +
-			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
-		String found = execParser("TT.g", grammar, "TTParser", "TTLexer",
-				    "a", "abc", debug);
-		assertEquals("ID\n", found);
-	}
-
 	@Test public void testSingleCharLiteral() throws Exception {
 		String grammar =
 			"grammar T;\n" +
@@ -199,7 +198,7 @@ public class TestRewriteAST extends BaseTest {
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {DUH;}\n" +
-			"a : ID INT ID INT -> ^( DUH ID ^( DUH INT) )+ ;\n" +
+			"a : ID INT ID INT -> ^( DUH ID ^( DUH INT) )* ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
@@ -234,19 +233,6 @@ public class TestRewriteAST extends BaseTest {
 		assertEquals("a b\n", found);
 	}
 
-	@Test public void testPositiveClosureSingleToken() throws Exception {
-		String grammar =
-			"grammar T;\n" +
-			"options {output=AST;}\n" +
-			"a : ID ID -> ID+ ;\n" +
-			"ID : 'a'..'z'+ ;\n" +
-			"INT : '0'..'9'+;\n" +
-			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
-		String found = execParser("T.g", grammar, "TParser", "TLexer",
-				    "a", "a b", debug);
-		assertEquals("a b\n", found);
-	}
-
 	@Test public void testOptionalSingleRule() throws Exception {
 		String grammar =
 			"grammar T;\n" +
@@ -407,7 +393,7 @@ public class TestRewriteAST extends BaseTest {
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : 'var' (ID ':' type ';')+ -> ^('var' ^(':' ID type)+) ;\n" +
+			"a : 'var' (ID ':' type ';')+ -> ^('var' ^(':' ID type)*) ;\n" +
 			"type : 'int' | 'float' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -422,7 +408,7 @@ public class TestRewriteAST extends BaseTest {
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {VAR;}\n" +
-			"a : ID (',' ID)*-> ^(VAR ID)+ ;\n" +
+			"a : ID (',' ID)*-> ^(VAR ID)* ;\n" +
 			"type : 'int' | 'float' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -452,7 +438,7 @@ public class TestRewriteAST extends BaseTest {
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {VAR;}\n" +
-			"a : ID (',' ID)*-> ^(VAR[\"var\"] ID)+ ;\n" +
+			"a : ID (',' ID)*-> ^(VAR[\"var\"] ID)* ;\n" +
 			"type : 'int' | 'float' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -467,7 +453,7 @@ public class TestRewriteAST extends BaseTest {
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {BLOCK;}\n" +
-			"a : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;\n" +
+			"a : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID*) ;\n" +
 			"type : 'int' | 'float' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -482,7 +468,7 @@ public class TestRewriteAST extends BaseTest {
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {BLOCK;}\n" +
-			"a : lc='{' ID+ '}' -> ^(BLOCK[$lc,\"block\"] ID+) ;\n" +
+			"a : lc='{' ID+ '}' -> ^(BLOCK[$lc,\"block\"] ID*) ;\n" +
 			"type : 'int' | 'float' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -551,7 +537,7 @@ public class TestRewriteAST extends BaseTest {
 			"options {output=AST;}\n" +
 			"tokens {BLOCK;}\n" +
 			"a : b b ;\n" +
-			"b : ID ( ID (last=ID -> $last)+ ) ';'\n" + // get last ID
+			"b : ID ( ID (last=ID -> $last)* ) ';'\n" + // get last ID
 			"  | INT\n" + // should still get auto AST construction
 			"  ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
@@ -621,11 +607,11 @@ public class TestRewriteAST extends BaseTest {
 	}
 
 	@Test public void testCopySemanticsForRules2() throws Exception {
-		// copy type as a root for each invocation of (...)+ in rewrite
+		// copy type as a root for each invocation of (...)* in rewrite
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : type ID (',' ID)* ';' -> ^(type ID)+ ;\n" +
+			"a : type ID (',' ID)* ';' -> ^(type ID)* ;\n" +
 			"type : 'int' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
@@ -636,11 +622,11 @@ public class TestRewriteAST extends BaseTest {
 
 	@Test public void testCopySemanticsForRules3() throws Exception {
 		// copy type *and* modifier even though it's optional
-		// for each invocation of (...)+ in rewrite
+		// for each invocation of (...)* in rewrite
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)+ ;\n" +
+			"a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)* ;\n" +
 			"type : 'int' ;\n" +
 			"modifier : 'public' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
@@ -652,11 +638,11 @@ public class TestRewriteAST extends BaseTest {
 
 	@Test public void testCopySemanticsForRules3Double() throws Exception {
 		// copy type *and* modifier even though it's optional
-		// for each invocation of (...)+ in rewrite
+		// for each invocation of (...)* in rewrite
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)+ ^(type modifier? ID)+ ;\n" +
+			"a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)* ^(type modifier? ID)* ;\n" +
 			"type : 'int' ;\n" +
 			"modifier : 'public' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
@@ -668,12 +654,12 @@ public class TestRewriteAST extends BaseTest {
 
 	@Test public void testCopySemanticsForRules4() throws Exception {
 		// copy type *and* modifier even though it's optional
-		// for each invocation of (...)+ in rewrite
+		// for each invocation of (...)* in rewrite
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {MOD;}\n" +
-			"a : modifier? type ID (',' ID)* ';' -> ^(type ^(MOD modifier)? ID)+ ;\n" +
+			"a : modifier? type ID (',' ID)* ';' -> ^(type ^(MOD modifier)? ID)* ;\n" +
 			"type : 'int' ;\n" +
 			"modifier : 'public' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
@@ -688,7 +674,7 @@ public class TestRewriteAST extends BaseTest {
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {MOD;}\n" +
-			"a : ID (',' ID)* ';' -> ID+ ID+ ;\n"+
+			"a : ID (',' ID)* ';' -> ID* ID* ;\n"+
 			"ID : 'a'..'z'+ ;\n" +
 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
 		String found = execParser("T.g", grammar, "TParser", "TLexer",
@@ -728,7 +714,7 @@ public class TestRewriteAST extends BaseTest {
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : 'int' ID (',' ID)* ';' -> ^('int' ID+) ;\n" +
+			"a : 'int' ID (',' ID)* ';' -> ^('int' ID*) ;\n" +
 			"op : '+'|'-' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -756,7 +742,7 @@ public class TestRewriteAST extends BaseTest {
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : 'int' ID (',' ID)* ';' -> ^('int' ID)+ ;\n" +
+			"a : 'int' ID (',' ID)* ';' -> ^('int' ID)* ;\n" +
 			"op : '+'|'-' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -771,7 +757,7 @@ public class TestRewriteAST extends BaseTest {
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : 'int' ID ':' INT (',' ID ':' INT)* ';' -> ^('int' ID INT)+ ;\n" +
+			"a : 'int' ID ':' INT (',' ID ':' INT)* ';' -> ^('int' ID INT)* ;\n" +
 			"op : '+'|'-' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
@@ -993,7 +979,7 @@ public class TestRewriteAST extends BaseTest {
 		String grammar =
 			"grammar T;\n" +
 			"options { output = AST; } \n" +
-			"a: (INT|ID)+ -> INT+ ID+ ;\n" +
+			"a: (INT|ID)+ -> INT* ID* ;\n" +
 			"INT: '0'..'9'+;\n" +
 			"ID : 'a'..'z'+;\n" +
 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
@@ -1049,8 +1035,6 @@ public class TestRewriteAST extends BaseTest {
 	}
 
 	@Test public void testOptionalSubruleWithoutRealElements() throws Exception {
-		// copy type *and* modifier even though it's optional
-		// for each invocation of (...)+ in rewrite
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;} \n" +
@@ -1074,7 +1058,7 @@ public class TestRewriteAST extends BaseTest {
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
 			"tokens {BLOCK;}\n" +
-			"a : ID ID INT INT INT -> (ID INT)+;\n"+
+			"a : ID ID INT INT INT -> (ID INT)*;\n"+
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+; \n" +
 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
@@ -1124,7 +1108,7 @@ public class TestRewriteAST extends BaseTest {
 		String grammar =
 			"grammar T;\n" +
 			"options {output=AST;}\n" +
-			"a : ID? INT -> ID+ INT ;\n" +
+			"a : ID? INT -> ID* INT ;\n" +
 			"op : '+'|'-' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +

tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java

@@ -95,7 +95,7 @@ public class TestTokenTypeAssignment extends BaseTest {
 		String[] typeToTokenName = g.getTokenNames();
 		Set<String> tokens = new HashSet<String>();
 		for (String t : typeToTokenName) if ( t!=null ) tokens.add(t);
-		assertEquals("[E]", tokens.toString());
+		assertEquals("[E, 'x']", tokens.toString());
 	}
 
 	@Test public void testCombinedGrammarWithRefToLiteralButNoTokenIDRef() throws Exception {
@@ -155,7 +155,7 @@ public class TestTokenTypeAssignment extends BaseTest {
 			"tokens { B='}'; }\n"+
 			"a : A '}' {System.out.println(input);} ;\n"+
 			"A : 'a' ;\n" +
-			"B : '}' {/* */} ;\n"+
+			"B : '}' ;\n"+
 			"WS : (' '|'\\n') {skip();} ;";
 		String found = execParser("P.g", grammar, "PParser", "PLexer",
 								  "a", "a}", false);
@@ -164,7 +164,7 @@ public class TestTokenTypeAssignment extends BaseTest {
 
 	protected void checkSymbols(Grammar g,
 								String rulesStr,
-								String tokensStr)
+								String allValidTokensStr)
 		throws Exception
 	{
 		Tool antlr = new Tool();
@@ -175,7 +175,7 @@ public class TestTokenTypeAssignment extends BaseTest {
 		for (String t : typeToTokenName) if ( t!=null ) tokens.add(t);
 
 		// make sure expected tokens are there
-		StringTokenizer st = new StringTokenizer(tokensStr, ", ");
+		StringTokenizer st = new StringTokenizer(allValidTokensStr, ", ");
 		while ( st.hasMoreTokens() ) {
 			String tokenName = st.nextToken();
 			assertTrue("token "+tokenName+" expected, but was undefined",