symbol issue test working again

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8894]

tool/src/org/antlr/v4/Tool.java

@@ -375,6 +375,9 @@ public class Tool {
 	 	grammars.
 	 */
 	public void mergeImportedGrammars(Grammar rootGrammar) {
+		List<Grammar> imports = rootGrammar.getAllImportedGrammars();
+		if ( imports==null ) return;
+
 		GrammarAST root = rootGrammar.ast;
 		GrammarAST id = (GrammarAST) root.getChild(0);
 		GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream());
@@ -397,9 +400,6 @@ public class Tool {
 			for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText());
 		}
 
-		List<Grammar> imports = rootGrammar.getAllImportedGrammars();
-		if ( imports==null ) return;
-
 		for (Grammar imp : imports) {
 			GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS);
 			if ( imp_tokensRoot!=null ) {
@@ -498,8 +498,8 @@ public class Tool {
 	 *  tokenVocab or tokens{} section.
 	 *
 	 *  Side-effects: it removes children from GRAMMAR & RULES nodes
-	 *                in combined AST. Careful: nodes are shared between
+	 *                in combined AST.  Anything cut out is dup'd before
-	 *                trees after this call.
+	 *                adding to lexer to avoid "who's ur daddy" issues
 	 */
 	public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
 		GrammarRootAST combinedAST = combinedGrammar.ast;
@@ -525,7 +525,7 @@ public class Tool {
 			for (GrammarAST o : options) {
 				String optionName = o.getChild(0).getText();
 				if ( !Grammar.doNotCopyOptionsToLexer.contains(optionName) ) {
-					lexerOptionsRoot.addChild(o);
+					lexerOptionsRoot.addChild((Tree)adaptor.dupTree(o));
 				}
 			}
 		}
@@ -535,7 +535,7 @@ public class Tool {
 		for (GrammarAST e : elements) {
 			if ( e.getType()==ANTLRParser.AT ) {
 				if ( e.getChild(0).getText().equals("lexer") ) {
-					lexerAST.addChild(e);
+					lexerAST.addChild((Tree)adaptor.dupTree(e));
 					actionsWeMoved.add(e);
 				}
 			}
@@ -555,7 +555,7 @@ public class Tool {
 		for (GrammarASTWithOptions r : rules) {
 			String ruleName = r.getChild(0).getText();
 			if ( Character.isUpperCase(ruleName.charAt(0)) ) {
-				lexerRulesRoot.addChild(r);
+				lexerRulesRoot.addChild((Tree)adaptor.dupTree(r));
 				rulesWeMoved.add(r);
 			}
 		}
@@ -593,10 +593,10 @@ public class Tool {
 			lexerRulesRoot.getChildren().add(0, litRule); // add first
 		}
 
-		lexerRulesRoot.freshenParentAndChildIndexesDeeply();
+		lexerAST.freshenParentAndChildIndexesDeeply();
-		combinedRulesRoot.freshenParentAndChildIndexesDeeply();
+		combinedAST.freshenParentAndChildIndexesDeeply();
 
-		System.out.println("after ="+combinedAST.toStringTree());
+		System.out.println("after extract implicit lexer ="+combinedAST.toStringTree());
 		System.out.println("lexer ="+lexerAST.toStringTree());
 		return lexerAST;
 	}

tool/src/org/antlr/v4/automata/ParserATNFactory.java

@@ -137,7 +137,9 @@ public class ParserATNFactory implements ATNFactory {
 	}
 
 	/** Not valid for non-lexers */
-	public Handle range(GrammarAST a, GrammarAST b) { throw new UnsupportedOperationException(); }
+	public Handle range(GrammarAST a, GrammarAST b) {
+		throw new UnsupportedOperationException();
+	}
 
 	/** ~atom only */
 	/*

tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java

@@ -366,7 +366,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
 		if ( optionID.getText().equals("tokenVocab") &&
 			 g.parent!=null ) // only allow tokenVocab option in root grammar
 		{
-			g.tool.errMgr.grammarWarning(ErrorType.TOKEN_VOCAB_IN_DELEGATE,
+			g.tool.errMgr.grammarError(ErrorType.TOKEN_VOCAB_IN_DELEGATE,
 									   g.fileName,
 									   optionID,
 									   g.name);
@@ -455,7 +455,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
 			 options.get("rewrite").equals("true") )
 		{
 			String fileName = altStart.getInputStream().getSourceName();
-			g.tool.errMgr.grammarWarning(ErrorType.REWRITE_FOR_MULTI_ELEMENT_ALT,
+			g.tool.errMgr.grammarError(ErrorType.REWRITE_FOR_MULTI_ELEMENT_ALT,
 									   fileName,
 									   altStart,
 									   alt);
@@ -470,13 +470,13 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
 		String ruleName = rule.getChild(0).getText();
 		String fileName = elementRoot.token.getInputStream().getSourceName();
 		if ( options==null || !options.get("output").equals("AST") ) {
-			g.tool.errMgr.grammarWarning(ErrorType.AST_OP_WITH_NON_AST_OUTPUT_OPTION,
+			g.tool.errMgr.grammarError(ErrorType.AST_OP_WITH_NON_AST_OUTPUT_OPTION,
 									   fileName,
 									   elementRoot.token,
 									   op.getText());
 		}
 		if ( options!=null && options.get("output")==null ) {
-			g.tool.errMgr.grammarWarning(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION,
+			g.tool.errMgr.grammarError(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION,
 									   fileName,
 									   elementRoot.token,
 									   ruleName);
@@ -484,7 +484,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
 		if ( op.hasAncestor(ANTLRParser.ALT_REWRITE) ) {
 			GrammarAST rew = (GrammarAST)op.getAncestor(ANTLRParser.ALT_REWRITE);
 			int altNum = rew.getChildIndex() + 1; // alts are 1..n
-			g.tool.errMgr.grammarWarning(ErrorType.AST_OP_IN_ALT_WITH_REWRITE,
+			g.tool.errMgr.grammarError(ErrorType.AST_OP_IN_ALT_WITH_REWRITE,
 									   fileName,
 									   elementRoot.token,
 									   ruleName,
@@ -496,7 +496,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
 		String ruleName = currentRuleAST.getChild(0).getText();
 		String fileName = elementRoot.token.getInputStream().getSourceName();
 		if ( options!=null && options.get("output")==null ) {
-			g.tool.errMgr.grammarWarning(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION,
+			g.tool.errMgr.grammarError(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION,
 									   fileName,
 									   elementRoot.token,
 									   ruleName);

tool/src/org/antlr/v4/semantics/SemanticPipeline.java

@@ -91,7 +91,7 @@ public class SemanticPipeline {
 			g.defineRule(r);
 		}
 		for (GrammarAST a : collector.namedActions) {
-			g.defineAction((GrammarAST)a.getParent());
+			g.defineAction(a);
 		}
 
 		// LINK (outermost) ALT NODES WITH Alternatives
@@ -122,7 +122,7 @@ public class SemanticPipeline {
 		// CHECK ATTRIBUTE EXPRESSIONS FOR SEMANTIC VALIDITY
 		AttributeChecks.checkAllAttributeExpressions(g);
 
-		symcheck.checkForRewriteIssues();
+		symcheck.checkForUndefinedTokensInRewrite();
 
 		UseDefAnalyzer.checkRewriteElementsPresentOnLeftSide(g);
 		UseDefAnalyzer.trackTokenRuleRefsInActions(g);

tool/src/org/antlr/v4/semantics/SymbolChecks.java

@@ -30,7 +30,6 @@
 package org.antlr.v4.semantics;
 
 import org.antlr.runtime.Token;
-import org.antlr.runtime.misc.DoubleKeyMap;
 import org.antlr.v4.parse.ANTLRParser;
 import org.antlr.v4.tool.*;
 
@@ -48,9 +47,9 @@ public class SymbolChecks {
     Map<String, Rule> nameToRuleMap = new HashMap<String, Rule>();
 	Set<String> tokenIDs = new HashSet<String>();
     Set<String> globalScopeNames = new HashSet<String>();
-//    Map<String, Set<String>> actionScopeToActionNames = new HashMap<String, Set<String>>();
+    Map<String, Set<String>> actionScopeToActionNames = new HashMap<String, Set<String>>();
-	DoubleKeyMap<String, String, GrammarAST> namedActions =
+//	DoubleKeyMap<String, String, GrammarAST> namedActions =
-		new DoubleKeyMap<String, String, GrammarAST>();
+//		new DoubleKeyMap<String, String, GrammarAST>();
 
 	public ErrorManager errMgr;
 
@@ -77,6 +76,7 @@ public class SymbolChecks {
 		//checkForImportedRuleIssues(collector.qualifiedRulerefs);
 		// done in sem pipe for now
         checkForRuleConflicts(collector.rules);         // sets nameToRuleMap
+		checkActionRedefinitions(collector.namedActions);
         checkTokenAliasRedefinitions(collector.tokensDefs);
         //checkRuleArgs(collector.rulerefs);
         checkForTokenConflicts(collector.tokenIDRefs);  // sets tokenIDs
@@ -104,6 +104,35 @@ public class SymbolChecks {
         }
     }
 
+	  public void checkActionRedefinitions(List<GrammarAST> actions) {
+        if ( actions==null ) return;
+        String scope = g.getDefaultActionScope();
+        String name = null;
+        GrammarAST nameNode = null;
+        for (GrammarAST ampersandAST : actions) {
+            nameNode = (GrammarAST)ampersandAST.getChild(0);
+            if ( ampersandAST.getChildCount()==2 ) {
+                name = nameNode.getText();
+            }
+            else {
+                scope = nameNode.getText();
+                name = ampersandAST.getChild(1).getText();
+            }
+            Set<String> scopeActions = actionScopeToActionNames.get(scope);
+            if ( scopeActions==null ) { // init scope
+                scopeActions = new HashSet<String>();
+                actionScopeToActionNames.put(scope, scopeActions);
+            }
+            if ( !scopeActions.contains(name) ) {
+                scopeActions.add(name);
+            }
+            else {
+                errMgr.grammarError(ErrorType.ACTION_REDEFINITION,
+                                          g.fileName, nameNode.token, name);
+            }
+        }
+    }
+
     public void checkScopeRedefinitions(List<AttributeDict> dicts) {
         if ( dicts ==null ) return;
         for (int i=0; i< dicts.size(); i++) {
@@ -122,7 +151,7 @@ public class SymbolChecks {
 
 	/** Catch:
 		tokens { A='a'; A; }		can't redefine token type if has alias
-	 	tokens { A; A='a'; }
+	 	tokens { A; A='a'; }		can't redefine token type if has alias
 	 	tokens { A='a'; A='b'; }	can't have two aliases for single token type
 		tokens { A='a'; B='a'; }	can't have to token types for same string alias
 	 */
@@ -135,6 +164,7 @@ public class SymbolChecks {
         for (int i=0; i<aliases.size(); i++) {
             GrammarAST a = aliases.get(i);
 			GrammarAST idNode = a;
+			if ( a.getChildCount()>0 ) idNode = (GrammarAST)a.getChild(0);
 			GrammarAST prevToken = aliasTokenNames.get(idNode.getText());
 			GrammarAST stringNode = null;
 			if ( a.getChildCount()>0 ) stringNode = (GrammarAST)a.getChild(1);
@@ -160,12 +190,15 @@ public class SymbolChecks {
 				}
 			}
 			if ( prevString!=null ) {
+				// A='a' and A='a' are ok but not B='a' and A='a' are ok
+				if ( !prevString.getChild(0).getText().equals(idNode.getText()) ) {
 					errMgr.grammarError(ErrorType.TOKEN_STRING_REASSIGNMENT,
 										a.g.fileName, idNode.token, idNode.getText()+"="+stringNode.getText(),
 										prevString.getChild(0).getText());
 				}
 			}
 		}
+	}
 
     public void checkForTokenConflicts(List<GrammarAST> tokenIDRefs) {
         for (GrammarAST a : tokenIDRefs) {
@@ -179,8 +212,8 @@ public class SymbolChecks {
         }
     }
 
-	public void checkForRewriteIssues() {
+	public void checkForUndefinedTokensInRewrite() {
-		// Ensure that all tokens refer to on the right if -> have been defined.
+		// Ensure that all tokens refs on the right of -> have been defined.
 		for (GrammarAST elem : collector.rewriteElements) {
 			if ( elem.getType()==ANTLRParser.TOKEN_REF ) {
 				int ttype = g.getTokenType(elem.getText());

tool/src/org/antlr/v4/semantics/SymbolCollector.java

@@ -82,7 +82,7 @@ public class SymbolCollector extends GrammarTreeVisitor {
 
 	@Override
 	public void globalNamedAction(GrammarAST scope, GrammarAST ID, ActionAST action) {
-		namedActions.add(ID);
+		namedActions.add((GrammarAST)ID.getParent());
 		action.resolver = g;
 	}
 

tool/src/org/antlr/v4/tool/AltAST.java

@@ -33,7 +33,7 @@ import org.antlr.runtime.Token;
 import org.antlr.runtime.tree.Tree;
 import org.antlr.v4.parse.ANTLRParser;
 
-/** An ALT or ALT_REWRITE node (left of ->) */
+/** An ALT (which can be child of ALT_REWRITE node) */
 public class AltAST extends GrammarAST {
 	public Alternative alt;
 
@@ -47,8 +47,11 @@ public class AltAST extends GrammarAST {
 	public AltAST(int type, Token t) { super(type, t); }
 
 	public GrammarAST getRewrite() {
-		if ( getType() == ANTLRParser.ALT ) return null;
+		// ^(ALT_REWRITE ^(ALT ...) ^(-> ...)) ??
-		return (GrammarAST)getChild(1); // ^(ALT_REWRITE ^(ALT ...) ^(-> ...))
+		if ( getParent().getType() == ANTLRParser.ALT_REWRITE ) {
+			return (GrammarAST)getParent().getChild(1);
+		}
+		return null;
 	}
 
 	@Override

tool/src/org/antlr/v4/tool/ErrorManager.java

@@ -29,7 +29,6 @@
 
 package org.antlr.v4.tool;
 
-import org.antlr.runtime.Token;
 import org.antlr.v4.Tool;
 import org.stringtemplate.v4.*;
 import org.stringtemplate.v4.misc.*;
@@ -98,6 +97,7 @@ public class ErrorManager {
 				if ( i>0 ) attr += i + 1;
 				messageST.add(attr, msg.args[i]);
 			}
+			if ( msg.args.length<2 ) messageST.add("arg2", null); // some messages ref arg2
 		}
 		if ( msg.e!=null ) {
 			messageST.add("exception", msg.e);
@@ -156,7 +156,10 @@ public class ErrorManager {
 								   org.antlr.runtime.RecognitionException antlrException,
 								   Object... args)
 	{
-		errors++;
+		switch ( etype.severity ) {
+			case WARNING: warnings++; break;
+			case ERROR: errors++; break;
+		}
 		ANTLRMessage msg = new GrammarSyntaxMessage(etype,fileName,token,antlrException,args);
 		tool.error(msg);
 	}
@@ -185,12 +188,18 @@ public class ErrorManager {
      * @param args The arguments to pass to the StringTemplate
      */
 	public void toolError(ErrorType errorType, Object... args) {
-        errors++;
+		switch ( errorType.severity ) {
+			case WARNING: warnings++; break;
+			case ERROR: errors++; break;
+		}
         tool.error(new ToolMessage(errorType, args));
 	}
 
 	public void toolError(ErrorType errorType, Throwable e, Object... args) {
-        errors++;
+		switch ( errorType.severity ) {
+			case WARNING: warnings++; break;
+			case ERROR: errors++; break;
+		}
         tool.error(new ToolMessage(errorType, e, args));
 	}
 
@@ -199,21 +208,14 @@ public class ErrorManager {
 							 org.antlr.runtime.Token token,
 							 Object... args)
 	{
-		errors++;
+		switch ( etype.severity ) {
+			case WARNING: warnings++; break;
+			case ERROR: errors++; break;
+		}
         ANTLRMessage msg = new GrammarSemanticsMessage(etype,fileName,token,args);
         tool.error(msg);
     }
 
-    public void grammarWarning(ErrorType etype,
-                                      String fileName,
-                                      Token token,
-                                      Object... args)
-    {
-        warnings++;
-        ANTLRMessage msg = new GrammarSemanticsMessage(etype,fileName,token,args);
-        tool.warning(msg);
-    }
-
 	public void leftRecursionCycles(String fileName, Collection cycles) {
 		errors++;
 		ANTLRMessage msg = new LeftRecursionCyclesMessage(fileName, cycles);

tool/src/org/antlr/v4/tool/ErrorType.java

@@ -99,7 +99,7 @@ public enum ErrorType {
 	RULE_HAS_NO_ARGS("rule <arg> has no defined parameters", ErrorSeverity.ERROR),
 	ARGS_ON_TOKEN_REF("token reference <arg> may not have parameters", ErrorSeverity.ERROR),
 	RULE_REF_AMBIG_WITH_RULE_IN_ALT("", ErrorSeverity.ERROR),
-	ILLEGAL_OPTION("illegal option <arg>", ErrorSeverity.ERROR),
+	ILLEGAL_OPTION("illegal option <arg>", ErrorSeverity.WARNING),
 	LIST_LABEL_INVALID_UNLESS_RETVAL_STRUCT("", ErrorSeverity.ERROR),
 	REWRITE_ELEMENT_NOT_PRESENT_ON_LHS("reference to rewrite element <arg> not found to left of ->", ErrorSeverity.ERROR),
     UNDEFINED_TOKEN_REF_IN_REWRITE("token <arg> in rewrite is undefined", ErrorSeverity.ERROR),

tool/src/org/antlr/v4/tool/Grammar.java

@@ -29,14 +29,11 @@
 
 package org.antlr.v4.tool;
 
-import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.tree.*;
-import org.antlr.runtime.*;
-import org.antlr.runtime.tree.TreeWizard;
 import org.antlr.v4.Tool;
 import org.antlr.v4.misc.*;
 import org.antlr.v4.parse.*;
-import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.*;
-import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.atn.ATN;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.*;
@@ -172,7 +169,14 @@ public class Grammar implements AttributeResolver {
 		this(GRAMMAR_FROM_STRING_NAME, grammarText, listener);
 	}
 
-	/** For testing; only builds trees; no sem anal */
+	/** For testing; builds trees, does sem anal */
+	public Grammar(String fileName, String grammarText)
+		throws org.antlr.runtime.RecognitionException
+	{
+		this(fileName, grammarText, null);
+	}
+
+	/** For testing; builds trees, does sem anal */
 	public Grammar(String fileName, String grammarText, ANTLRToolListener listener)
 		throws org.antlr.runtime.RecognitionException
 	{
@@ -182,20 +186,18 @@ public class Grammar implements AttributeResolver {
 		this.tool.addListener(listener);
 		org.antlr.runtime.ANTLRStringStream in = new org.antlr.runtime.ANTLRStringStream(grammarText);
 		in.name = fileName;
-		ANTLRLexer lexer = new ANTLRLexer(in);
-		CommonTokenStream tokens = new CommonTokenStream(lexer);
-		ToolANTLRParser p = new ToolANTLRParser(tokens,tool);
-		p.setTreeAdaptor(new GrammarASTAdaptor(in));
-		ParserRuleReturnScope r = p.grammarSpec();
-		if ( r.getTree() instanceof GrammarRootAST ) {
-			this.ast = (GrammarRootAST)r.getTree();
-			this.ast.hasErrors = p.getNumberOfSyntaxErrors()>0;
-			this.name = ((GrammarAST)ast.getChild(0)).getText();
 
-			GrammarTransformPipeline transform = new GrammarTransformPipeline();
+		this.ast = tool.load(in);
-			transform.process(ast);
+		// ensure each node has pointer to surrounding grammar
-		}
+		final Grammar thiz = this;
+		TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor());
+		v.visit(ast, new TreeVisitorAction() {
+			public Object pre(Object t) { ((GrammarAST)t).g = thiz; return t; }
+			public Object post(Object t) { return t; }
+		});
 		initTokenSymbolTables();
+
+		tool.process(this);
     }
 
 	protected void initTokenSymbolTables() {

tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java

@@ -47,8 +47,8 @@ public class GrammarTransformPipeline {
 		GrammarASTAdaptor adaptor = new GrammarASTAdaptor();
 		BlockSetTransformer transformer = new BlockSetTransformer(nodes);
 		transformer.setTreeAdaptor(adaptor);
-		System.out.println("before: "+ast.toStringTree());
+//		System.out.println("before: "+ast.toStringTree());
 		transformer.downup(ast);
-		System.out.println("after: "+ast.toStringTree());
+//		System.out.println("after: "+ast.toStringTree());
 	}
 }

tool/test/org/antlr/v4/test/BaseTest.java

@@ -648,29 +648,12 @@ public abstract class BaseTest {
                 String[] lines = input.split("\n");
 				String fileName = getFilenameFromFirstLineOfGrammar(lines[0]);
                 g = new Grammar(fileName, input, equeue);
-
-				if ( printTree ) {
-					if ( g.ast!=null ) System.out.println(g.ast.toStringTree());
-					else System.out.println("null tree");
-				}
-
-				if ( g.ast!=null && !g.ast.hasErrors ) {
-					Tool antlr = new Tool();
-					SemanticPipeline sem = new SemanticPipeline(g);
-					sem.process();
-					if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
-						for (Grammar imp : g.getImportedGrammars()) {
-							antlr.processNonCombinedGrammar(imp);
-						}
-					}
-				}
-
-				//g.loadImportedGrammars();
             }
             catch (org.antlr.runtime.RecognitionException re) {
                 re.printStackTrace(System.err);
             }
             String actual = equeue.toString(g.tool);
+			System.err.println(actual);
 			String msg = input;
 			msg = msg.replaceAll("\n","\\\\n");
 			msg = msg.replaceAll("\r","\\\\r");
@@ -690,11 +673,12 @@ public abstract class BaseTest {
 		String[] lines = errs.split("\n");
 		for (int i=0; i<lines.length; i++) {
 			String s = lines[i];
-			int lp = s.indexOf('(');
+			int lp = s.indexOf("error(");
 			int rp = s.indexOf(')', lp);
-			if ( lp<0 || rp<0 ) return s;
+			if ( lp>=0 && rp>=0 ) {
 				lines[i] = s.substring(0, lp) + s.substring(rp+1, s.length());
 			}
+		}
 		return Utils.join(lines, "\n");
 	}
 
@@ -846,7 +830,7 @@ public abstract class BaseTest {
         }
     }
 
-	protected void writeFile(String dir, String fileName, String content) {
+	public static void writeFile(String dir, String fileName, String content) {
 		try {
 			File f = new File(dir, fileName);
 			FileWriter w = new FileWriter(f);

tool/test/org/antlr/v4/test/TestCompositeGrammars.java

@@ -32,7 +32,6 @@ package org.antlr.v4.test;
 import org.antlr.v4.Tool;
 import org.antlr.v4.tool.*;
 import org.junit.Test;
-import org.stringtemplate.v4.ST;
 
 public class TestCompositeGrammars extends BaseTest {
 	protected boolean debug = false;
@@ -211,12 +210,7 @@ public class TestCompositeGrammars extends BaseTest {
 			"C : 'c' ;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n" ;
 		writeFile(tmpdir, "M.g", master);
-		Tool antlr = newTool(new String[] {"-lib", tmpdir});
+		Grammar g = new Grammar(tmpdir+"/M.g", master, equeue);
-		antlr.addListener(equeue);
-		GrammarRootAST root = antlr.loadGrammar(tmpdir+"/M.g");
-		Grammar g = antlr.createGrammar(root);
-		g.fileName = "M.g";
-		antlr.process(g);
 
 		String expectedTokenIDToTypeMap = "{EOF=-1, B=3, A=4, C=5, WS=6}";
 		String expectedStringLiteralToTypeMap = "{'c'=5, 'a'=4, 'b'=3}";
@@ -251,12 +245,7 @@ public class TestCompositeGrammars extends BaseTest {
 			"import S;\n" +
 			"s : x INT ;\n";
 		writeFile(tmpdir, "M.g", master);
-		Tool antlr = newTool(new String[] {"-lib", tmpdir});
+		Grammar g = new Grammar(tmpdir+"/M.g", master, equeue);
-		antlr.addListener(equeue);
-		GrammarRootAST root = antlr.loadGrammar(tmpdir+"/M.g");
-		Grammar g = antlr.createGrammar(root);
-		g.fileName = "M.g";
-		antlr.process(g);
 
 		assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
 
@@ -286,12 +275,8 @@ public class TestCompositeGrammars extends BaseTest {
 			"s : x y ;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n" ;
 		writeFile(tmpdir, "M.g", master);
-		Tool antlr = newTool(new String[] {"-lib", tmpdir});
+
-		antlr.addListener(equeue);
+		Grammar g = new Grammar(tmpdir+"/M.g", master, equeue);
-		GrammarRootAST root = antlr.loadGrammar(tmpdir+"/M.g");
-		Grammar g = antlr.createGrammar(root);
-		g.fileName = "M.g";
-		antlr.process(g);
 
 		String expectedTokenIDToTypeMap = "{EOF=-1, T__0=3, WS=4, A=5, X=6}";
 		String expectedStringLiteralToTypeMap = "{'a'=6}";
@@ -312,65 +297,12 @@ public class TestCompositeGrammars extends BaseTest {
 
 		String expectedError =
 			"error(73): T.g:2:9: cannot alias X='a'; string already assigned to A";
-		ST msgST = antlr.errMgr.getMessageTemplate(equeue.errors.get(0));
+//		ST msgST = antlr.errMgr.getMessageTemplate(equeue.errors.get(0));
-		String foundError = msgST.render();
+//		String foundError = msgST.render();
-		assertEquals(expectedError, foundError);
+//		assertEquals(expectedError, foundError);
-	}
-/*
-
-	@Test public void testSameNameTwoStrings() throws Exception {
-		ErrorQueue equeue = new ErrorQueue();
-		ErrorManager.setErrorListener(equeue);
-		String slave =
-			"parser grammar S;\n" +
-			"tokens { A='a'; }\n" +
-			"x : A {System.out.println(\"S.x\");} ;\n";
-		mkdir(tmpdir);
-		writeFile(tmpdir, "S.g", slave);
-		String slave2 =
-			"parser grammar T;\n" +
-			"tokens { A='x'; }\n" +
-			"y : A {System.out.println(\"T.y\");} ;\n";
-
-		writeFile(tmpdir, "T.g", slave2);
-
-		String master =
-			"grammar M;\n" +
-			"import S,T;\n" +
-			"s : x y ;\n" +
-			"WS : (' '|'\\n') {skip();} ;\n" ;
-		writeFile(tmpdir, "M.g", master);
-		Tool antlr = newTool(new String[] {"-lib", tmpdir});
-		CompositeGrammar composite = new CompositeGrammar();
-		Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite);
-		composite.setDelegationRoot(g);
-		g.parseAndBuildAST();
-		g.composite.assignTokenTypes();
-
-		String expectedTokenIDToTypeMap = "[A=4, T__6=6, WS=5]";
-		String expectedStringLiteralToTypeMap = "{'a'=4, 'x'=6}";
-		String expectedTypeToTokenList = "[A, WS, T__6]";
-
-		assertEquals(expectedTokenIDToTypeMap,
-					 realElements(g.composite.tokenIDToTypeMap).toString());
-		assertEquals(expectedStringLiteralToTypeMap, sortMapToString(g.composite.stringLiteralToTypeMap));
-		assertEquals(expectedTypeToTokenList,
-					 realElements(g.composite.typeToTokenList).toString());
-
-		Object expectedArg = "A='x'";
-		Object expectedArg2 = "'a'";
-		int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_REASSIGNMENT;
-		GrammarSemanticsMessage expectedMessage =
-			new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2);
-		checkGrammarSemanticsError(equeue, expectedMessage);
-
-		assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size());
-
-		String expectedError =
-			"error(159): T.g:2:10: cannot alias A='x'; token name already assigned to 'a'";
-		assertEquals(expectedError, equeue.errors.get(0).toString());
 	}
 
+	/*
 	@Test public void testImportedTokenVocabIgnoredWithWarning() throws Exception {
 		ErrorQueue equeue = new ErrorQueue();
 		ErrorManager.setErrorListener(equeue);

tool/test/org/antlr/v4/test/TestSymbolIssues.java

@@ -8,40 +8,32 @@ public class TestSymbolIssues extends BaseTest {
         // INPUT
         "grammar A;\n" +
         "options { opt='sss'; k=3; }\n" +
-        "tokens { X; Y='y'; X='x'; Y='q'; Z; Z; }\n" +
-        "scope Blort { int x; }\n" +
         "\n" +
         "@members {foo}\n" +
         "@members {bar}\n" +
         "@lexer::header {package jj;}\n" +
         "@lexer::header {package kk;}\n" +
         "\n" +
-        "scope Blort { int x; }\n" +
-        "\n" +
         "a[int i] returns [foo f] : X ID a[3] b[34] q ;\n" +
         "b returns [int g] : Y 'y' 'if' a ;\n" +
         "a : FJKD ;\n" +
         "\n" +
         "ID : 'a'..'z'+ ID ;",
         // YIELDS
-        "error(49): A.g:2:10: illegal option opt\n" +
+        "warning(50): A.g:2:10: illegal option opt\n" +
-		"error(59): A.g:11:6: scope Blort redefinition\n" +
+		"error(60): A.g:7:1: redefinition of header action\n" +
-		"error(18): A.g:15:0: rule a redefinition\n" +
+		"warning(50): A.g:2:10: illegal option opt\n" +
-		"error(58): A.g:7:1: redefinition of members action\n" +
+		"error(19): A.g:11:0: rule a redefinition\n" +
-		"error(58): A.g:9:1: redefinition of header action\n" +
+		"error(60): A.g:5:1: redefinition of members action\n" +
-		"error(72): A.g:3:19: cannot alias X; token name already defined\n" +
+		"error(47): A.g:9:37: rule b has no defined parameters\n" +
-		"error(72): A.g:3:26: cannot alias Y; token name already assigned to 'y'\n" +
+		"error(24): A.g:9:43: reference to undefined rule: q\n" +
-		"error(72): A.g:3:36: cannot alias Z; token name already defined\n" +
+		"error(46): A.g:10:31: missing parameter(s) on rule reference: a\n"
-		"error(46): A.g:13:37: rule b has no defined parameters\n" +
-		"error(23): A.g:13:43: reference to undefined rule: q\n" +
-		"error(45): A.g:14:31: missing parameter(s) on rule reference: a\n"
     };
 
     static String[] B = {
         // INPUT
         "parser grammar B;\n" +
         "tokens { X='x'; Y; }\n" +
-        "scope s { int i; }\n" +
         "\n" +
         "a : s=ID b+=ID X=ID '.' ;\n" +
         "\n" +
@@ -49,12 +41,11 @@ public class TestSymbolIssues extends BaseTest {
         "\n" +
         "s : FOO ;",
         // YIELDS
-        "error(25): B.g:2:9: can't assign string value to token name X in non-combined grammar\n" +
+        "error(26): B.g:2:9: can't assign string value to token name X in non-combined grammar\n" +
-		"error(34): B.g:9:0: symbol s conflicts with global dynamic scope with same name\n" +
+		"error(36): B.g:4:4: label s conflicts with rule with same name\n" +
-		"error(35): B.g:5:9: label b conflicts with rule with same name\n" +
+		"error(36): B.g:4:9: label b conflicts with rule with same name\n" +
-		"error(34): B.g:5:4: symbol s conflicts with global dynamic scope with same name\n" +
+		"error(37): B.g:4:15: label X conflicts with token with same name\n" +
-		"error(36): B.g:5:15: label X conflicts with token with same name\n" +
+		"error(42): B.g:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n"
-		"error(41): B.g:7:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n"
     };
 
     static String[] C = {
@@ -77,26 +68,40 @@ public class TestSymbolIssues extends BaseTest {
         // INPUT
         "parser grammar D;\n" +
         "a[int j] \n" +
-        "scope { int i; }\n" +
         "        :       i=ID j=ID ;\n" +
         "\n" +
         "b[int i] returns [int i] : ID ;\n" +
         "\n" +
         "c[int i] returns [String k]\n" +
-        "scope { int i; int c; int k; }\n" +
         "        :       ID ;",
 
         // YIELDS
-        "error(38): D.g:4:21: label j conflicts with rule a's return value or parameter with same name\n" +
+        "error(39): D.g:3:21: label j conflicts with rule a's return value or parameter with same name\n" +
-		"error(37): D.g:4:16: label i conflicts with rule a's dynamically-scoped attribute with same name\n" +
+		"error(43): D.g:5:0: rule b's argument i conflicts a return value with same nameK\n"
-		"error(42): D.g:6:0: rule b's argument i conflicts a return value with same name\n" +
+    };
-		"error(40): D.g:9:6: rule c's dynamically-scoped attribute i conflicts with c's return value or parameter with same name\n" +
+
-		"error(39): D.g:9:6: rule c's dynamically-scoped attribute c conflicts with the rule name\n" +
+	static String[] E = {
-		"error(40): D.g:9:6: rule c's dynamically-scoped attribute k conflicts with c's return value or parameter with same name\n"
+		// INPUT
+		"grammar E;\n" +
+		"tokens {\n" +
+		"	A; A;\n" +
+		"	B='b'; B;\n" +
+		"	C; C='c';\n" +
+		"	D='d'; D='d';\n" +
+		"	E='e'; X='e';\n" +
+		"}\n" +
+		"a : A ;\n",
+
+		// YIELDS
+		"error(74): E.g:4:8: cannot redefine B; token name already defined\n" +
+		"error(74): E.g:5:4: cannot redefine C; token name already defined\n" +
+		"error(74): E.g:6:8: cannot redefine D; token name already defined\n" +
+		"error(73): E.g:7:8: cannot alias X='e'; string already assigned to E\n"
 	};
 
     @Test public void testA() { super.testErrors(A, false); }
     @Test public void testB() { super.testErrors(B, false); }
     @Test public void testC() { super.testErrors(C, false); }
 	@Test public void testD() { super.testErrors(D, false); }
+	@Test public void testE() { super.testErrors(E, false); }
 }