From 6dab71e160dffa56267b7280961c62f654fa54a0 Mon Sep 17 00:00:00 2001 From: parrt Date: Tue, 19 Jul 2011 16:44:35 -0800 Subject: [PATCH] symbol issue test working again [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8894] --- tool/src/org/antlr/v4/Tool.java | 22 ++--- .../antlr/v4/automata/ParserATNFactory.java | 4 +- .../v4/semantics/BasicSemanticChecks.java | 50 +++++------ .../antlr/v4/semantics/SemanticPipeline.java | 4 +- .../org/antlr/v4/semantics/SymbolChecks.java | 57 ++++++++++--- .../antlr/v4/semantics/SymbolCollector.java | 2 +- tool/src/org/antlr/v4/tool/AltAST.java | 9 +- tool/src/org/antlr/v4/tool/ErrorManager.java | 32 +++---- tool/src/org/antlr/v4/tool/ErrorType.java | 2 +- tool/src/org/antlr/v4/tool/Grammar.java | 38 +++++---- .../v4/tool/GrammarTransformPipeline.java | 4 +- tool/test/org/antlr/v4/test/BaseTest.java | 28 ++----- .../antlr/v4/test/TestCompositeGrammars.java | 84 ++----------------- .../org/antlr/v4/test/TestSymbolIssues.java | 67 ++++++++------- 14 files changed, 183 insertions(+), 220 deletions(-) diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 5627c62cd..7d3b00eb2 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -375,6 +375,9 @@ public class Tool { grammars. */ public void mergeImportedGrammars(Grammar rootGrammar) { + List imports = rootGrammar.getAllImportedGrammars(); + if ( imports==null ) return; + GrammarAST root = rootGrammar.ast; GrammarAST id = (GrammarAST) root.getChild(0); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream()); @@ -397,9 +400,6 @@ public class Tool { for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText()); } - List imports = rootGrammar.getAllImportedGrammars(); - if ( imports==null ) return; - for (Grammar imp : imports) { GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS); if ( imp_tokensRoot!=null ) { @@ -498,8 +498,8 @@ public class Tool { * tokenVocab or tokens{} section. * * Side-effects: it removes children from GRAMMAR & RULES nodes - * in combined AST. Careful: nodes are shared between - * trees after this call. + * in combined AST. Anything cut out is dup'd before + * adding to lexer to avoid "who's ur daddy" issues */ public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) { GrammarRootAST combinedAST = combinedGrammar.ast; @@ -525,7 +525,7 @@ public class Tool { for (GrammarAST o : options) { String optionName = o.getChild(0).getText(); if ( !Grammar.doNotCopyOptionsToLexer.contains(optionName) ) { - lexerOptionsRoot.addChild(o); + lexerOptionsRoot.addChild((Tree)adaptor.dupTree(o)); } } } @@ -535,7 +535,7 @@ public class Tool { for (GrammarAST e : elements) { if ( e.getType()==ANTLRParser.AT ) { if ( e.getChild(0).getText().equals("lexer") ) { - lexerAST.addChild(e); + lexerAST.addChild((Tree)adaptor.dupTree(e)); actionsWeMoved.add(e); } } @@ -555,7 +555,7 @@ public class Tool { for (GrammarASTWithOptions r : rules) { String ruleName = r.getChild(0).getText(); if ( Character.isUpperCase(ruleName.charAt(0)) ) { - lexerRulesRoot.addChild(r); + lexerRulesRoot.addChild((Tree)adaptor.dupTree(r)); rulesWeMoved.add(r); } } @@ -593,10 +593,10 @@ public class Tool { lexerRulesRoot.getChildren().add(0, litRule); // add first } - lexerRulesRoot.freshenParentAndChildIndexesDeeply(); - combinedRulesRoot.freshenParentAndChildIndexesDeeply(); + lexerAST.freshenParentAndChildIndexesDeeply(); + combinedAST.freshenParentAndChildIndexesDeeply(); - System.out.println("after ="+combinedAST.toStringTree()); + System.out.println("after extract implicit lexer ="+combinedAST.toStringTree()); System.out.println("lexer ="+lexerAST.toStringTree()); return lexerAST; } diff --git a/tool/src/org/antlr/v4/automata/ParserATNFactory.java b/tool/src/org/antlr/v4/automata/ParserATNFactory.java index 602a75e4f..1537360ee 100644 --- a/tool/src/org/antlr/v4/automata/ParserATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ParserATNFactory.java @@ -137,7 +137,9 @@ public class ParserATNFactory implements ATNFactory { } /** Not valid for non-lexers */ - public Handle range(GrammarAST a, GrammarAST b) { throw new UnsupportedOperationException(); } + public Handle range(GrammarAST a, GrammarAST b) { + throw new UnsupportedOperationException(); + } /** ~atom only */ /* diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index d2db16f32..ca5740221 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -366,10 +366,10 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { if ( optionID.getText().equals("tokenVocab") && g.parent!=null ) // only allow tokenVocab option in root grammar { - g.tool.errMgr.grammarWarning(ErrorType.TOKEN_VOCAB_IN_DELEGATE, - g.fileName, - optionID, - g.name); + g.tool.errMgr.grammarError(ErrorType.TOKEN_VOCAB_IN_DELEGATE, + g.fileName, + optionID, + g.name); ok = false; } @@ -455,10 +455,10 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { options.get("rewrite").equals("true") ) { String fileName = altStart.getInputStream().getSourceName(); - g.tool.errMgr.grammarWarning(ErrorType.REWRITE_FOR_MULTI_ELEMENT_ALT, - fileName, - altStart, - alt); + g.tool.errMgr.grammarError(ErrorType.REWRITE_FOR_MULTI_ELEMENT_ALT, + fileName, + altStart, + alt); } } @@ -470,25 +470,25 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { String ruleName = rule.getChild(0).getText(); String fileName = elementRoot.token.getInputStream().getSourceName(); if ( options==null || !options.get("output").equals("AST") ) { - g.tool.errMgr.grammarWarning(ErrorType.AST_OP_WITH_NON_AST_OUTPUT_OPTION, - fileName, - elementRoot.token, - op.getText()); + g.tool.errMgr.grammarError(ErrorType.AST_OP_WITH_NON_AST_OUTPUT_OPTION, + fileName, + elementRoot.token, + op.getText()); } if ( options!=null && options.get("output")==null ) { - g.tool.errMgr.grammarWarning(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION, - fileName, - elementRoot.token, - ruleName); + g.tool.errMgr.grammarError(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION, + fileName, + elementRoot.token, + ruleName); } if ( op.hasAncestor(ANTLRParser.ALT_REWRITE) ) { GrammarAST rew = (GrammarAST)op.getAncestor(ANTLRParser.ALT_REWRITE); int altNum = rew.getChildIndex() + 1; // alts are 1..n - g.tool.errMgr.grammarWarning(ErrorType.AST_OP_IN_ALT_WITH_REWRITE, - fileName, - elementRoot.token, - ruleName, - altNum); + g.tool.errMgr.grammarError(ErrorType.AST_OP_IN_ALT_WITH_REWRITE, + fileName, + elementRoot.token, + ruleName, + altNum); } } @@ -496,10 +496,10 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { String ruleName = currentRuleAST.getChild(0).getText(); String fileName = elementRoot.token.getInputStream().getSourceName(); if ( options!=null && options.get("output")==null ) { - g.tool.errMgr.grammarWarning(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION, - fileName, - elementRoot.token, - ruleName); + g.tool.errMgr.grammarError(ErrorType.REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION, + fileName, + elementRoot.token, + ruleName); } } diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java index 2ae108dd1..91b023353 100644 --- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java +++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java @@ -91,7 +91,7 @@ public class SemanticPipeline { g.defineRule(r); } for (GrammarAST a : collector.namedActions) { - g.defineAction((GrammarAST)a.getParent()); + g.defineAction(a); } // LINK (outermost) ALT NODES WITH Alternatives @@ -122,7 +122,7 @@ public class SemanticPipeline { // CHECK ATTRIBUTE EXPRESSIONS FOR SEMANTIC VALIDITY AttributeChecks.checkAllAttributeExpressions(g); - symcheck.checkForRewriteIssues(); + symcheck.checkForUndefinedTokensInRewrite(); UseDefAnalyzer.checkRewriteElementsPresentOnLeftSide(g); UseDefAnalyzer.trackTokenRuleRefsInActions(g); diff --git a/tool/src/org/antlr/v4/semantics/SymbolChecks.java b/tool/src/org/antlr/v4/semantics/SymbolChecks.java index 27345c321..cd9210adf 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolChecks.java +++ b/tool/src/org/antlr/v4/semantics/SymbolChecks.java @@ -30,7 +30,6 @@ package org.antlr.v4.semantics; import org.antlr.runtime.Token; -import org.antlr.runtime.misc.DoubleKeyMap; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.tool.*; @@ -48,9 +47,9 @@ public class SymbolChecks { Map nameToRuleMap = new HashMap(); Set tokenIDs = new HashSet(); Set globalScopeNames = new HashSet(); -// Map> actionScopeToActionNames = new HashMap>(); - DoubleKeyMap namedActions = - new DoubleKeyMap(); + Map> actionScopeToActionNames = new HashMap>(); +// DoubleKeyMap namedActions = +// new DoubleKeyMap(); public ErrorManager errMgr; @@ -77,6 +76,7 @@ public class SymbolChecks { //checkForImportedRuleIssues(collector.qualifiedRulerefs); // done in sem pipe for now checkForRuleConflicts(collector.rules); // sets nameToRuleMap + checkActionRedefinitions(collector.namedActions); checkTokenAliasRedefinitions(collector.tokensDefs); //checkRuleArgs(collector.rulerefs); checkForTokenConflicts(collector.tokenIDRefs); // sets tokenIDs @@ -104,6 +104,35 @@ public class SymbolChecks { } } + public void checkActionRedefinitions(List actions) { + if ( actions==null ) return; + String scope = g.getDefaultActionScope(); + String name = null; + GrammarAST nameNode = null; + for (GrammarAST ampersandAST : actions) { + nameNode = (GrammarAST)ampersandAST.getChild(0); + if ( ampersandAST.getChildCount()==2 ) { + name = nameNode.getText(); + } + else { + scope = nameNode.getText(); + name = ampersandAST.getChild(1).getText(); + } + Set scopeActions = actionScopeToActionNames.get(scope); + if ( scopeActions==null ) { // init scope + scopeActions = new HashSet(); + actionScopeToActionNames.put(scope, scopeActions); + } + if ( !scopeActions.contains(name) ) { + scopeActions.add(name); + } + else { + errMgr.grammarError(ErrorType.ACTION_REDEFINITION, + g.fileName, nameNode.token, name); + } + } + } + public void checkScopeRedefinitions(List dicts) { if ( dicts ==null ) return; for (int i=0; i< dicts.size(); i++) { @@ -122,7 +151,7 @@ public class SymbolChecks { /** Catch: tokens { A='a'; A; } can't redefine token type if has alias - tokens { A; A='a'; } + tokens { A; A='a'; } can't redefine token type if has alias tokens { A='a'; A='b'; } can't have two aliases for single token type tokens { A='a'; B='a'; } can't have to token types for same string alias */ @@ -135,6 +164,7 @@ public class SymbolChecks { for (int i=0; i0 ) idNode = (GrammarAST)a.getChild(0); GrammarAST prevToken = aliasTokenNames.get(idNode.getText()); GrammarAST stringNode = null; if ( a.getChildCount()>0 ) stringNode = (GrammarAST)a.getChild(1); @@ -160,9 +190,12 @@ public class SymbolChecks { } } if ( prevString!=null ) { - errMgr.grammarError(ErrorType.TOKEN_STRING_REASSIGNMENT, - a.g.fileName, idNode.token, idNode.getText()+"="+stringNode.getText(), - prevString.getChild(0).getText()); + // A='a' and A='a' are ok but not B='a' and A='a' are ok + if ( !prevString.getChild(0).getText().equals(idNode.getText()) ) { + errMgr.grammarError(ErrorType.TOKEN_STRING_REASSIGNMENT, + a.g.fileName, idNode.token, idNode.getText()+"="+stringNode.getText(), + prevString.getChild(0).getText()); + } } } } @@ -179,14 +212,14 @@ public class SymbolChecks { } } - public void checkForRewriteIssues() { - // Ensure that all tokens refer to on the right if -> have been defined. + public void checkForUndefinedTokensInRewrite() { + // Ensure that all tokens refs on the right of -> have been defined. for (GrammarAST elem : collector.rewriteElements) { if ( elem.getType()==ANTLRParser.TOKEN_REF ) { int ttype = g.getTokenType(elem.getText()); if ( ttype == Token.INVALID_TOKEN_TYPE ) { - g.tool.errMgr.grammarError(ErrorType.UNDEFINED_TOKEN_REF_IN_REWRITE, - g.fileName, elem.token, elem.getText()); + g.tool.errMgr.grammarError(ErrorType.UNDEFINED_TOKEN_REF_IN_REWRITE, + g.fileName, elem.token, elem.getText()); } } } diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index f4aa0fe4b..d862a5c57 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -82,7 +82,7 @@ public class SymbolCollector extends GrammarTreeVisitor { @Override public void globalNamedAction(GrammarAST scope, GrammarAST ID, ActionAST action) { - namedActions.add(ID); + namedActions.add((GrammarAST)ID.getParent()); action.resolver = g; } diff --git a/tool/src/org/antlr/v4/tool/AltAST.java b/tool/src/org/antlr/v4/tool/AltAST.java index 41b3c1a8c..05cc2bd67 100644 --- a/tool/src/org/antlr/v4/tool/AltAST.java +++ b/tool/src/org/antlr/v4/tool/AltAST.java @@ -33,7 +33,7 @@ import org.antlr.runtime.Token; import org.antlr.runtime.tree.Tree; import org.antlr.v4.parse.ANTLRParser; -/** An ALT or ALT_REWRITE node (left of ->) */ +/** An ALT (which can be child of ALT_REWRITE node) */ public class AltAST extends GrammarAST { public Alternative alt; @@ -47,8 +47,11 @@ public class AltAST extends GrammarAST { public AltAST(int type, Token t) { super(type, t); } public GrammarAST getRewrite() { - if ( getType() == ANTLRParser.ALT ) return null; - return (GrammarAST)getChild(1); // ^(ALT_REWRITE ^(ALT ...) ^(-> ...)) + // ^(ALT_REWRITE ^(ALT ...) ^(-> ...)) ?? + if ( getParent().getType() == ANTLRParser.ALT_REWRITE ) { + return (GrammarAST)getParent().getChild(1); + } + return null; } @Override diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java index f43973747..158ab2292 100644 --- a/tool/src/org/antlr/v4/tool/ErrorManager.java +++ b/tool/src/org/antlr/v4/tool/ErrorManager.java @@ -29,7 +29,6 @@ package org.antlr.v4.tool; -import org.antlr.runtime.Token; import org.antlr.v4.Tool; import org.stringtemplate.v4.*; import org.stringtemplate.v4.misc.*; @@ -98,6 +97,7 @@ public class ErrorManager { if ( i>0 ) attr += i + 1; messageST.add(attr, msg.args[i]); } + if ( msg.args.length<2 ) messageST.add("arg2", null); // some messages ref arg2 } if ( msg.e!=null ) { messageST.add("exception", msg.e); @@ -156,7 +156,10 @@ public class ErrorManager { org.antlr.runtime.RecognitionException antlrException, Object... args) { - errors++; + switch ( etype.severity ) { + case WARNING: warnings++; break; + case ERROR: errors++; break; + } ANTLRMessage msg = new GrammarSyntaxMessage(etype,fileName,token,antlrException,args); tool.error(msg); } @@ -185,12 +188,18 @@ public class ErrorManager { * @param args The arguments to pass to the StringTemplate */ public void toolError(ErrorType errorType, Object... args) { - errors++; + switch ( errorType.severity ) { + case WARNING: warnings++; break; + case ERROR: errors++; break; + } tool.error(new ToolMessage(errorType, args)); } public void toolError(ErrorType errorType, Throwable e, Object... args) { - errors++; + switch ( errorType.severity ) { + case WARNING: warnings++; break; + case ERROR: errors++; break; + } tool.error(new ToolMessage(errorType, e, args)); } @@ -199,21 +208,14 @@ public class ErrorManager { org.antlr.runtime.Token token, Object... args) { - errors++; + switch ( etype.severity ) { + case WARNING: warnings++; break; + case ERROR: errors++; break; + } ANTLRMessage msg = new GrammarSemanticsMessage(etype,fileName,token,args); tool.error(msg); } - public void grammarWarning(ErrorType etype, - String fileName, - Token token, - Object... args) - { - warnings++; - ANTLRMessage msg = new GrammarSemanticsMessage(etype,fileName,token,args); - tool.warning(msg); - } - public void leftRecursionCycles(String fileName, Collection cycles) { errors++; ANTLRMessage msg = new LeftRecursionCyclesMessage(fileName, cycles); diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index e32ebd4f5..fd69b3ac1 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -99,7 +99,7 @@ public enum ErrorType { RULE_HAS_NO_ARGS("rule has no defined parameters", ErrorSeverity.ERROR), ARGS_ON_TOKEN_REF("token reference may not have parameters", ErrorSeverity.ERROR), RULE_REF_AMBIG_WITH_RULE_IN_ALT("", ErrorSeverity.ERROR), - ILLEGAL_OPTION("illegal option ", ErrorSeverity.ERROR), + ILLEGAL_OPTION("illegal option ", ErrorSeverity.WARNING), LIST_LABEL_INVALID_UNLESS_RETVAL_STRUCT("", ErrorSeverity.ERROR), REWRITE_ELEMENT_NOT_PRESENT_ON_LHS("reference to rewrite element not found to left of ->", ErrorSeverity.ERROR), UNDEFINED_TOKEN_REF_IN_REWRITE("token in rewrite is undefined", ErrorSeverity.ERROR), diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index 3bad3b93e..19028d01c 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -29,14 +29,11 @@ package org.antlr.v4.tool; -import org.antlr.runtime.CommonTokenStream; -import org.antlr.runtime.*; -import org.antlr.runtime.tree.TreeWizard; +import org.antlr.runtime.tree.*; import org.antlr.v4.Tool; import org.antlr.v4.misc.*; import org.antlr.v4.parse.*; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.*; @@ -172,7 +169,14 @@ public class Grammar implements AttributeResolver { this(GRAMMAR_FROM_STRING_NAME, grammarText, listener); } - /** For testing; only builds trees; no sem anal */ + /** For testing; builds trees, does sem anal */ + public Grammar(String fileName, String grammarText) + throws org.antlr.runtime.RecognitionException + { + this(fileName, grammarText, null); + } + + /** For testing; builds trees, does sem anal */ public Grammar(String fileName, String grammarText, ANTLRToolListener listener) throws org.antlr.runtime.RecognitionException { @@ -182,20 +186,18 @@ public class Grammar implements AttributeResolver { this.tool.addListener(listener); org.antlr.runtime.ANTLRStringStream in = new org.antlr.runtime.ANTLRStringStream(grammarText); in.name = fileName; - ANTLRLexer lexer = new ANTLRLexer(in); - CommonTokenStream tokens = new CommonTokenStream(lexer); - ToolANTLRParser p = new ToolANTLRParser(tokens,tool); - p.setTreeAdaptor(new GrammarASTAdaptor(in)); - ParserRuleReturnScope r = p.grammarSpec(); - if ( r.getTree() instanceof GrammarRootAST ) { - this.ast = (GrammarRootAST)r.getTree(); - this.ast.hasErrors = p.getNumberOfSyntaxErrors()>0; - this.name = ((GrammarAST)ast.getChild(0)).getText(); - GrammarTransformPipeline transform = new GrammarTransformPipeline(); - transform.process(ast); - } + this.ast = tool.load(in); + // ensure each node has pointer to surrounding grammar + final Grammar thiz = this; + TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor()); + v.visit(ast, new TreeVisitorAction() { + public Object pre(Object t) { ((GrammarAST)t).g = thiz; return t; } + public Object post(Object t) { return t; } + }); initTokenSymbolTables(); + + tool.process(this); } protected void initTokenSymbolTables() { diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index 839ed7292..e8708dac5 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -47,8 +47,8 @@ public class GrammarTransformPipeline { GrammarASTAdaptor adaptor = new GrammarASTAdaptor(); BlockSetTransformer transformer = new BlockSetTransformer(nodes); transformer.setTreeAdaptor(adaptor); - System.out.println("before: "+ast.toStringTree()); +// System.out.println("before: "+ast.toStringTree()); transformer.downup(ast); - System.out.println("after: "+ast.toStringTree()); +// System.out.println("after: "+ast.toStringTree()); } } diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index dd68fbe4b..b74154e00 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -648,29 +648,12 @@ public abstract class BaseTest { String[] lines = input.split("\n"); String fileName = getFilenameFromFirstLineOfGrammar(lines[0]); g = new Grammar(fileName, input, equeue); - - if ( printTree ) { - if ( g.ast!=null ) System.out.println(g.ast.toStringTree()); - else System.out.println("null tree"); - } - - if ( g.ast!=null && !g.ast.hasErrors ) { - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) - for (Grammar imp : g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp); - } - } - } - - //g.loadImportedGrammars(); } catch (org.antlr.runtime.RecognitionException re) { re.printStackTrace(System.err); } String actual = equeue.toString(g.tool); + System.err.println(actual); String msg = input; msg = msg.replaceAll("\n","\\\\n"); msg = msg.replaceAll("\r","\\\\r"); @@ -690,10 +673,11 @@ public abstract class BaseTest { String[] lines = errs.split("\n"); for (int i=0; i=0 && rp>=0 ) { + lines[i] = s.substring(0, lp) + s.substring(rp+1, s.length()); + } } return Utils.join(lines, "\n"); } @@ -846,7 +830,7 @@ public abstract class BaseTest { } } - protected void writeFile(String dir, String fileName, String content) { + public static void writeFile(String dir, String fileName, String content) { try { File f = new File(dir, fileName); FileWriter w = new FileWriter(f); diff --git a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java index 3e10ce542..440cb5585 100644 --- a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java +++ b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java @@ -32,7 +32,6 @@ package org.antlr.v4.test; import org.antlr.v4.Tool; import org.antlr.v4.tool.*; import org.junit.Test; -import org.stringtemplate.v4.ST; public class TestCompositeGrammars extends BaseTest { protected boolean debug = false; @@ -211,12 +210,7 @@ public class TestCompositeGrammars extends BaseTest { "C : 'c' ;\n" + "WS : (' '|'\\n') {skip();} ;\n" ; writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - antlr.addListener(equeue); - GrammarRootAST root = antlr.loadGrammar(tmpdir+"/M.g"); - Grammar g = antlr.createGrammar(root); - g.fileName = "M.g"; - antlr.process(g); + Grammar g = new Grammar(tmpdir+"/M.g", master, equeue); String expectedTokenIDToTypeMap = "{EOF=-1, B=3, A=4, C=5, WS=6}"; String expectedStringLiteralToTypeMap = "{'c'=5, 'a'=4, 'b'=3}"; @@ -251,12 +245,7 @@ public class TestCompositeGrammars extends BaseTest { "import S;\n" + "s : x INT ;\n"; writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - antlr.addListener(equeue); - GrammarRootAST root = antlr.loadGrammar(tmpdir+"/M.g"); - Grammar g = antlr.createGrammar(root); - g.fileName = "M.g"; - antlr.process(g); + Grammar g = new Grammar(tmpdir+"/M.g", master, equeue); assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); @@ -286,12 +275,8 @@ public class TestCompositeGrammars extends BaseTest { "s : x y ;\n" + "WS : (' '|'\\n') {skip();} ;\n" ; writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - antlr.addListener(equeue); - GrammarRootAST root = antlr.loadGrammar(tmpdir+"/M.g"); - Grammar g = antlr.createGrammar(root); - g.fileName = "M.g"; - antlr.process(g); + + Grammar g = new Grammar(tmpdir+"/M.g", master, equeue); String expectedTokenIDToTypeMap = "{EOF=-1, T__0=3, WS=4, A=5, X=6}"; String expectedStringLiteralToTypeMap = "{'a'=6}"; @@ -312,65 +297,12 @@ public class TestCompositeGrammars extends BaseTest { String expectedError = "error(73): T.g:2:9: cannot alias X='a'; string already assigned to A"; - ST msgST = antlr.errMgr.getMessageTemplate(equeue.errors.get(0)); - String foundError = msgST.render(); - assertEquals(expectedError, foundError); - } -/* - - @Test public void testSameNameTwoStrings() throws Exception { - ErrorQueue equeue = new ErrorQueue(); - ErrorManager.setErrorListener(equeue); - String slave = - "parser grammar S;\n" + - "tokens { A='a'; }\n" + - "x : A {System.out.println(\"S.x\");} ;\n"; - mkdir(tmpdir); - writeFile(tmpdir, "S.g", slave); - String slave2 = - "parser grammar T;\n" + - "tokens { A='x'; }\n" + - "y : A {System.out.println(\"T.y\");} ;\n"; - - writeFile(tmpdir, "T.g", slave2); - - String master = - "grammar M;\n" + - "import S,T;\n" + - "s : x y ;\n" + - "WS : (' '|'\\n') {skip();} ;\n" ; - writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - CompositeGrammar composite = new CompositeGrammar(); - Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); - composite.setDelegationRoot(g); - g.parseAndBuildAST(); - g.composite.assignTokenTypes(); - - String expectedTokenIDToTypeMap = "[A=4, T__6=6, WS=5]"; - String expectedStringLiteralToTypeMap = "{'a'=4, 'x'=6}"; - String expectedTypeToTokenList = "[A, WS, T__6]"; - - assertEquals(expectedTokenIDToTypeMap, - realElements(g.composite.tokenIDToTypeMap).toString()); - assertEquals(expectedStringLiteralToTypeMap, sortMapToString(g.composite.stringLiteralToTypeMap)); - assertEquals(expectedTypeToTokenList, - realElements(g.composite.typeToTokenList).toString()); - - Object expectedArg = "A='x'"; - Object expectedArg2 = "'a'"; - int expectedMsgID = ErrorManager.MSG_TOKEN_ALIAS_REASSIGNMENT; - GrammarSemanticsMessage expectedMessage = - new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); - checkGrammarSemanticsError(equeue, expectedMessage); - - assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); - - String expectedError = - "error(159): T.g:2:10: cannot alias A='x'; token name already assigned to 'a'"; - assertEquals(expectedError, equeue.errors.get(0).toString()); +// ST msgST = antlr.errMgr.getMessageTemplate(equeue.errors.get(0)); +// String foundError = msgST.render(); +// assertEquals(expectedError, foundError); } + /* @Test public void testImportedTokenVocabIgnoredWithWarning() throws Exception { ErrorQueue equeue = new ErrorQueue(); ErrorManager.setErrorListener(equeue); diff --git a/tool/test/org/antlr/v4/test/TestSymbolIssues.java b/tool/test/org/antlr/v4/test/TestSymbolIssues.java index 1b86366f9..bdce6725b 100644 --- a/tool/test/org/antlr/v4/test/TestSymbolIssues.java +++ b/tool/test/org/antlr/v4/test/TestSymbolIssues.java @@ -8,40 +8,32 @@ public class TestSymbolIssues extends BaseTest { // INPUT "grammar A;\n" + "options { opt='sss'; k=3; }\n" + - "tokens { X; Y='y'; X='x'; Y='q'; Z; Z; }\n" + - "scope Blort { int x; }\n" + "\n" + "@members {foo}\n" + "@members {bar}\n" + "@lexer::header {package jj;}\n" + "@lexer::header {package kk;}\n" + "\n" + - "scope Blort { int x; }\n" + - "\n" + "a[int i] returns [foo f] : X ID a[3] b[34] q ;\n" + "b returns [int g] : Y 'y' 'if' a ;\n" + "a : FJKD ;\n" + "\n" + "ID : 'a'..'z'+ ID ;", // YIELDS - "error(49): A.g:2:10: illegal option opt\n" + - "error(59): A.g:11:6: scope Blort redefinition\n" + - "error(18): A.g:15:0: rule a redefinition\n" + - "error(58): A.g:7:1: redefinition of members action\n" + - "error(58): A.g:9:1: redefinition of header action\n" + - "error(72): A.g:3:19: cannot alias X; token name already defined\n" + - "error(72): A.g:3:26: cannot alias Y; token name already assigned to 'y'\n" + - "error(72): A.g:3:36: cannot alias Z; token name already defined\n" + - "error(46): A.g:13:37: rule b has no defined parameters\n" + - "error(23): A.g:13:43: reference to undefined rule: q\n" + - "error(45): A.g:14:31: missing parameter(s) on rule reference: a\n" + "warning(50): A.g:2:10: illegal option opt\n" + + "error(60): A.g:7:1: redefinition of header action\n" + + "warning(50): A.g:2:10: illegal option opt\n" + + "error(19): A.g:11:0: rule a redefinition\n" + + "error(60): A.g:5:1: redefinition of members action\n" + + "error(47): A.g:9:37: rule b has no defined parameters\n" + + "error(24): A.g:9:43: reference to undefined rule: q\n" + + "error(46): A.g:10:31: missing parameter(s) on rule reference: a\n" }; static String[] B = { // INPUT "parser grammar B;\n" + "tokens { X='x'; Y; }\n" + - "scope s { int i; }\n" + "\n" + "a : s=ID b+=ID X=ID '.' ;\n" + "\n" + @@ -49,12 +41,11 @@ public class TestSymbolIssues extends BaseTest { "\n" + "s : FOO ;", // YIELDS - "error(25): B.g:2:9: can't assign string value to token name X in non-combined grammar\n" + - "error(34): B.g:9:0: symbol s conflicts with global dynamic scope with same name\n" + - "error(35): B.g:5:9: label b conflicts with rule with same name\n" + - "error(34): B.g:5:4: symbol s conflicts with global dynamic scope with same name\n" + - "error(36): B.g:5:15: label X conflicts with token with same name\n" + - "error(41): B.g:7:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n" + "error(26): B.g:2:9: can't assign string value to token name X in non-combined grammar\n" + + "error(36): B.g:4:4: label s conflicts with rule with same name\n" + + "error(36): B.g:4:9: label b conflicts with rule with same name\n" + + "error(37): B.g:4:15: label X conflicts with token with same name\n" + + "error(42): B.g:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n" }; static String[] C = { @@ -77,26 +68,40 @@ public class TestSymbolIssues extends BaseTest { // INPUT "parser grammar D;\n" + "a[int j] \n" + - "scope { int i; }\n" + " : i=ID j=ID ;\n" + "\n" + "b[int i] returns [int i] : ID ;\n" + "\n" + "c[int i] returns [String k]\n" + - "scope { int i; int c; int k; }\n" + " : ID ;", // YIELDS - "error(38): D.g:4:21: label j conflicts with rule a's return value or parameter with same name\n" + - "error(37): D.g:4:16: label i conflicts with rule a's dynamically-scoped attribute with same name\n" + - "error(42): D.g:6:0: rule b's argument i conflicts a return value with same name\n" + - "error(40): D.g:9:6: rule c's dynamically-scoped attribute i conflicts with c's return value or parameter with same name\n" + - "error(39): D.g:9:6: rule c's dynamically-scoped attribute c conflicts with the rule name\n" + - "error(40): D.g:9:6: rule c's dynamically-scoped attribute k conflicts with c's return value or parameter with same name\n" + "error(39): D.g:3:21: label j conflicts with rule a's return value or parameter with same name\n" + + "error(43): D.g:5:0: rule b's argument i conflicts a return value with same nameK\n" }; + static String[] E = { + // INPUT + "grammar E;\n" + + "tokens {\n" + + " A; A;\n" + + " B='b'; B;\n" + + " C; C='c';\n" + + " D='d'; D='d';\n" + + " E='e'; X='e';\n" + + "}\n" + + "a : A ;\n", + + // YIELDS + "error(74): E.g:4:8: cannot redefine B; token name already defined\n" + + "error(74): E.g:5:4: cannot redefine C; token name already defined\n" + + "error(74): E.g:6:8: cannot redefine D; token name already defined\n" + + "error(73): E.g:7:8: cannot alias X='e'; string already assigned to E\n" + }; + @Test public void testA() { super.testErrors(A, false); } @Test public void testB() { super.testErrors(B, false); } @Test public void testC() { super.testErrors(C, false); } - @Test public void testD() { super.testErrors(D, false); } + @Test public void testD() { super.testErrors(D, false); } + @Test public void testE() { super.testErrors(E, false); } }