From 36f3590f6c395537cc76276cbafeb25c15647518 Mon Sep 17 00:00:00 2001 From: parrt Date: Tue, 19 Jul 2011 19:18:16 -0800 Subject: [PATCH] got rule props in for current rule; fixed yet another freakin bug in implicit lexer extraction. [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8895] --- .../v4/tool/templates/codegen/Java/Java.stg | 6 + tool/src/org/antlr/v4/Tool.java | 27 ++- .../antlr/v4/codegen/ActionTranslator.java | 38 +++- .../model/actions/ThisRulePropertyRef_st.java | 37 ++++ .../actions/ThisRulePropertyRef_start.java | 37 ++++ .../actions/ThisRulePropertyRef_stop.java | 37 ++++ .../actions/ThisRulePropertyRef_text.java | 37 ++++ .../actions/ThisRulePropertyRef_tree.java | 37 ++++ .../org/antlr/v4/parse/GrammarTreeVisitor.g | 11 +- .../v4/semantics/BasicSemanticChecks.java | 12 +- .../antlr/v4/semantics/SymbolCollector.java | 15 +- tool/src/org/antlr/v4/tool/ErrorManager.java | 37 ++-- tool/src/org/antlr/v4/tool/ErrorType.java | 5 +- tool/src/org/antlr/v4/tool/Grammar.java | 1 - tool/src/org/antlr/v4/tool/GrammarAST.java | 29 ++- tool/test/org/antlr/v4/test/BaseTest.java | 26 ++- .../antlr/v4/test/TestActionTranslation.java | 14 +- .../antlr/v4/test/TestCompositeGrammars.java | 166 +++--------------- 18 files changed, 363 insertions(+), 209 deletions(-) create mode 100644 tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_st.java create mode 100644 tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_start.java create mode 100644 tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_stop.java create mode 100644 tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_text.java create mode 100644 tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_tree.java diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index ecb064284..f76dea73c 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -341,6 +341,12 @@ RulePropertyRef_tree(r) ::= "(_localctx.!=null?(()_ RulePropertyRef_text(r) ::= "(_localctx.!=null?((TokenStream)input).toString(_localctx..start,_localctx..stop):null)" RulePropertyRef_st(r) ::= "(_localctx.!=null?_localctx..st:null)" +ThisRulePropertyRef_start(r) ::= "_localctx.start" +ThisRulePropertyRef_stop(r) ::= "_localctx.stop" +ThisRulePropertyRef_tree(r) ::= "_localctx.tree" +ThisRulePropertyRef_text(r) ::= "((TokenStream)input).toString(_localctx.start, input.LT(-1))" +ThisRulePropertyRef_st(r) ::= "_localctx.st" + DynScopeRef(s) ::= "" DynScopeAttrRef(s) ::= ".peek()." DynScopeAttrRef_negIndex(s, indexChunks) ::= diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 7d3b00eb2..228ab2900 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -225,7 +225,7 @@ public class Tool { public void process(Grammar g) { g.loadImportedGrammars(); - mergeImportedGrammars(g); + integrateImportedGrammars(g); GrammarTransformPipeline transform = new GrammarTransformPipeline(); transform.process(g.ast); @@ -374,7 +374,7 @@ public class Tool { The goal is a complete combined grammar so we can ignore subordinate grammars. */ - public void mergeImportedGrammars(Grammar rootGrammar) { + public void integrateImportedGrammars(Grammar rootGrammar) { List imports = rootGrammar.getAllImportedGrammars(); if ( imports==null ) return; @@ -401,6 +401,7 @@ public class Tool { } for (Grammar imp : imports) { + // COPY TOKENS GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS); if ( imp_tokensRoot!=null ) { System.out.println("imported tokens: "+imp_tokensRoot.getChildren()); @@ -417,6 +418,7 @@ public class Tool { if ( actionRoots!=null ) all_actionRoots.addAll(actionRoots); all_actionRoots.addAll(imp_actionRoots); + // COPY ACTIONS if ( imp_actionRoots!=null ) { DoubleKeyMap namedActions = new DoubleKeyMap(); @@ -468,6 +470,7 @@ public class Tool { } } + // COPY RULES List rules = imp.ast.getNodesWithType(ANTLRParser.RULE); if ( rules!=null ) { for (GrammarAST r : rules) { @@ -480,6 +483,12 @@ public class Tool { } } } + + GrammarAST optionsRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.OPTIONS); + if ( optionsRoot!=null ) { + errMgr.grammarError(ErrorType.OPTIONS_IN_DELEGATE, + optionsRoot.g.fileName, optionsRoot.token, imp.name); + } } System.out.println("Grammar: "+rootGrammar.ast.toStringTree()); } @@ -560,7 +569,9 @@ public class Tool { } } int nLexicalRules = rulesWeMoved.size(); - rules.removeAll(rulesWeMoved); + for (GrammarAST r : rulesWeMoved) { + combinedRulesRoot.deleteChild( r ); + } // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if' Map litAliases = @@ -576,6 +587,7 @@ public class Tool { Set stringLiterals = combinedGrammar.getStringLiterals(); // add strings from combined grammar (and imported grammars) into lexer // put them first as they are keywords; must resolve ambigs to these rules +// System.out.println("strings from parser: "+stringLiterals); for (String lit : stringLiterals) { if ( litAliases!=null && litAliases.containsKey(lit) ) continue; // already has rule // create for each literal: (RULE (BLOCK (ALT )) @@ -591,10 +603,15 @@ public class Tool { litRule.addChild(new TerminalAST(idToken)); litRule.addChild(blk); lexerRulesRoot.getChildren().add(0, litRule); // add first + lexerRulesRoot.freshenParentAndChildIndexes(); // reset indexes and set litRule parent } - lexerAST.freshenParentAndChildIndexesDeeply(); - combinedAST.freshenParentAndChildIndexesDeeply(); + lexerAST.sanityCheckParentAndChildIndexes(); + combinedAST.sanityCheckParentAndChildIndexes(); +// System.out.println(combinedAST.toTokenString()); + +// lexerAST.freshenParentAndChildIndexesDeeply(); +// combinedAST.freshenParentAndChildIndexesDeeply(); System.out.println("after extract implicit lexer ="+combinedAST.toStringTree()); System.out.println("lexer ="+lexerAST.toStringTree()); diff --git a/tool/src/org/antlr/v4/codegen/ActionTranslator.java b/tool/src/org/antlr/v4/codegen/ActionTranslator.java index 2621fb438..97e4fb9d2 100644 --- a/tool/src/org/antlr/v4/codegen/ActionTranslator.java +++ b/tool/src/org/antlr/v4/codegen/ActionTranslator.java @@ -40,6 +40,14 @@ import java.util.*; /** */ public class ActionTranslator implements ActionSplitterListener { + public static final Map thisRulePropToModelMap = new HashMap() {{ + put("start", ThisRulePropertyRef_start.class); + put("stop", ThisRulePropertyRef_stop.class); + put("tree", ThisRulePropertyRef_tree.class); + put("text", ThisRulePropertyRef_text.class); + put("st", ThisRulePropertyRef_st.class); + }}; + public static final Map rulePropToModelMap = new HashMap() {{ put("start", RulePropertyRef_start.class); put("stop", RulePropertyRef_stop.class); @@ -123,7 +131,7 @@ public class ActionTranslator implements ActionSplitterListener { switch ( a.dict.type ) { case ARG: chunks.add(new ArgRef(x.getText())); break; case RET: chunks.add(new RetValueRef(x.getText())); break; -// case PREDEFINED_RULE: chunks.add(new RetValueRef(x.getText())); break; + case PREDEFINED_RULE: chunks.add(getRulePropertyRef(x)); break; // case PREDEFINED_TREE_RULE: chunks.add(new RetValueRef(x.getText())); break; } } @@ -175,7 +183,14 @@ public class ActionTranslator implements ActionSplitterListener { else { chunks.add(new QRetValueRef(getRuleLabel(x.getText()), y.getText())); break; } - case PREDEFINED_RULE: chunks.add(getRulePropertyRef(x, y)); break; + case PREDEFINED_RULE: + if ( a.dict == Rule.predefinedRulePropertiesDict ) { + chunks.add(getRulePropertyRef(y)); + } + else { + chunks.add(getRulePropertyRef(x, y)); + } + break; case TOKEN: chunks.add(getTokenPropertyRef(x, y)); break; // case PREDEFINED_LEXER_RULE: chunks.add(new RetValueRef(x.getText())); break; // case PREDEFINED_TREE_RULE: chunks.add(new RetValueRef(x.getText())); break; @@ -259,9 +274,24 @@ public class ActionTranslator implements ActionSplitterListener { return null; } - RulePropertyRef getRulePropertyRef(Token x, Token y) { + // $text + RulePropertyRef getRulePropertyRef(Token prop) { try { - Class c = rulePropToModelMap.get(y.getText()); + Class c = thisRulePropToModelMap.get(prop.getText()); + Constructor ctor = c.getConstructor(new Class[] {String.class}); + RulePropertyRef ref = + (RulePropertyRef)ctor.newInstance(getRuleLabel(prop.getText())); + return ref; + } + catch (Exception e) { + factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, e); + } + return null; + } + + RulePropertyRef getRulePropertyRef(Token x, Token prop) { + try { + Class c = rulePropToModelMap.get(prop.getText()); Constructor ctor = c.getConstructor(new Class[] {String.class}); RulePropertyRef ref = (RulePropertyRef)ctor.newInstance(getRuleLabel(x.getText())); diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_st.java b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_st.java new file mode 100644 index 000000000..b022eb74a --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_st.java @@ -0,0 +1,37 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model.actions; + +/** */ +public class ThisRulePropertyRef_st extends RulePropertyRef { + public ThisRulePropertyRef_st(String label) { + super(label); + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_start.java b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_start.java new file mode 100644 index 000000000..a32be77e3 --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_start.java @@ -0,0 +1,37 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model.actions; + +/** */ +public class ThisRulePropertyRef_start extends RulePropertyRef { + public ThisRulePropertyRef_start(String label) { + super(label); + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_stop.java b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_stop.java new file mode 100644 index 000000000..0cfe627fc --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_stop.java @@ -0,0 +1,37 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model.actions; + +/** */ +public class ThisRulePropertyRef_stop extends RulePropertyRef { + public ThisRulePropertyRef_stop(String label) { + super(label); + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_text.java new file mode 100644 index 000000000..c92cc3994 --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_text.java @@ -0,0 +1,37 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model.actions; + +/** */ +public class ThisRulePropertyRef_text extends RulePropertyRef { + public ThisRulePropertyRef_text(String label) { + super(label); + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_tree.java b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_tree.java new file mode 100644 index 000000000..9a37634ec --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_tree.java @@ -0,0 +1,37 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model.actions; + +/** */ +public class ThisRulePropertyRef_tree extends RulePropertyRef { + public ThisRulePropertyRef_tree(String label) { + super(label); + } +} diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index ca80bc982..bff0f1be6 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -85,7 +85,7 @@ public String grammarName; public GrammarAST currentRuleAST; public String currentModeName = LexerGrammar.DEFAULT_MODE_NAME; public String currentRuleName; -public GrammarAST currentRuleBlock; +//public GrammarAST currentRuleBlock; public GrammarAST currentOuterAltRoot; public int currentOuterAltNumber = 1; // 1..n public int rewriteEBNFLevel = 0; @@ -136,12 +136,11 @@ public void discoverRules(GrammarAST rules) { } public void finishRules(GrammarAST rule) { } public void discoverRule(RuleAST rule, GrammarAST ID, List modifiers, ActionAST arg, ActionAST returns, GrammarAST thrws, - GrammarAST options, List actions, + GrammarAST options, List actions, GrammarAST block) { } public void finishRule(GrammarAST rule, GrammarAST ID, GrammarAST block) { } public void ruleCatch(GrammarAST arg, ActionAST action) { } public void finallyAction(ActionAST action) { } -public void ruleNamedAction(GrammarAST ID, ActionAST action) { } /** outermost alt */ public void discoverAlt(AltAST alt) { } /** outermost alt */ @@ -253,7 +252,7 @@ mode : ^( MODE ID {currentModeName=$ID.text; modeDef($MODE, $ID);} rule+ ) ; rule @init { List mods = new ArrayList(); -List actions = new ArrayList(); +List actions = new ArrayList(); // track roots } : ^( RULE ID {currentRuleName=$ID.text; currentRuleAST=$RULE;} DOC_COMMENT? (^(RULEMODIFIERS (m=ruleModifier{mods.add($m.start);})+))? @@ -262,7 +261,7 @@ List actions = new ArrayList(); thr=throwsSpec? ( ruleScopeSpec | opts=optionsSpec - | a=ruleAction {actions.add((ActionAST)$a.start);} + | a=ruleAction {actions.add($a.start);} )* {discoverRule((RuleAST)$RULE, $ID, mods, (ActionAST)$ARG_ACTION, $ret.start!=null?(ActionAST)$ret.start.getChild(0):null, @@ -298,7 +297,7 @@ ruleScopeSpec ; ruleAction - : ^(AT ID ACTION) {ruleNamedAction($ID, (ActionAST)$ACTION);} + : ^(AT ID ACTION) ; ruleModifier diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index ca5740221..a32d2b68e 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -207,7 +207,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { List modifiers, ActionAST arg, ActionAST returns, GrammarAST thrws, GrammarAST options, - List actions, GrammarAST block) + List actions, GrammarAST block) { checkInvalidRuleDef(ID.token); } @@ -363,16 +363,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { Token optionID, String value) { boolean ok = true; - if ( optionID.getText().equals("tokenVocab") && - g.parent!=null ) // only allow tokenVocab option in root grammar - { - g.tool.errMgr.grammarError(ErrorType.TOKEN_VOCAB_IN_DELEGATE, - g.fileName, - optionID, - g.name); - ok = false; - } - if ( parent.getType()==ANTLRParser.BLOCK ) { if ( !legalBlockOptions.contains(optionID.getText()) ) { // block g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index d862a5c57..d91389014 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -105,7 +105,7 @@ public class SymbolCollector extends GrammarTreeVisitor { public void discoverRule(RuleAST rule, GrammarAST ID, List modifiers, ActionAST arg, ActionAST returns, GrammarAST thrws, - GrammarAST options, List actions, + GrammarAST options, List actions, GrammarAST block) { int numAlts = block.getChildCount(); @@ -127,6 +127,13 @@ public class SymbolCollector extends GrammarTreeVisitor { r.retvals.type = AttributeDict.DictType.RET; r.retvals.ast = returns; } + + for (GrammarAST a : actions) { + // a = ^(AT ID ACTION) + ActionAST action = (ActionAST) a.getChild(1); + currentRule.namedActions.put(a.getChild(0).getText(), action); + action.resolver = currentRule; + } } @Override @@ -137,12 +144,6 @@ public class SymbolCollector extends GrammarTreeVisitor { currentRule.alt[currentOuterAltNumber].ast = alt; } - @Override - public void ruleNamedAction(GrammarAST ID, ActionAST action) { - currentRule.namedActions.put(ID.getText(), action); - action.resolver = currentRule; - } - @Override public void actionInAlt(ActionAST action) { currentRule.defineActionInAlt(currentOuterAltNumber, action); diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java index 158ab2292..22ff5f29e 100644 --- a/tool/src/org/antlr/v4/tool/ErrorManager.java +++ b/tool/src/org/antlr/v4/tool/ErrorManager.java @@ -156,12 +156,8 @@ public class ErrorManager { org.antlr.runtime.RecognitionException antlrException, Object... args) { - switch ( etype.severity ) { - case WARNING: warnings++; break; - case ERROR: errors++; break; - } ANTLRMessage msg = new GrammarSyntaxMessage(etype,fileName,token,antlrException,args); - tool.error(msg); + emit(etype, msg); } public static void fatalInternalError(String error, Throwable e) { @@ -188,19 +184,14 @@ public class ErrorManager { * @param args The arguments to pass to the StringTemplate */ public void toolError(ErrorType errorType, Object... args) { - switch ( errorType.severity ) { - case WARNING: warnings++; break; - case ERROR: errors++; break; - } - tool.error(new ToolMessage(errorType, args)); + ToolMessage msg = new ToolMessage(errorType, args); + emit(errorType, msg); + tool.error(msg); } public void toolError(ErrorType errorType, Throwable e, Object... args) { - switch ( errorType.severity ) { - case WARNING: warnings++; break; - case ERROR: errors++; break; - } - tool.error(new ToolMessage(errorType, e, args)); + ToolMessage msg = new ToolMessage(errorType, e, args); + emit(errorType, msg); } public void grammarError(ErrorType etype, @@ -208,13 +199,10 @@ public class ErrorManager { org.antlr.runtime.Token token, Object... args) { - switch ( etype.severity ) { - case WARNING: warnings++; break; - case ERROR: errors++; break; - } ANTLRMessage msg = new GrammarSemanticsMessage(etype,fileName,token,args); - tool.error(msg); - } + emit(etype, msg); + + } public void leftRecursionCycles(String fileName, Collection cycles) { errors++; @@ -242,6 +230,13 @@ public class ErrorManager { // S U P P O R T C O D E + public void emit(ErrorType etype, ANTLRMessage msg) { + switch ( etype.severity ) { + case WARNING: warnings++; tool.warning(msg); break; + case ERROR: errors++; tool.error(msg); break; + } + } + /** The format gets reset either from the Tool if the user supplied a command line option to that effect * Otherwise we just use the default "antlr". */ diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index fd69b3ac1..da8fd0111 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -125,8 +125,9 @@ public enum ErrorType { NO_SUCH_RULE_IN_SCOPE("rule is not defined in grammar ", ErrorSeverity.ERROR), TOKEN_STRING_REASSIGNMENT("cannot alias ; string already assigned to ", ErrorSeverity.ERROR), TOKEN_NAME_REASSIGNMENT("cannot redefine ; token name already assigned to defined", ErrorSeverity.ERROR), - TOKEN_VOCAB_IN_DELEGATE("tokenVocab option ignored in imported grammar ", ErrorSeverity.ERROR), - TOKEN_ALIAS_IN_DELEGATE("can't assign string to token name to string in imported grammar ", ErrorSeverity.ERROR), + //TOKEN_VOCAB_IN_DELEGATE("tokenVocab option ignored in imported grammar ", ErrorSeverity.ERROR), + OPTIONS_IN_DELEGATE("options ignored in imported grammar ", ErrorSeverity.WARNING), +// TOKEN_ALIAS_IN_DELEGATE("can't assign string to token name to string in imported grammar ", ErrorSeverity.ERROR), CANNOT_FIND_IMPORTED_FILE("can't find or load grammar ", ErrorSeverity.ERROR), INVALID_IMPORT(" grammar cannot import grammar ", ErrorSeverity.ERROR), IMPORTED_TOKENS_RULE_EMPTY("", ErrorSeverity.ERROR), diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index 19028d01c..4187049e6 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -711,7 +711,6 @@ public class Grammar implements AttributeResolver { public Set getStringLiterals() { // TODO: super inefficient way to get these. - GrammarASTAdaptor adaptor = new GrammarASTAdaptor(); SymbolCollector collector = new SymbolCollector(this); collector.process(ast); // no side-effects; find strings return collector.strings; diff --git a/tool/src/org/antlr/v4/tool/GrammarAST.java b/tool/src/org/antlr/v4/tool/GrammarAST.java index 4bbee6280..601ccf63a 100644 --- a/tool/src/org/antlr/v4/tool/GrammarAST.java +++ b/tool/src/org/antlr/v4/tool/GrammarAST.java @@ -31,7 +31,7 @@ package org.antlr.v4.tool; import org.antlr.runtime.*; import org.antlr.runtime.tree.*; -import org.antlr.v4.parse.ANTLRParser; +import org.antlr.v4.parse.*; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.IntervalSet; @@ -89,6 +89,14 @@ public class GrammarAST extends CommonTree { return null; } + public void deleteChild(GrammarAST t) { + List dup = new ArrayList(); + dup.addAll(children); + for (Object c : dup) { + if ( c == t ) deleteChild(t.getChildIndex()); + } + } + // TODO: move to basetree when i settle on how runtime works // TODO: don't include this node!! // TODO: reuse other method @@ -131,4 +139,23 @@ public class GrammarAST extends CommonTree { public String toString() { return super.toString(); } + + public String toTokenString() { + CharStream input = this.token.getInputStream(); + GrammarASTAdaptor adaptor = new GrammarASTAdaptor(input); + CommonTreeNodeStream nodes = + new CommonTreeNodeStream(adaptor, this); + StringBuffer buf = new StringBuffer(); + GrammarAST o = (GrammarAST)nodes.LT(1); + int type = adaptor.getType(o); + while ( type!=Token.EOF ) { + buf.append(" "); + buf.append(o.token.getText()); + nodes.consume(); + o = (GrammarAST)nodes.LT(1); + type = adaptor.getType(o); + } + return buf.toString(); + } + } diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index b74154e00..3cf58df58 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -774,12 +774,6 @@ public abstract class BaseTest { GrammarSemanticsMessage expectedMessage) throws Exception { - /* - System.out.println(equeue.infos); - System.out.println(equeue.warnings); - System.out.println(equeue.errors); - assertTrue("number of errors mismatch", n, equeue.errors.size()); - */ ANTLRMessage foundMsg = null; for (int i = 0; i < equeue.errors.size(); i++) { ANTLRMessage m = (ANTLRMessage)equeue.errors.get(i); @@ -796,6 +790,26 @@ public abstract class BaseTest { } } + protected void checkGrammarSemanticsWarning(ErrorQueue equeue, + GrammarSemanticsMessage expectedMessage) + throws Exception + { + ANTLRMessage foundMsg = null; + for (int i = 0; i < equeue.warnings.size(); i++) { + ANTLRMessage m = equeue.warnings.get(i); + if (m.errorType==expectedMessage.errorType ) { + foundMsg = m; + } + } + assertNotNull("no error; "+expectedMessage.errorType+" expected", foundMsg); + assertTrue("error is not a GrammarSemanticsMessage", + foundMsg instanceof GrammarSemanticsMessage); + assertEquals(Arrays.toString(expectedMessage.args), Arrays.toString(foundMsg.args)); + if ( equeue.size()!=1 ) { + System.err.println(equeue); + } + } + protected void checkError(ErrorQueue equeue, ANTLRMessage expectedMessage) throws Exception diff --git a/tool/test/org/antlr/v4/test/TestActionTranslation.java b/tool/test/org/antlr/v4/test/TestActionTranslation.java index 96604e29e..94df05f21 100644 --- a/tool/test/org/antlr/v4/test/TestActionTranslation.java +++ b/tool/test/org/antlr/v4/test/TestActionTranslation.java @@ -235,7 +235,15 @@ public class TestActionTranslation extends BaseTest { testActions(scopeTemplate, "inline", action, expected); } - + @Test public void testRefToTextAttributeForCurrentRule() throws Exception { + String action = "$a.text; $text"; + String expected = + "((TokenStream)input).toString(_localctx.start, _localctx.stop); " + + "((TokenStream)input).toString(_localctx.start, _localctx.stop)"; + testActions(attributeTemplate, "init", action, expected); + testActions(attributeTemplate, "inline", action, expected); + testActions(attributeTemplate, "finally", action, expected); + } @Test public void testDynamicRuleScopeRefInSubrule() throws Exception { String action = "$a::n;"; @@ -276,9 +284,7 @@ public class TestActionTranslation extends BaseTest { @Test public void testRefToTemplateAttributeForCurrentRule() throws Exception { String action = "$st=null;"; } - @Test public void testRefToTextAttributeForCurrentRule() throws Exception { - String action = "$text"; - } + @Test public void testRefToStartAttributeForCurrentRule() throws Exception { String action = "$start;"; } diff --git a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java index 440cb5585..b13693cbe 100644 --- a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java +++ b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java @@ -67,6 +67,22 @@ public class TestCompositeGrammars extends BaseTest { assertEquals("S.a\n", found); } + @Test public void testBringInLiteralsFromDelegate() throws Exception { + String slave = + "parser grammar S;\n" + + "a : '=' 'a' {System.out.println(\"S.a\");} ;\n"; + mkdir(tmpdir); + writeFile(tmpdir, "S.g", slave); + String master = + "grammar M;\n" + + "import S;\n" + + "s : a ;\n" + + "WS : (' '|'\\n') {skip();} ;\n" ; + String found = execParser("M.g", master, "MParser", "MLexer", + "s", "=a", debug); + assertEquals("S.a\n", found); + } + @Test public void testDelegatorInvokesDelegateRuleWithArgs() throws Exception { // must generate something like: // public int a(int x) throws RecognitionException { return gS.a(x); } @@ -254,58 +270,8 @@ public class TestCompositeGrammars extends BaseTest { assertEquals("S.x\n", found); } - @Test public void testSameStringTwoNames() throws Exception { - ErrorQueue equeue = new ErrorQueue(); - String slave = - "parser grammar S;\n" + - "tokens { A='a'; }\n" + - "x : A {System.out.println(\"S.x\");} ;\n"; - mkdir(tmpdir); - writeFile(tmpdir, "S.g", slave); - String slave2 = - "parser grammar T;\n" + - "tokens { X='a'; }\n" + - "y : X {System.out.println(\"T.y\");} ;\n"; - mkdir(tmpdir); - writeFile(tmpdir, "T.g", slave2); - - String master = - "grammar M;\n" + - "import S,T;\n" + - "s : x y ;\n" + - "WS : (' '|'\\n') {skip();} ;\n" ; - writeFile(tmpdir, "M.g", master); - - Grammar g = new Grammar(tmpdir+"/M.g", master, equeue); - - String expectedTokenIDToTypeMap = "{EOF=-1, T__0=3, WS=4, A=5, X=6}"; - String expectedStringLiteralToTypeMap = "{'a'=6}"; - String expectedTypeToTokenList = "[T__0, WS, A, X]"; - - assertEquals(expectedTokenIDToTypeMap, g.tokenNameToTypeMap.toString()); - assertEquals(expectedStringLiteralToTypeMap, g.stringLiteralToTypeMap.toString()); - assertEquals(expectedTypeToTokenList, realElements(g.typeToTokenList).toString()); - - Object expectedArg = "X='a'"; - Object expectedArg2 = "A"; - ErrorType expectedMsgID = ErrorType.TOKEN_STRING_REASSIGNMENT; - GrammarSemanticsMessage expectedMessage = - new GrammarSemanticsMessage(expectedMsgID, g.fileName, null, expectedArg, expectedArg2); - checkGrammarSemanticsError(equeue, expectedMessage); - - assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); - - String expectedError = - "error(73): T.g:2:9: cannot alias X='a'; string already assigned to A"; -// ST msgST = antlr.errMgr.getMessageTemplate(equeue.errors.get(0)); -// String foundError = msgST.render(); -// assertEquals(expectedError, foundError); - } - - /* @Test public void testImportedTokenVocabIgnoredWithWarning() throws Exception { ErrorQueue equeue = new ErrorQueue(); - ErrorManager.setErrorListener(equeue); String slave = "parser grammar S;\n" + "options {tokenVocab=whatever;}\n" + @@ -320,71 +286,20 @@ public class TestCompositeGrammars extends BaseTest { "s : x ;\n" + "WS : (' '|'\\n') {skip();} ;\n" ; writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - CompositeGrammar composite = new CompositeGrammar(); - Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); - composite.setDelegationRoot(g); - g.parseAndBuildAST(); - g.composite.assignTokenTypes(); + Grammar g = new Grammar(tmpdir+"/M.g", master, equeue); Object expectedArg = "S"; - int expectedMsgID = ErrorManager.MSG_TOKEN_VOCAB_IN_DELEGATE; + ErrorType expectedMsgID = ErrorType.OPTIONS_IN_DELEGATE; GrammarSemanticsMessage expectedMessage = - new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg); + new GrammarSemanticsMessage(expectedMsgID, g.fileName, null, expectedArg); checkGrammarSemanticsWarning(equeue, expectedMessage); assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); - assertEquals("unexpected errors: "+equeue, 1, equeue.warnings.size()); - - String expectedError = - "warning(160): S.g:2:10: tokenVocab option ignored in imported grammar S"; - assertEquals(expectedError, equeue.warnings.get(0).toString()); - } - - @Test public void testImportedTokenVocabWorksInRoot() throws Exception { - ErrorQueue equeue = new ErrorQueue(); - ErrorManager.setErrorListener(equeue); - String slave = - "parser grammar S;\n" + - "tokens { A='a'; }\n" + - "x : A {System.out.println(\"S.x\");} ;\n"; - mkdir(tmpdir); - writeFile(tmpdir, "S.g", slave); - - String tokens = - "A=99\n"; - writeFile(tmpdir, "Test.tokens", tokens); - - String master = - "grammar M;\n" + - "options {tokenVocab=Test;}\n" + - "import S;\n" + - "s : x ;\n" + - "WS : (' '|'\\n') {skip();} ;\n" ; - writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - CompositeGrammar composite = new CompositeGrammar(); - Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); - composite.setDelegationRoot(g); - g.parseAndBuildAST(); - g.composite.assignTokenTypes(); - - String expectedTokenIDToTypeMap = "[A=99, WS=101]"; - String expectedStringLiteralToTypeMap = "{'a'=100}"; - String expectedTypeToTokenList = "[A, 'a', WS]"; - - assertEquals(expectedTokenIDToTypeMap, - realElements(g.composite.tokenIDToTypeMap).toString()); - assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString()); - assertEquals(expectedTypeToTokenList, - realElements(g.composite.typeToTokenList).toString()); - - assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size()); + assertEquals("unexpected warnings: "+equeue, 1, equeue.warnings.size()); } @Test public void testSyntaxErrorsInImportsNotThrownOut() throws Exception { ErrorQueue equeue = new ErrorQueue(); - ErrorManager.setErrorListener(equeue); String slave = "parser grammar S;\n" + "options {toke\n"; @@ -397,41 +312,9 @@ public class TestCompositeGrammars extends BaseTest { "s : x ;\n" + "WS : (' '|'\\n') {skip();} ;\n" ; writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - CompositeGrammar composite = new CompositeGrammar(); - Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); - composite.setDelegationRoot(g); - g.parseAndBuildAST(); - g.composite.assignTokenTypes(); + Grammar g = new Grammar(tmpdir+"/M.g", master, equeue); - // whole bunch of errors from bad S.g file - assertEquals("unexpected errors: "+equeue, 5, equeue.errors.size()); - } - - @Test public void testSyntaxErrorsInImportsNotThrownOut2() throws Exception { - ErrorQueue equeue = new ErrorQueue(); - ErrorManager.setErrorListener(equeue); - String slave = - "parser grammar S;\n" + - ": A {System.out.println(\"S.x\");} ;\n"; - mkdir(tmpdir); - writeFile(tmpdir, "S.g", slave); - - String master = - "grammar M;\n" + - "import S;\n" + - "s : x ;\n" + - "WS : (' '|'\\n') {skip();} ;\n" ; - writeFile(tmpdir, "M.g", master); - Tool antlr = newTool(new String[] {"-lib", tmpdir}); - CompositeGrammar composite = new CompositeGrammar(); - Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite); - composite.setDelegationRoot(g); - g.parseAndBuildAST(); - g.composite.assignTokenTypes(); - - // whole bunch of errors from bad S.g file - assertEquals("unexpected errors: "+equeue, 3, equeue.errors.size()); + assertEquals(ErrorType.SYNTAX_ERROR, equeue.errors.get(0).errorType); } @Test public void testDelegatorRuleOverridesDelegate() throws Exception { @@ -456,9 +339,9 @@ public class TestCompositeGrammars extends BaseTest { "parser grammar JavaDecl;\n" + "type : 'int' ;\n" + "decl : type ID ';'\n" + - " | type ID init ';' {System.out.println(\"JavaDecl: \"+$decl.text);}\n" + + " | type ID init ';' {System.out.println(\"JavaDecl: \"+$text);}\n" + " ;\n" + - "init : '=' INT ;\n" ; + "init : '=' INT ;\n"; mkdir(tmpdir); writeFile(tmpdir, "JavaDecl.g", slave); String master = @@ -476,6 +359,7 @@ public class TestCompositeGrammars extends BaseTest { assertEquals("JavaDecl: floatx=3;\n", found); } + /* @Test public void testDelegatorRuleOverridesDelegates() throws Exception { String slave = "parser grammar S;\n" +