diff --git a/build.xml b/build.xml index 051f0bce4..e3aa63238 100644 --- a/build.xml +++ b/build.xml @@ -28,7 +28,7 @@ - parse + parse grammars @@ -51,6 +51,7 @@ + @@ -89,7 +90,9 @@ - + + + - - - - - - + + + + + + + + + + + + + + + + + + diff --git a/tool/src/org/antlr/v4/runtime/atn/ATNContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNContext.java similarity index 100% rename from tool/src/org/antlr/v4/runtime/atn/ATNContext.java rename to runtime/Java/src/org/antlr/v4/runtime/atn/ATNContext.java diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java index 571d82af5..93a5f80a8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java @@ -53,7 +53,7 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor { */ public List createElementList() { - return new ArrayList(3); + return new ElementList(this); } // END v4 stuff diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ElementList.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ElementList.java new file mode 100644 index 000000000..989dd272d --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ElementList.java @@ -0,0 +1,79 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.runtime.tree; + +import java.util.*; + +/** This list tracks elements to left of -> for use on right of -> */ +public class ElementList extends ArrayList { + protected TreeAdaptor adaptor; + + public class ElementListIterator implements Iterator { + int cursor = 0; + + public boolean hasNext() { + int n = size(); + return (n==1 && cursor<1) || (n>1 && cursor= n) { // out of elements? + if ( n == 1 ) { // if size is 1, it's ok; return and we'll dup + return (E)adaptor.dupTree( get(0) ); + } + // out of elements and size was not 1, so we can't dup + throw new RewriteCardinalityException("size=="+n+" and out of elements"); + } + + // we have elements + if ( n == 1 ) { + cursor++; // move cursor even for single element list + return (E)adaptor.dupTree( get(0) ); + } + // must have more than one in list, pull from elements + E e = get(cursor); + cursor++; + return e; + } + + public void remove() { throw new UnsupportedOperationException(); } + } + + public ElementList(TreeAdaptor adaptor) { + this.adaptor = adaptor; + } + + @Override + public Iterator iterator() { + return new ElementListIterator(); + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/RewriteCardinalityException.java b/runtime/Java/src/org/antlr/v4/runtime/tree/RewriteCardinalityException.java new file mode 100644 index 000000000..5c881643f --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/RewriteCardinalityException.java @@ -0,0 +1,49 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.runtime.tree; + +/** Base class for all exceptions thrown during AST rewrite construction. + * This signifies a case where the cardinality of two or more elements + * in a subrule are different: (ID INT)* where |ID|!=|INT| + */ +public class RewriteCardinalityException extends RuntimeException { + public String elementDescription; + + public RewriteCardinalityException(String elementDescription) { + this.elementDescription = elementDescription; + } + + public String getMessage() { + if ( elementDescription!=null ) { + return elementDescription; + } + return null; + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/RewriteEmptyStreamException.java b/runtime/Java/src/org/antlr/v4/runtime/tree/RewriteEmptyStreamException.java new file mode 100644 index 000000000..adb5a4825 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/RewriteEmptyStreamException.java @@ -0,0 +1,37 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.runtime.tree; + +/** Ref to ID or expr but no tokens in ID stream or subtrees in expr stream */ +public class RewriteEmptyStreamException extends RewriteCardinalityException { + public RewriteEmptyStreamException(String elementDescription) { + super(elementDescription); + } +} diff --git a/tool/playground/T.g b/tool/playground/T.g index 62e0318d1..3d2ff170e 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,15 +1,15 @@ grammar T; options {output=AST;} -tokens {I;} -/* -a : A b C -> ^(A ^(b C)) - | B +tokens {I;D;} + +a : A B+ -> ^(A B)* ; b : B | C ; -*/ -c : A B C -> (A B C*)* ; +/* +c : A B C -> A ( D A B C*)* (B A*)? ; +*/ A : 'a'; B : 'b'; diff --git a/tool/playground/TestT.java b/tool/playground/TestT.java index 73a905eaa..94301b49b 100644 --- a/tool/playground/TestT.java +++ b/tool/playground/TestT.java @@ -2,6 +2,7 @@ import org.antlr.v4.Tool; import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.tree.Tree; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.tool.*; @@ -12,8 +13,8 @@ public class TestT { TLexer t = new TLexer(new ANTLRFileStream(args[0])); CommonTokenStream tokens = new CommonTokenStream(t); TParser p = new TParser(tokens); -// ParserRuleContext ret = p.a(); -// System.out.println(((Tree)ret.tree).toStringTree()); + ParserRuleContext ret = p.a(); + System.out.println(((Tree)ret.tree).toStringTree()); } public static void dump() throws Exception { diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index a55e84c06..a866e96d1 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -381,8 +381,10 @@ ElementListName(elemName) ::= "_track_" TrackRuleElement(e) ::= ".add(.tree);" TrackTokenElement(e) ::= ".add(_adaptor.create());" +// assume roots are always locals in tree rewrites + TreeRewrite(tr, locals, preamble, ops) ::= << -// rewrite: ... +// rewrite: code level= , tree level = @@ -394,41 +396,52 @@ RewriteIteratorInit(i) ::= " = .iterator();" RewriteIteratorName(elemName,level) ::= "it_" RewriteTreeOptional(o, locals, preamble, ops) ::= << -// ? - - -//if ( true ) { - -//} ->> - -RewriteTreeClosure(c, locals, preamble, ops) ::= << -// * - - -//while ( .hasNext()}; separator=" || "> ) { - -//} ->> - -RewriteTreeStructure(t, locals, preamble, ops) ::= << +// ? code level= , tree level = { - - - - _adaptor.addChild(, ); + + + if ( .hasNext()}; separator=" || "> ) { + + } } >> -RewriteTokenRef(t) ::= "_adaptor.addChild(, .next());" -RewriteTokenRefIsRoot(t) ::= << - = _adaptor.becomeRoot(.next(), ); +RewriteTreeClosure(c, locals, preamble, ops) ::= << +// * code level= , tree level = +{ + + + while ( .hasNext()}; separator=" || "> ) { + + } +} >> -RewriteRuleRef(r) ::= "_adaptor.addChild(, .next());" -RewriteRuleRefIsRoot(r) ::= <% - = - _adaptor.becomeRoot(.next(), ); + +RewriteTreeStructure(t, locals, ops) ::= << +// TREE code level= , tree level = +{ + + + _adaptor.addChild(_root, _root); +} +>> + +RewriteTokenRef(t) ::= "_adaptor.addChild(, .next());" +RewriteTokenRefIsRoot(t) ::= << + = _adaptor.becomeRoot(.next(), ); +>> + +//adaptor.addChild(root_0, (Object)adaptor.create(D, "D")); +RewriteImagTokenRef(t) ::= + "_adaptor.addChild(, _adaptor.create(, \"\"));" +RewriteImagTokenRefIsRoot(t) ::= <% + = + _adaptor.becomeRoot(_adaptor.create(, ""), + ); %> +RewriteRuleRef(r) ::= "_adaptor.addChild(, .next());" +RewriteRuleRefIsRoot(r) ::= + " = _adaptor.becomeRoot(.next(), );" /* BitSetDecl(b) ::= << diff --git a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java index fedb4ea7e..164eb669a 100644 --- a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java @@ -111,6 +111,7 @@ public abstract class DefaultOutputModelFactory extends BlankOutputModelFactory } public Decl getCurrentDeclForName(String name) { + if ( getCurrentBlock().locals==null ) return null; for (Decl d : getCurrentBlock().locals.elements()) { if ( d.name.equals(name) ) return d; } diff --git a/tool/src/org/antlr/v4/codegen/OutputModelController.java b/tool/src/org/antlr/v4/codegen/OutputModelController.java index c2eee8272..00f6556bc 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelController.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelController.java @@ -296,7 +296,6 @@ public class OutputModelController implements OutputModelFactory { } public RewriteTreeClosure rewrite_closure(GrammarAST ast) { - List refs = getElementReferencesShallow(ast); RewriteTreeClosure c = delegate.rewrite_closure(ast); for (CodeGeneratorExtension ext : extensions) c = ext.rewrite_closure(c); return c; @@ -348,69 +347,12 @@ public class OutputModelController implements OutputModelFactory { // SUPPORT - /** Given (('?'|'*') (REWRITE_BLOCK (ALT ...))) return list of element refs at - * top level of REWRITE_BLOCK. - */ - public List getElementReferencesShallow(GrammarAST ebnfRoot) { - if ( ebnfRoot.getType()!=ANTLRParser.CLOSURE && - ebnfRoot.getType()!=ANTLRParser.OPTIONAL ) - { - return null; - } - GrammarAST blkAST = (GrammarAST)ebnfRoot.getChild(0); - if ( blkAST.getType()!=ANTLRParser.REWRITE_BLOCK ) return null; - GrammarAST altAST = (GrammarAST)blkAST.getChild(0); - if ( altAST.getType()!=ANTLRParser.ALT ) return null; - - IntervalSet elementTokenTypes = getRewriteElementTokenTypeSet(); - Alternative alt = getCurrentAlt(); - List elems = new ArrayList(); - for (Object o : altAST.getChildren()) { - GrammarAST ref = (GrammarAST)o; - if ( elementTokenTypes.member(ref.getType()) ) { - boolean imaginary = ref.getType()==ANTLRParser.TOKEN_REF && - !alt.tokenRefs.containsKey(ref.getText()); - if ( !imaginary ) elems.add(ref); - } - } - - return elems; - } - - /** Given (('?'|'*') (REWRITE_BLOCK (ALT ...))) return list of element refs at - * or below toplevel REWRITE_BLOCK. - */ - public List getElementReferencesDeep(GrammarAST ebnfRoot) { - if ( ebnfRoot.getType()!=ANTLRParser.CLOSURE && - ebnfRoot.getType()!=ANTLRParser.OPTIONAL ) - { - return null; - } - GrammarAST blkAST = (GrammarAST)ebnfRoot.getChild(0); - if ( blkAST.getType()!=ANTLRParser.REWRITE_BLOCK ) return null; - GrammarAST altAST = (GrammarAST)blkAST.getChild(0); - if ( altAST.getType()!=ANTLRParser.ALT ) return null; - - List elems = new ArrayList(); - Alternative alt = getCurrentAlt(); - IntervalSet elementTokenTypes = getRewriteElementTokenTypeSet(); - List refs = altAST.getNodesWithType(elementTokenTypes); - if ( refs!=null ) { - for (GrammarAST ref : refs) { - boolean imaginary = ref.getType()==ANTLRParser.TOKEN_REF && - !alt.tokenRefs.containsKey(ref.getText()); - if ( !imaginary ) elems.add(ref); - } - } - return elems; - } - - public IntervalSet getRewriteElementTokenTypeSet() { - IntervalSet elementTokenTypes = new IntervalSet(); - elementTokenTypes.add(ANTLRParser.TOKEN_REF); // might be imaginary - elementTokenTypes.add(ANTLRParser.RULE_REF); - elementTokenTypes.add(ANTLRParser.STRING_LITERAL); - elementTokenTypes.add(ANTLRParser.LABEL); - return elementTokenTypes; - } +// public IntervalSet getRewriteElementTokenTypeSet() { +// IntervalSet elementTokenTypes = new IntervalSet(); +// elementTokenTypes.add(ANTLRParser.TOKEN_REF); // might be imaginary +// elementTokenTypes.add(ANTLRParser.RULE_REF); +// elementTokenTypes.add(ANTLRParser.STRING_LITERAL); +// elementTokenTypes.add(ANTLRParser.LABEL); +// return elementTokenTypes; +// } } diff --git a/tool/src/org/antlr/v4/codegen/ParserASTExtension.java b/tool/src/org/antlr/v4/codegen/ParserASTExtension.java index 9843c1ad9..ca4cde3af 100644 --- a/tool/src/org/antlr/v4/codegen/ParserASTExtension.java +++ b/tool/src/org/antlr/v4/codegen/ParserASTExtension.java @@ -45,7 +45,7 @@ public class ParserASTExtension extends CodeGeneratorExtension { @Override public CodeBlockForAlt alternative(CodeBlockForAlt blk) { Alternative alt = factory.getCurrentAlt(); - if ( !alt.hasRewrite() ) blk.addLocalDecl( new RootDecl(factory) ); + if ( !alt.hasRewrite() ) blk.addLocalDecl( new RootDecl(factory, 0) ); return blk; } diff --git a/tool/src/org/antlr/v4/codegen/ParserFactory.java b/tool/src/org/antlr/v4/codegen/ParserFactory.java index ba926ac48..22d0a042c 100644 --- a/tool/src/org/antlr/v4/codegen/ParserFactory.java +++ b/tool/src/org/antlr/v4/codegen/ParserFactory.java @@ -36,6 +36,7 @@ import org.antlr.v4.codegen.model.decl.*; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.semantics.UseDefAnalyzer; import org.antlr.v4.tool.*; import java.util.List; @@ -179,23 +180,53 @@ public class ParserFactory extends DefaultOutputModelFactory { @Override public TreeRewrite treeRewrite(GrammarAST ast) { - return new TreeRewrite(this, getTreeLevel(), getCodeBlockLevel()); + TreeRewrite tr = new TreeRewrite(this, getTreeLevel(), getCodeBlockLevel()); + tr.addLocalDecl(new RootDecl(this, 0)); + List refs = + UseDefAnalyzer.getElementReferencesShallowInOuterAlt(getGrammar(), ast); + refs = UseDefAnalyzer.filterForRuleAndTokenRefs(getCurrentAlt(), refs); + if ( refs!=null ) { + for (GrammarAST ref : refs) { + RewriteIteratorDecl d = new RewriteIteratorDecl(this, ref, getCodeBlockLevel()); + tr.addLocalDecl(d); + RewriteIteratorInit init = new RewriteIteratorInit(this, d); + tr.addPreambleOp(init); + } + } + return tr; } @Override public RewriteTreeOptional rewrite_optional(GrammarAST ast) { - List refs = controller.getElementReferencesDeep(ast); - return new RewriteTreeOptional(this, ast); + RewriteTreeOptional o = + new RewriteTreeOptional(this, ast, getTreeLevel(), getCodeBlockLevel()); + List refs = UseDefAnalyzer.getElementReferencesInEBNF(getGrammar(), + ast, + true); + refs = UseDefAnalyzer.filterForRuleAndTokenRefs(getCurrentAlt(), refs); + if ( refs!=null ) { + for (GrammarAST ref : refs) { + RewriteIteratorDecl d = new RewriteIteratorDecl(this, ref, getCodeBlockLevel()); + o.addLocalDecl(d); + o.conditionalDecls.add(d); + RewriteIteratorInit init = new RewriteIteratorInit(this, d); + o.addPreambleOp(init); + } + } + return o; } @Override public RewriteTreeClosure rewrite_closure(GrammarAST ast) { RewriteTreeClosure c = new RewriteTreeClosure(this, ast, getTreeLevel(), getCodeBlockLevel()); - List refs = controller.getElementReferencesShallow(ast); + List refs = UseDefAnalyzer.getElementReferencesInEBNF(getGrammar(), + ast, + false); + refs = UseDefAnalyzer.filterForRuleAndTokenRefs(getCurrentAlt(), refs); if ( refs!=null ) { for (GrammarAST ref : refs) { - RewriteIteratorDecl d = new RewriteIteratorDecl(this, ref); + RewriteIteratorDecl d = new RewriteIteratorDecl(this, ref, getCodeBlockLevel()); c.addLocalDecl(d); c.iteratorDecls.add(d); RewriteIteratorInit init = new RewriteIteratorInit(this, d); @@ -207,24 +238,34 @@ public class ParserFactory extends DefaultOutputModelFactory { @Override public RewriteTreeStructure rewrite_tree(GrammarAST root) { - return new RewriteTreeStructure(this, root, getTreeLevel(), getCodeBlockLevel()); + RewriteTreeStructure t = new RewriteTreeStructure(this, root, getTreeLevel(), getCodeBlockLevel()); + t.addLocalDecl( new RootDecl(this, getTreeLevel()) ); + return t; } public List rewrite_ruleRef(GrammarAST ID, boolean isRoot) { - RewriteIteratorDecl d = new RewriteIteratorDecl(this, ID); + String rootName = gen.target.getRootName(getTreeLevel()); RewriteRuleRef ruleRef; - if ( isRoot ) ruleRef = new RewriteRuleRefIsRoot(this, ID, d); - else ruleRef = new RewriteRuleRef(this, ID, d); + String iterName = gen.target.getRewriteIteratorName(ID, getCodeBlockLevel()); + if ( isRoot ) ruleRef = new RewriteRuleRefIsRoot(this, ID, rootName, iterName); + else ruleRef = new RewriteRuleRef(this, ID, rootName, iterName); return list(ruleRef); } public List rewrite_tokenRef(GrammarAST ID, boolean isRoot) { - String itName = gen.target.getRewriteIteratorName(ID, getCodeBlockLevel()); + Alternative alt = getCurrentAlt(); String rootName = gen.target.getRootName(getTreeLevel()); - RewriteIteratorDecl d = (RewriteIteratorDecl)getCurrentDeclForName(itName); + String iterName = gen.target.getRewriteIteratorName(ID, getCodeBlockLevel()); + if ( alt.tokenRefs.get(ID.getText())==null ) { // not ref'd on left hand side; imaginary + RewriteImagTokenRef tokenRef; + if ( isRoot ) tokenRef = new RewriteImagTokenRefIsRoot(this, ID, rootName); + else tokenRef = new RewriteImagTokenRef(this, ID, rootName); + return list(tokenRef); + } + // must be token ref on left of -> RewriteTokenRef tokenRef; - if ( isRoot ) tokenRef = new RewriteTokenRefIsRoot(this, ID, rootName, d); - else tokenRef = new RewriteTokenRef(this, ID, rootName, d); + if ( isRoot ) tokenRef = new RewriteTokenRefIsRoot(this, ID, rootName, iterName); + else tokenRef = new RewriteTokenRef(this, ID, rootName, iterName); return list(tokenRef); } diff --git a/tool/src/org/antlr/v4/codegen/SourceGenTriggers.g b/tool/src/org/antlr/v4/codegen/SourceGenTriggers.g index 254cba465..9dae48a95 100644 --- a/tool/src/org/antlr/v4/codegen/SourceGenTriggers.g +++ b/tool/src/org/antlr/v4/codegen/SourceGenTriggers.g @@ -269,12 +269,12 @@ rewriteTreeEbnf returns [CodeBlock op] rewriteTree returns [List omos] : { - codeBlockLevel++; +// codeBlockLevel++; treeLevel++; List elems = new ArrayList(); RewriteTreeStructure t = factory.rewrite_tree($start); - CodeBlock save = factory.getCurrentBlock(); - factory.setCurrentBlock(t); +// CodeBlock save = factory.getCurrentBlock(); +// factory.setCurrentBlock(t); } ^( TREE_BEGIN rewriteTreeAtom[true] {elems.addAll($rewriteTreeAtom.omos);} @@ -283,9 +283,9 @@ rewriteTree returns [List omos] { t.ops = elems; $omos = DefaultOutputModelFactory.list(t); - factory.setCurrentBlock(save); +// factory.setCurrentBlock(save); treeLevel--; - codeBlockLevel--; +// codeBlockLevel--; } ; diff --git a/tool/src/org/antlr/v4/codegen/model/Rewrite.java b/tool/src/org/antlr/v4/codegen/model/Rewrite.java index 5124bcc5c..9e6ce4892 100644 --- a/tool/src/org/antlr/v4/codegen/model/Rewrite.java +++ b/tool/src/org/antlr/v4/codegen/model/Rewrite.java @@ -32,6 +32,7 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.model.decl.CodeBlock; +/** Either an ST or Tree rewrite */ public class Rewrite extends CodeBlock { public Rewrite(OutputModelFactory factory, int treeLevel, int codeBlockLevel) { super(factory, treeLevel, codeBlockLevel); diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteImagTokenRef.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteImagTokenRef.java new file mode 100644 index 000000000..b82b08519 --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteImagTokenRef.java @@ -0,0 +1,46 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model.ast; + +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.codegen.model.SrcOp; +import org.antlr.v4.tool.GrammarAST; + +public class RewriteImagTokenRef extends SrcOp { + public String rootName; + + public RewriteImagTokenRef(OutputModelFactory factory, + GrammarAST ast, + String rootName) + { + super(factory, ast); + this.rootName = rootName; + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteImagTokenRefIsRoot.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteImagTokenRefIsRoot.java new file mode 100644 index 000000000..563c19102 --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteImagTokenRefIsRoot.java @@ -0,0 +1,42 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model.ast; + +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.tool.GrammarAST; + +public class RewriteImagTokenRefIsRoot extends RewriteImagTokenRef { + public RewriteImagTokenRefIsRoot(OutputModelFactory factory, + GrammarAST ast, + String rootName) + { + super(factory, ast, rootName); + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRef.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRef.java index f6c367296..eff6ba6cf 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRef.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRef.java @@ -31,14 +31,17 @@ package org.antlr.v4.codegen.model.ast; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.model.SrcOp; -import org.antlr.v4.codegen.model.decl.RewriteIteratorDecl; import org.antlr.v4.tool.GrammarAST; public class RewriteRuleRef extends SrcOp { - /** Which iterator decl are we associated with? */ - public RewriteIteratorDecl decl; - public RewriteRuleRef(OutputModelFactory factory, GrammarAST ast, RewriteIteratorDecl decl) { + public String rootName; + public String iterName; + + public RewriteRuleRef(OutputModelFactory factory, GrammarAST ast, + String rootName, String iterName) + { super(factory, ast); - this.decl = decl; + this.rootName = rootName; + this.iterName = iterName; } } diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRefIsRoot.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRefIsRoot.java index e61432a30..68c05d02a 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRefIsRoot.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteRuleRefIsRoot.java @@ -30,11 +30,12 @@ package org.antlr.v4.codegen.model.ast; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.codegen.model.decl.RewriteIteratorDecl; import org.antlr.v4.tool.GrammarAST; public class RewriteRuleRefIsRoot extends RewriteRuleRef { - public RewriteRuleRefIsRoot(OutputModelFactory factory, GrammarAST ast, RewriteIteratorDecl decl) { - super(factory, ast, decl); + public RewriteRuleRefIsRoot(OutputModelFactory factory, GrammarAST ast, + String rootName, String iterName) + { + super(factory, ast, rootName, iterName); } } diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRef.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRef.java index 751e908d0..d19dd1b20 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRef.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRef.java @@ -31,18 +31,16 @@ package org.antlr.v4.codegen.model.ast; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.model.SrcOp; -import org.antlr.v4.codegen.model.decl.RewriteIteratorDecl; import org.antlr.v4.tool.GrammarAST; public class RewriteTokenRef extends SrcOp { - /** Which iterator decl are we associated with? */ - public RewriteIteratorDecl decl; public String rootName; + public String iterName; public RewriteTokenRef(OutputModelFactory factory, GrammarAST ast, - String rootName, RewriteIteratorDecl decl) + String rootName, String iterName) { super(factory, ast); this.rootName = rootName; - this.decl = decl; + this.iterName = iterName; } } diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRefIsRoot.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRefIsRoot.java index f2e2f315c..8b2899cf8 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRefIsRoot.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTokenRefIsRoot.java @@ -30,13 +30,12 @@ package org.antlr.v4.codegen.model.ast; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.codegen.model.decl.RewriteIteratorDecl; import org.antlr.v4.tool.GrammarAST; public class RewriteTokenRefIsRoot extends RewriteTokenRef { public RewriteTokenRefIsRoot(OutputModelFactory factory, GrammarAST ast, - String rootName, RewriteIteratorDecl decl) + String rootName, String iterName) { - super(factory, ast, rootName, decl); + super(factory, ast, rootName, iterName); } } diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeOptional.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeOptional.java index fd53095d4..466784e96 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeOptional.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeOptional.java @@ -36,10 +36,11 @@ import org.antlr.v4.tool.GrammarAST; import java.util.*; public class RewriteTreeOptional extends CodeBlock { - public List iteratorDecls = new ArrayList(); + public List conditionalDecls = new ArrayList(); - public RewriteTreeOptional(OutputModelFactory factory, GrammarAST ast) { - super(factory); + public RewriteTreeOptional(OutputModelFactory factory, GrammarAST ast, + int treeLevel, int codeBlockLevel) { + super(factory, treeLevel, codeBlockLevel); this.ast = ast; } } diff --git a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeStructure.java b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeStructure.java index 7a27fdd81..4b9337b32 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeStructure.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/RewriteTreeStructure.java @@ -30,18 +30,37 @@ package org.antlr.v4.codegen.model.ast; import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.codegen.model.*; +import org.antlr.v4.codegen.model.decl.Decl; +import org.antlr.v4.runtime.misc.OrderedHashSet; import org.antlr.v4.tool.GrammarAST; +import java.util.List; + /** ^(A B C) */ -public class RewriteTreeStructure extends TreeRewrite { +public class RewriteTreeStructure extends SrcOp { public int treeLevel; + public int codeBlockLevel; + + @ModelElement public List ops; + @ModelElement public OrderedHashSet locals; public RewriteTreeStructure(OutputModelFactory factory, GrammarAST ast, int treeLevel, int codeBlockLevel) { - super(factory, treeLevel, codeBlockLevel); + super(factory, ast); this.treeLevel = treeLevel; + this.codeBlockLevel = codeBlockLevel; } + + /** Add local var decl */ + public void addLocalDecl(Decl d) { + if ( locals==null ) locals = new OrderedHashSet(); + locals.add(d); + d.isLocal = true; + } + + public int getEnclosingTreeLevel() { return treeLevel - 1; } } diff --git a/tool/src/org/antlr/v4/codegen/model/ast/TreeRewrite.java b/tool/src/org/antlr/v4/codegen/model/ast/TreeRewrite.java index b77cabd37..b8b7af837 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/TreeRewrite.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/TreeRewrite.java @@ -31,14 +31,9 @@ package org.antlr.v4.codegen.model.ast; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.model.Rewrite; -import org.antlr.v4.codegen.model.decl.RootDecl; public class TreeRewrite extends Rewrite { - public RootDecl rootDecl; - public TreeRewrite(OutputModelFactory factory, int treeLevel, int codeBlockLevel) { super(factory, treeLevel, codeBlockLevel); - rootDecl = new RootDecl(factory); - addLocalDecl(rootDecl); } } diff --git a/tool/src/org/antlr/v4/codegen/model/decl/RewriteIteratorDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/RewriteIteratorDecl.java index 6f987b913..02e217580 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/RewriteIteratorDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/RewriteIteratorDecl.java @@ -35,10 +35,11 @@ import org.antlr.v4.tool.GrammarAST; public class RewriteIteratorDecl extends Decl { public String listName; public RewriteIteratorDecl(OutputModelFactory factory, - GrammarAST elem) + GrammarAST elem, + int codeBlockLevel) { super(factory, factory.getGenerator().target - .getRewriteIteratorName(elem, factory.getCodeBlockLevel())); + .getRewriteIteratorName(elem, codeBlockLevel)); listName = factory.getGenerator().target.getElementListName(elem); } } diff --git a/tool/src/org/antlr/v4/codegen/model/decl/RootDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/RootDecl.java index 9c234437e..6ba641220 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/RootDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/RootDecl.java @@ -32,8 +32,7 @@ package org.antlr.v4.codegen.model.decl; import org.antlr.v4.codegen.OutputModelFactory; public class RootDecl extends Decl { - public RootDecl(OutputModelFactory factory) { - super(factory, - factory.getGenerator().target.getRootName(factory.getCodeBlockLevel())); + public RootDecl(OutputModelFactory factory, int treeLevel) { + super(factory, factory.getGenerator().target.getRootName(treeLevel)); } } diff --git a/tool/src/org/antlr/v4/parse/ANTLRParser.g b/tool/src/org/antlr/v4/parse/ANTLRParser.g index f097af925..b98369211 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRParser.g +++ b/tool/src/org/antlr/v4/parse/ANTLRParser.g @@ -884,7 +884,12 @@ rewriteTreeEbnf $rewriteTreeEbnf.tree.getToken().setLine(firstToken.getLine()); $rewriteTreeEbnf.tree.getToken().setCharPositionInLine(firstToken.getCharPositionInLine()); } - : lp=LPAREN rewriteTreeAlt RPAREN ebnfSuffix -> ^(ebnfSuffix ^(REWRITE_BLOCK[$lp] rewriteTreeAlt)) + : lp=LPAREN rewriteTreeAlt RPAREN rewriteEbnfSuffix -> ^(rewriteEbnfSuffix ^(REWRITE_BLOCK[$lp] rewriteTreeAlt)) + ; + +rewriteEbnfSuffix + : OPTIONAL + | CLOSURE ; rewriteTree diff --git a/tool/src/org/antlr/v4/semantics/Refs.g b/tool/src/org/antlr/v4/semantics/Refs.g new file mode 100644 index 000000000..fcaac182e --- /dev/null +++ b/tool/src/org/antlr/v4/semantics/Refs.g @@ -0,0 +1,87 @@ +tree grammar Refs; +options { + language = Java; + tokenVocab = ANTLRParser; + ASTLabelType = GrammarAST; +} + +@header { +package org.antlr.v4.semantics; +import org.antlr.v4.misc.Utils; +import org.antlr.v4.tool.*; +import java.util.List; +import java.util.ArrayList; +} + +@members { +List shallow = new ArrayList(); +List deep = new ArrayList(); +public int desiredShallowLevel; + +public Refs(TreeNodeStream input, int desiredShallowLevel) { + this(input); + this.desiredShallowLevel = desiredShallowLevel; +} + +public void track(GrammarAST t, int level) { + deep.add(t); + if ( level==desiredShallowLevel ) shallow.add(t); +} + +// TODO: visitor would be better here +} + +/* +rewrite + : predicatedRewrite* nakedRewrite + ; + +predicatedRewrite + : ^(RESULT SEMPRED rewriteAlt) + ; + +nakedRewrite + : ^(RESULT rewriteAlt) + ; + + */ + +start + : ^(RESULT rewriteAlt) + | rewriteTreeEbnf[0] + ; + +rewriteAlt + : rewriteTreeAlt[0] + | ETC + | EPSILON + ; + +rewriteTreeAlt[int level] + : ^(ALT rewriteTreeElement[level]+) + ; + +rewriteTreeElement[int level] + : rewriteTreeAtom[level] + | rewriteTree[level] + | rewriteTreeEbnf[level] + ; + +rewriteTreeAtom[int level] + : ^(TOKEN_REF . .) {track($start, level);} + | ^(TOKEN_REF .) {track($start, level);} + | TOKEN_REF {track($start, level);} + | RULE_REF {track($start, level);} + | ^(STRING_LITERAL .) {track($start, level);} + | STRING_LITERAL {track($start, level);} + | LABEL {track($start, level);} + | ACTION + ; + +rewriteTreeEbnf[int level] + : ^((OPTIONAL | CLOSURE) ^(REWRITE_BLOCK rewriteTreeAlt[level+1])) + ; + +rewriteTree[int level] + : ^(TREE_BEGIN rewriteTreeAtom[level] rewriteTreeElement[level]* ) + ; diff --git a/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java b/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java index 6bef8a2f1..c105a8bcb 100644 --- a/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java +++ b/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java @@ -29,6 +29,7 @@ package org.antlr.v4.semantics; +import org.antlr.runtime.tree.CommonTreeNodeStream; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.Token; import org.antlr.v4.tool.*; @@ -42,11 +43,12 @@ public class UseDefAnalyzer { for (int a=1; a<=r.numberOfAlts; a++) { Alternative alt = r.alt[a]; for (GrammarAST e : alt.rewriteElements) { - if ( !(alt.ruleRefs.containsKey(e.getText()) || - g.getTokenType(e.getText())!= Token.INVALID_TYPE || - alt.labelDefs.containsKey(e.getText()) || - e.getText().equals(r.name)) ) // $r ok in rule r - { + boolean ok = + alt.ruleRefs.containsKey(e.getText()) || + g.getTokenType(e.getText()) != Token.INVALID_TYPE || + alt.labelDefs.containsKey(e.getText()) || + e.getText().equals(r.name); + if ( !ok ) { // $r ok in rule r g.tool.errMgr.grammarError(ErrorType.REWRITE_ELEMENT_NOT_PRESENT_ON_LHS, g.fileName, e.token, e.getText()); } @@ -68,6 +70,61 @@ public class UseDefAnalyzer { } } + /** Given -> (ALT ...), return list of element refs at + * top level + */ + public static List getElementReferencesShallowInOuterAlt(Grammar g, + GrammarAST altAST) + { + return UseDefAnalyzer.getRewriteElementRefs(g, altAST, 0, false); + } + + /** Given (('?'|'*') (REWRITE_BLOCK (ALT ...))) return list of element refs at + * top level of REWRITE_BLOCK. Must see into (nested) tree structures if + * optional but not if closure (that might lead to inf loop when building tree). + */ + public static List getElementReferencesInEBNF(Grammar g, + GrammarAST ebnfRoot, + boolean deep) + { + return UseDefAnalyzer.getRewriteElementRefs(g, ebnfRoot, 1, deep); + } + + public static List filterForRuleAndTokenRefs(Alternative alt, + List refs) + { + List elems = new ArrayList(); + if ( refs!=null ) { + for (GrammarAST ref : refs) { + boolean imaginary = ref.getType()== ANTLRParser.TOKEN_REF && + !alt.tokenRefs.containsKey(ref.getText()); + if ( !imaginary ) elems.add(ref); + } + } + return elems; + } + + public static List getRewriteElementRefs(Grammar g, + GrammarAST root, + int desiredShallowLevel, + boolean deep) + { + CommonTreeNodeStream nodes = new CommonTreeNodeStream(root); + Refs collector = new Refs(nodes, desiredShallowLevel); + try { + collector.start(); + } + catch (org.antlr.runtime.RecognitionException re) { + g.tool.errMgr.grammarError(ErrorType.INTERNAL_ERROR, + g.fileName, re.token, re); + + } +// System.out.println("from "+root.toStringTree()); +// System.out.println("shallow: "+collector.shallow); +// System.out.println("deep: "+collector.deep); + return deep ? collector.deep : collector.shallow; + } + /** Find all rules reachable from r directly or indirectly for all r in g */ public static Map> getRuleDependencies(Grammar g) { return getRuleDependencies(g, g.rules.values());