From 83411ef056c5a2ce4fcca83584cc922c3ddf8224 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Thu, 10 Jul 2014 18:00:16 -0700 Subject: [PATCH 1/4] add test --- .../antlr/v4/test/TestATNConstruction.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tool/test/org/antlr/v4/test/TestATNConstruction.java b/tool/test/org/antlr/v4/test/TestATNConstruction.java index 2d836a7fd..9a01590cd 100644 --- a/tool/test/org/antlr/v4/test/TestATNConstruction.java +++ b/tool/test/org/antlr/v4/test/TestATNConstruction.java @@ -28,6 +28,8 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.test; + +import org.antlr.v4.Tool; import org.antlr.v4.automata.ATNPrinter; import org.antlr.v4.automata.LexerATNFactory; import org.antlr.v4.automata.ParserATNFactory; @@ -35,9 +37,11 @@ import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.ast.GrammarRootAST; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class TestATNConstruction extends BaseTest { @Test @@ -390,6 +394,30 @@ public class TestATNConstruction extends BaseTest { "RuleStop_a_1-EOF->s8\n"; checkRuleATN(g, "a", expecting); } + + @Test public void testParserRuleRefInLexerRule() throws Exception { + boolean threwException = false; + try { + String gstr = + "lexer grammar U;\n"+ + "A : a;"; + + Tool tool = new Tool(); + tool.removeListeners(); + ErrorQueue errorQueue = new ErrorQueue(); + tool.addListener(errorQueue); + System.out.println("errors:"+errorQueue.errors); + System.out.println("warns:"+errorQueue.warnings); + GrammarRootAST grammarRootAST = tool.parseGrammarFromString(gstr); + Grammar g = tool.createGrammar(grammarRootAST); + tool.process(g, false); + } + catch (Exception e) { + threwException = true; + } + assertTrue(!threwException); + } + /* @Test public void testMultiplePredicates() throws Exception { Grammar g = new Grammar( From c22706135d8cc1b82d81135422e3bcec9102df2a Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Thu, 10 Jul 2014 18:07:24 -0700 Subject: [PATCH 2/4] fix build. separate antlr3,4 gen dirs --- bild.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/bild.py b/bild.py index 7bf4f5a49..03c36107f 100755 --- a/bild.py +++ b/bild.py @@ -4,6 +4,7 @@ import string # bootstrap by downloading bilder.py if not found import urllib +import os if not os.path.exists("bilder.py"): print "bootstrapping; downloading bilder.py" @@ -34,20 +35,20 @@ TARGETS = {"Java":uniformpath(JAVA_TARGET), "CSharp":uniformpath(CSHARP_TARGET)} def parsers(): - antlr3("tool/src/org/antlr/v4/parse", "gen", package="org.antlr.v4.parse") - antlr3("tool/src/org/antlr/v4/codegen", "gen", package="org.antlr.v4.codegen", args=["-lib","tool/src/org/antlr/v4/parse"]) - antlr4("runtime/Java/src/org/antlr/v4/runtime/tree/xpath", "gen", package="org.antlr.v4.runtime.tree.xpath") + antlr3("tool/src/org/antlr/v4/parse", "gen3", package="org.antlr.v4.parse") + antlr3("tool/src/org/antlr/v4/codegen", "gen3", package="org.antlr.v4.codegen", args=["-lib",uniformpath("gen3/org/antlr/v4/parse")]) + antlr4("runtime/Java/src/org/antlr/v4/runtime/tree/xpath", "gen4", package="org.antlr.v4.runtime.tree.xpath") def compile(): require(parsers) cp = uniformpath("out")+os.pathsep+ \ os.path.join(JARCACHE,"antlr-3.5.1-complete.jar")+os.pathsep+ \ - "runtime/Java/lib/org.abego.treelayout.core.jar" - args = ["-Xlint", "-Xlint:-serial", "-g"] - javac("runtime/JavaAnnotations/src/", "out", version="1.6", cp=cp, args=args) - javac("runtime/Java/src", "out", version="1.6", cp=cp, args=args) - javac("tool/src", "out", version="1.6", cp=cp, args=args) - javac("gen", "out", version="1.6", cp=cp, args=args) + "runtime/Java/lib/org.abego.treelayout.core.jar"+os.pathsep+ \ + JARCACHE+"/antlr-4.4-complete.jar" + srcpath = ["gen3", "gen4", "runtime/JavaAnnotations/src", "runtime/Java/src", "tool/src"] + args = ["-Xlint", "-Xlint:-serial", "-g", "-sourcepath", string.join(srcpath, os.pathsep)] + for sp in srcpath: + javac(sp, "out", version="1.6", cp=cp, args=args) # pull in targets for t in TARGETS: javac(TARGETS[t]+"/tool/src", "out", version="1.6", cp=cp, args=args) @@ -82,8 +83,10 @@ def mkruntimejar(): cp = uniformpath("out/runtime")+os.pathsep+ \ "runtime/Java/lib/org.abego.treelayout.core.jar" args = ["-Xlint", "-Xlint:-serial", "-g"] - javac("runtime/JavaAnnotations/src/", "out/runtime", version="1.6", cp=cp, args=args) - javac("runtime/Java/src", "out/runtime", version="1.6", cp=cp, args=args) + srcpath = ["runtime/JavaAnnotations/src", "runtime/Java/src"] + args = ["-Xlint", "-Xlint:-serial", "-g", "-sourcepath", string.join(srcpath, os.pathsep)] + for sp in srcpath: + javac(sp, "out", version="1.6", cp=cp, args=args) manifest = \ """Implementation-Vendor: ANTLR Implementation-Title: ANTLR 4 Runtime @@ -119,7 +122,8 @@ def all(): def clean(): rmdir("out") - rmdir("gen") + rmdir("gen3") + rmdir("gen4") rmdir("doc") def mkdoc(): From ee0345c1e223abe330448c6b9824aab18572afd3 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Thu, 10 Jul 2014 21:44:54 -0700 Subject: [PATCH 3/4] After gen of antlr 4.4 jar, rebuild XPathLexer with 4.4 then remake jars. --- bild.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/bild.py b/bild.py index 03c36107f..2e53fc06f 100755 --- a/bild.py +++ b/bild.py @@ -53,7 +53,7 @@ def compile(): for t in TARGETS: javac(TARGETS[t]+"/tool/src", "out", version="1.6", cp=cp, args=args) -def mkjar(): +def mkjar_complete(): require(compile) copytree(src="tool/resources", trg="out") # messages, Java code gen, etc... manifest = \ @@ -74,11 +74,11 @@ Created-By: http://www.bildtool.org mkdir(trgdir) copyfile(TARGETS[t]+"/tool/resources/org/antlr/v4/tool/templates/codegen/"+t+"/"+t+".stg", trgdir) - jar("dist/antlr-"+VERSION+"-complete.jar", srcdir="out", manifest=manifest) + jarfile = "dist/antlr-"+VERSION+"-complete.jar" + jar(jarfile, srcdir="out", manifest=manifest) + print "Generated "+jarfile - mkruntimejar() - -def mkruntimejar(): +def mkjar_runtime(): # out/... dir is full of tool-related stuff, make special dir out/runtime cp = uniformpath("out/runtime")+os.pathsep+ \ "runtime/Java/lib/org.abego.treelayout.core.jar" @@ -97,8 +97,20 @@ Created-By: http://www.bildtool.org """ % (VERSION,os.getlogin()) # unjar required library unjar("runtime/Java/lib/org.abego.treelayout.core.jar", trgdir="out/runtime") - jar("dist/antlr-runtime-"+VERSION+".jar", srcdir="out/runtime", manifest=manifest) + jarfile = "dist/antlr-runtime-" + VERSION + ".jar" + jar(jarfile, srcdir="out/runtime", manifest=manifest) + print "Generated "+jarfile +def mkjar(): + mkjar_complete() + # put it in JARCARCHE too so bild can find it during antlr4() + copyfile(src="dist/antlr-4.4-complete.jar", trg=JARCACHE) + # rebuild/bootstrap XPath with 4.4 so it can use 4.4 runtime (gen'd with 4.3 at this point) + rmdir("gen4/org/antlr/v4/runtime/tree/xpath") # kill 4.3-generated version + antlr4("runtime/Java/src/org/antlr/v4/runtime/tree/xpath", "gen4", version="4.4", package="org.antlr.v4.runtime.tree.xpath") + compile() + mkjar_complete() # make it again with up to date XPath lexer + mkjar_runtime() # now build the runtime jar def tests(): require(mkjar) From bb389782da9fe9a9a1c56c31a5af4f61c39c31fb Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Fri, 11 Jul 2014 12:09:47 -0700 Subject: [PATCH 4/4] Fixes #661. A parser rule reference within a lexer rule would get a bad error message and then an exception. Further, errors within the grammar tree visitor would go to standard error instead of to an error manager because none of the subclasses implemented getErrorManager(). Introduced an error type for this error PARSER_RULE_REF_IN_LEXER_RULE. Added to unit test. --- tool/src/org/antlr/v4/Tool.java | 21 ++- .../model/ElementFrequenciesVisitor.java | 166 ++++++++++++++++++ .../antlr/v4/codegen/model/RuleFunction.java | 153 ---------------- .../org/antlr/v4/parse/GrammarTreeVisitor.g | 5 +- .../v4/semantics/BasicSemanticChecks.java | 7 +- .../org/antlr/v4/semantics/RuleCollector.java | 10 +- .../antlr/v4/semantics/SymbolCollector.java | 11 +- tool/src/org/antlr/v4/tool/ErrorType.java | 6 + tool/src/org/antlr/v4/tool/Grammar.java | 3 +- .../antlr/v4/test/TestATNConstruction.java | 21 ++- 10 files changed, 234 insertions(+), 169 deletions(-) create mode 100644 tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index ae13dbc0a..e181e04f3 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -434,7 +434,8 @@ public class Tool { * Important enough to avoid multiple definitions that we do very early, * right after AST construction. Also check for undefined rules in * parser/lexer to avoid exceptions later. Return true if we find multiple - * definitions of the same rule or a reference to an undefined rule. + * definitions of the same rule or a reference to an undefined rule or + * parser rule ref in lexer rule. */ public boolean checkForRuleIssues(final Grammar g) { // check for redefined rules @@ -466,7 +467,7 @@ public class Tool { // check for undefined rules class UndefChecker extends GrammarTreeVisitor { - public boolean undefined = false; + public boolean badref = false; @Override public void tokenRef(TerminalAST ref) { if ("EOF".equals(ref.getText())) { @@ -480,18 +481,28 @@ public class Tool { @Override public void ruleRef(GrammarAST ref, ActionAST arg) { RuleAST ruleAST = ruleToAST.get(ref.getText()); - if ( ruleAST==null ) { - undefined = true; + if (Character.isUpperCase(currentRuleName.charAt(0)) && + Character.isLowerCase(ref.getText().charAt(0))) + { + badref = true; + String fileName = ref.getToken().getInputStream().getSourceName(); + errMgr.grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE, + fileName, ref.getToken(), ref.getText(), currentRuleName); + } + else if ( ruleAST==null ) { + badref = true; errMgr.grammarError(ErrorType.UNDEFINED_RULE_REF, g.fileName, ref.token, ref.getText()); } } + @Override + public ErrorManager getErrorManager() { return errMgr; } } UndefChecker chk = new UndefChecker(); chk.visitGrammar(g.ast); - return redefinition || chk.undefined; + return redefinition || chk.badref; } public List sortGrammarByTokenVocab(List fileNames) { diff --git a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java new file mode 100644 index 000000000..2226d3778 --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java @@ -0,0 +1,166 @@ +package org.antlr.v4.codegen.model; + +import org.antlr.runtime.tree.TreeNodeStream; +import org.antlr.v4.misc.FrequencySet; +import org.antlr.v4.misc.MutableInt; +import org.antlr.v4.parse.GrammarTreeVisitor; +import org.antlr.v4.tool.ErrorManager; +import org.antlr.v4.tool.ast.ActionAST; +import org.antlr.v4.tool.ast.AltAST; +import org.antlr.v4.tool.ast.GrammarAST; +import org.antlr.v4.tool.ast.TerminalAST; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Map; + +public class ElementFrequenciesVisitor extends GrammarTreeVisitor { + final Deque> frequencies; + + public ElementFrequenciesVisitor(TreeNodeStream input) { + super(input); + frequencies = new ArrayDeque>(); + frequencies.push(new FrequencySet()); + } + + /** During code gen, we can assume tree is in good shape */ + @Override + public ErrorManager getErrorManager() { return super.getErrorManager(); } + + /* + * Common + */ + + /** + * Generate a frequency set as the union of two input sets. If an + * element is contained in both sets, the value for the output will be + * the maximum of the two input values. + * + * @param a The first set. + * @param b The second set. + * @return The union of the two sets, with the maximum value chosen + * whenever both sets contain the same key. + */ + protected static FrequencySet combineMax(FrequencySet a, FrequencySet b) { + FrequencySet result = combineAndClip(a, b, 1); + for (Map.Entry entry : a.entrySet()) { + result.get(entry.getKey()).v = entry.getValue().v; + } + + for (Map.Entry entry : b.entrySet()) { + MutableInt slot = result.get(entry.getKey()); + slot.v = Math.max(slot.v, entry.getValue().v); + } + + return result; + } + + /** + * Generate a frequency set as the union of two input sets, with the + * values clipped to a specified maximum value. If an element is + * contained in both sets, the value for the output, prior to clipping, + * will be the sum of the two input values. + * + * @param a The first set. + * @param b The second set. + * @param clip The maximum value to allow for any output. + * @return The sum of the two sets, with the individual elements clipped + * to the maximum value gived by {@code clip}. + */ + protected static FrequencySet combineAndClip(FrequencySet a, FrequencySet b, int clip) { + FrequencySet result = new FrequencySet(); + for (Map.Entry entry : a.entrySet()) { + for (int i = 0; i < entry.getValue().v; i++) { + result.add(entry.getKey()); + } + } + + for (Map.Entry entry : b.entrySet()) { + for (int i = 0; i < entry.getValue().v; i++) { + result.add(entry.getKey()); + } + } + + for (Map.Entry entry : result.entrySet()) { + entry.getValue().v = Math.min(entry.getValue().v, clip); + } + + return result; + } + + @Override + public void tokenRef(TerminalAST ref) { + frequencies.peek().add(ref.getText()); + } + + @Override + public void ruleRef(GrammarAST ref, ActionAST arg) { + frequencies.peek().add(ref.getText()); + } + + /* + * Parser rules + */ + + @Override + protected void enterAlternative(AltAST tree) { + frequencies.push(new FrequencySet()); + } + + @Override + protected void exitAlternative(AltAST tree) { + frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); + } + + @Override + protected void enterElement(GrammarAST tree) { + frequencies.push(new FrequencySet()); + } + + @Override + protected void exitElement(GrammarAST tree) { + frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + } + + @Override + protected void exitSubrule(GrammarAST tree) { + if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) { + for (Map.Entry entry : frequencies.peek().entrySet()) { + entry.getValue().v = 2; + } + } + } + + /* + * Lexer rules + */ + + @Override + protected void enterLexerAlternative(GrammarAST tree) { + frequencies.push(new FrequencySet()); + } + + @Override + protected void exitLexerAlternative(GrammarAST tree) { + frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); + } + + @Override + protected void enterLexerElement(GrammarAST tree) { + frequencies.push(new FrequencySet()); + } + + @Override + protected void exitLexerElement(GrammarAST tree) { + frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + } + + @Override + protected void exitLexerSubrule(GrammarAST tree) { + if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) { + for (Map.Entry entry : frequencies.peek().entrySet()) { + entry.getValue().v = 2; + } + } + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java index 22aedd98d..eeef661ac 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java @@ -32,7 +32,6 @@ package org.antlr.v4.codegen.model; import org.antlr.runtime.RecognitionException; import org.antlr.runtime.tree.CommonTreeNodeStream; -import org.antlr.runtime.tree.TreeNodeStream; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.model.decl.AltLabelStructDecl; import org.antlr.v4.codegen.model.decl.AttributeDecl; @@ -45,10 +44,8 @@ import org.antlr.v4.codegen.model.decl.ContextTokenListIndexedGetterDecl; import org.antlr.v4.codegen.model.decl.Decl; import org.antlr.v4.codegen.model.decl.StructDecl; import org.antlr.v4.misc.FrequencySet; -import org.antlr.v4.misc.MutableInt; import org.antlr.v4.misc.Utils; import org.antlr.v4.parse.GrammarASTAdaptor; -import org.antlr.v4.parse.GrammarTreeVisitor; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.OrderedHashSet; @@ -59,12 +56,9 @@ import org.antlr.v4.tool.Rule; import org.antlr.v4.tool.ast.ActionAST; import org.antlr.v4.tool.ast.AltAST; import org.antlr.v4.tool.ast.GrammarAST; -import org.antlr.v4.tool.ast.TerminalAST; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; -import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -292,151 +286,4 @@ public class RuleFunction extends OutputModelObject { } ruleCtx.addDecl(d); // stick in overall rule's ctx } - - protected static class ElementFrequenciesVisitor extends GrammarTreeVisitor { - final Deque> frequencies; - - public ElementFrequenciesVisitor(TreeNodeStream input) { - super(input); - frequencies = new ArrayDeque>(); - frequencies.push(new FrequencySet()); - } - - /* - * Common - */ - - /** - * Generate a frequency set as the union of two input sets. If an - * element is contained in both sets, the value for the output will be - * the maximum of the two input values. - * - * @param a The first set. - * @param b The second set. - * @return The union of the two sets, with the maximum value chosen - * whenever both sets contain the same key. - */ - protected static FrequencySet combineMax(FrequencySet a, FrequencySet b) { - FrequencySet result = combineAndClip(a, b, 1); - for (Map.Entry entry : a.entrySet()) { - result.get(entry.getKey()).v = entry.getValue().v; - } - - for (Map.Entry entry : b.entrySet()) { - MutableInt slot = result.get(entry.getKey()); - slot.v = Math.max(slot.v, entry.getValue().v); - } - - return result; - } - - /** - * Generate a frequency set as the union of two input sets, with the - * values clipped to a specified maximum value. If an element is - * contained in both sets, the value for the output, prior to clipping, - * will be the sum of the two input values. - * - * @param a The first set. - * @param b The second set. - * @param clip The maximum value to allow for any output. - * @return The sum of the two sets, with the individual elements clipped - * to the maximum value gived by {@code clip}. - */ - protected static FrequencySet combineAndClip(FrequencySet a, FrequencySet b, int clip) { - FrequencySet result = new FrequencySet(); - for (Map.Entry entry : a.entrySet()) { - for (int i = 0; i < entry.getValue().v; i++) { - result.add(entry.getKey()); - } - } - - for (Map.Entry entry : b.entrySet()) { - for (int i = 0; i < entry.getValue().v; i++) { - result.add(entry.getKey()); - } - } - - for (Map.Entry entry : result.entrySet()) { - entry.getValue().v = Math.min(entry.getValue().v, clip); - } - - return result; - } - - @Override - public void tokenRef(TerminalAST ref) { - frequencies.peek().add(ref.getText()); - } - - @Override - public void ruleRef(GrammarAST ref, ActionAST arg) { - frequencies.peek().add(ref.getText()); - } - - /* - * Parser rules - */ - - @Override - protected void enterAlternative(AltAST tree) { - frequencies.push(new FrequencySet()); - } - - @Override - protected void exitAlternative(AltAST tree) { - frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); - } - - @Override - protected void enterElement(GrammarAST tree) { - frequencies.push(new FrequencySet()); - } - - @Override - protected void exitElement(GrammarAST tree) { - frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); - } - - @Override - protected void exitSubrule(GrammarAST tree) { - if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) { - for (Map.Entry entry : frequencies.peek().entrySet()) { - entry.getValue().v = 2; - } - } - } - - /* - * Lexer rules - */ - - @Override - protected void enterLexerAlternative(GrammarAST tree) { - frequencies.push(new FrequencySet()); - } - - @Override - protected void exitLexerAlternative(GrammarAST tree) { - frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); - } - - @Override - protected void enterLexerElement(GrammarAST tree) { - frequencies.push(new FrequencySet()); - } - - @Override - protected void exitLexerElement(GrammarAST tree) { - frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); - } - - @Override - protected void exitLexerSubrule(GrammarAST tree) { - if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) { - for (Map.Entry entry : frequencies.peek().entrySet()) { - entry.getValue().v = 2; - } - } - } - } } diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index cca2f627a..13a086823 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -76,6 +76,7 @@ options { THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.parse; +import org.antlr.v4.Tool; import org.antlr.v4.tool.*; import org.antlr.v4.tool.ast.*; import java.lang.reflect.Method; @@ -86,13 +87,14 @@ public String grammarName; public GrammarAST currentRuleAST; public String currentModeName = LexerGrammar.DEFAULT_MODE_NAME; public String currentRuleName; -//public GrammarAST currentRuleBlock; public GrammarAST currentOuterAltRoot; public int currentOuterAltNumber = 1; // 1..n public int rewriteEBNFLevel = 0; public GrammarTreeVisitor() { this(null); } +// Should be abstract but can't make gen'd parser abstract; +// subclasses should implement else everything goes to stderr! public ErrorManager getErrorManager() { return null; } public void visitGrammar(GrammarAST t) { visit(t, "grammarSpec"); } @@ -742,6 +744,7 @@ lexerAtom | WILDCARD | LEXER_CHAR_SET | range + | ruleref ; actionElement diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index ee52d603c..3f6ed9811 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -130,6 +130,9 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { this.errMgr = g.tool.errMgr; } + @Override + public ErrorManager getErrorManager() { return errMgr; } + public void process() { visitGrammar(g.ast); } // Routines to route visitor traffic to the checking routines @@ -376,8 +379,8 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { void checkInvalidRuleRef(Token ruleID) { String fileName = ruleID.getInputStream().getSourceName(); if ( g.isLexer() && Character.isLowerCase(ruleID.getText().charAt(0)) ) { - g.tool.errMgr.grammarError(ErrorType.PARSER_RULES_NOT_ALLOWED, - fileName, ruleID, ruleID.getText()); + g.tool.errMgr.grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE, + fileName, ruleID, ruleID.getText(), currentRuleName); } } diff --git a/tool/src/org/antlr/v4/semantics/RuleCollector.java b/tool/src/org/antlr/v4/semantics/RuleCollector.java index 17cae5415..fa7eb2d31 100644 --- a/tool/src/org/antlr/v4/semantics/RuleCollector.java +++ b/tool/src/org/antlr/v4/semantics/RuleCollector.java @@ -36,6 +36,7 @@ import org.antlr.v4.misc.Utils; import org.antlr.v4.parse.GrammarTreeVisitor; import org.antlr.v4.parse.ScopeParser; import org.antlr.v4.tool.AttributeDict; +import org.antlr.v4.tool.ErrorManager; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LeftRecursiveRule; import org.antlr.v4.tool.Rule; @@ -52,13 +53,20 @@ import java.util.Map; public class RuleCollector extends GrammarTreeVisitor { /** which grammar are we checking */ public Grammar g; + public ErrorManager errMgr; // stuff to collect. this is the output public OrderedHashMap rules = new OrderedHashMap(); public MultiMap ruleToAltLabels = new MultiMap(); public Map altLabelToRuleName = new HashMap(); - public RuleCollector(Grammar g) { this.g = g; } + public RuleCollector(Grammar g) { + this.g = g; + this.errMgr = g.tool.errMgr; + } + + @Override + public ErrorManager getErrorManager() { return errMgr; } public void process(GrammarAST ast) { visitGrammar(ast); } diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index fd2cad13c..cbe2a0f28 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -31,6 +31,7 @@ package org.antlr.v4.semantics; import org.antlr.v4.parse.GrammarTreeVisitor; +import org.antlr.v4.tool.ErrorManager; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LabelElementPair; import org.antlr.v4.tool.Rule; @@ -68,10 +69,18 @@ public class SymbolCollector extends GrammarTreeVisitor { /** Track action name node in @parser::members {...} or @members {...} */ List namedActions = new ArrayList(); + public ErrorManager errMgr; + // context public Rule currentRule; - public SymbolCollector(Grammar g) { this.g = g; } + public SymbolCollector(Grammar g) { + this.g = g; + this.errMgr = g.tool.errMgr; + } + + @Override + public ErrorManager getErrorManager() { return errMgr; } public void process(GrammarAST ast) { visitGrammar(ast); } diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index 2f2a1953c..59603f32d 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -224,6 +224,12 @@ public enum ErrorType { *

reference to undefined rule: rule

*/ UNDEFINED_RULE_REF(56, "reference to undefined rule: ", ErrorSeverity.ERROR), + /** + * Compiler Error 160. + * + *

reference to undefined rule: rule

+ */ + PARSER_RULE_REF_IN_LEXER_RULE(160, "reference to parser rule in lexer rule ", ErrorSeverity.ERROR), /** * Compiler Error 57. * diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index a716ebc72..c68ccc2bd 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -30,7 +30,6 @@ package org.antlr.v4.tool; -import org.antlr.runtime.tree.Tree; import org.antlr.v4.Tool; import org.antlr.v4.analysis.LeftRecursiveRuleTransformer; import org.antlr.v4.misc.CharSupport; @@ -1063,6 +1062,8 @@ public class Grammar implements AttributeResolver { public void stringRef(TerminalAST ref) { strings.add(ref.getText()); } + @Override + public ErrorManager getErrorManager() { return tool.errMgr; } }; collector.visitGrammar(ast); return strings; diff --git a/tool/test/org/antlr/v4/test/TestATNConstruction.java b/tool/test/org/antlr/v4/test/TestATNConstruction.java index 9a01590cd..2b973f839 100644 --- a/tool/test/org/antlr/v4/test/TestATNConstruction.java +++ b/tool/test/org/antlr/v4/test/TestATNConstruction.java @@ -35,11 +35,14 @@ import org.antlr.v4.automata.LexerATNFactory; import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; +import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.ast.GrammarRootAST; import org.junit.Test; +import java.util.Arrays; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -397,24 +400,32 @@ public class TestATNConstruction extends BaseTest { @Test public void testParserRuleRefInLexerRule() throws Exception { boolean threwException = false; + ErrorQueue errorQueue = new ErrorQueue(); try { String gstr = - "lexer grammar U;\n"+ - "A : a;"; + "grammar U;\n"+ + "a : A;\n"+ + "A : a;\n"; Tool tool = new Tool(); tool.removeListeners(); - ErrorQueue errorQueue = new ErrorQueue(); tool.addListener(errorQueue); - System.out.println("errors:"+errorQueue.errors); - System.out.println("warns:"+errorQueue.warnings); + assertEquals(0, errorQueue.size()); GrammarRootAST grammarRootAST = tool.parseGrammarFromString(gstr); + assertEquals(0, errorQueue.size()); Grammar g = tool.createGrammar(grammarRootAST); + assertEquals(0, errorQueue.size()); + g.fileName = ""; tool.process(g, false); } catch (Exception e) { threwException = true; + e.printStackTrace(); } + System.out.println(errorQueue); + assertEquals(1, errorQueue.errors.size()); + assertEquals(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE, errorQueue.errors.get(0).getErrorType()); + assertEquals("[a, A]", Arrays.toString(errorQueue.errors.get(0).getArgs())); assertTrue(!threwException); }