forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Fixes #661. A parser rule reference within a lexer rule would get a bad error message and then an exception. Further, errors within the grammar tree visitor would go to standard error instead of to an error manager because none of the subclasses implemented getErrorManager(). Introduced an error type for this error PARSER_RULE_REF_IN_LEXER_RULE. Added to unit test.

This commit is contained in:
Terence Parr 2014-07-11 12:09:47 -07:00
parent b4a53d56d4
commit bb389782da
10 changed files with 234 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tool/src/org/antlr/v4/Tool.java
View File

@ -434,7 +434,8 @@ public class Tool {
* Important enough to avoid multiple definitions that we do very early,
* right after AST construction. Also check for undefined rules in
* parser/lexer to avoid exceptions later. Return true if we find multiple
* definitions of the same rule or a reference to an undefined rule.
* definitions of the same rule or a reference to an undefined rule or
* parser rule ref in lexer rule.
*/
public boolean checkForRuleIssues(final Grammar g) {
// check for redefined rules
@ -466,7 +467,7 @@ public class Tool {
// check for undefined rules
class UndefChecker extends GrammarTreeVisitor {
public boolean undefined = false;
public boolean badref = false;
@Override
public void tokenRef(TerminalAST ref) {
if ("EOF".equals(ref.getText())) {
@ -480,18 +481,28 @@ public class Tool {
@Override
public void ruleRef(GrammarAST ref, ActionAST arg) {
RuleAST ruleAST = ruleToAST.get(ref.getText());
if ( ruleAST==null ) {
undefined = true;
if (Character.isUpperCase(currentRuleName.charAt(0)) &&
Character.isLowerCase(ref.getText().charAt(0)))
{
badref = true;
String fileName = ref.getToken().getInputStream().getSourceName();
errMgr.grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE,
fileName, ref.getToken(), ref.getText(), currentRuleName);
}
else if ( ruleAST==null ) {
badref = true;
errMgr.grammarError(ErrorType.UNDEFINED_RULE_REF,
g.fileName, ref.token, ref.getText());
}
}
@Override
public ErrorManager getErrorManager() { return errMgr; }
}
UndefChecker chk = new UndefChecker();
chk.visitGrammar(g.ast);
return redefinition || chk.undefined;
return redefinition || chk.badref;
}
public List<GrammarRootAST> sortGrammarByTokenVocab(List<String> fileNames) {

166
tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java Normal file
View File

@ -0,0 +1,166 @@
package org.antlr.v4.codegen.model;
import org.antlr.runtime.tree.TreeNodeStream;
import org.antlr.v4.misc.FrequencySet;
import org.antlr.v4.misc.MutableInt;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.TerminalAST;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.Map;
public class ElementFrequenciesVisitor extends GrammarTreeVisitor {
final Deque<FrequencySet<String>> frequencies;
public ElementFrequenciesVisitor(TreeNodeStream input) {
super(input);
frequencies = new ArrayDeque<FrequencySet<String>>();
frequencies.push(new FrequencySet<String>());
}
/** During code gen, we can assume tree is in good shape */
@Override
public ErrorManager getErrorManager() { return super.getErrorManager(); }
/*
* Common
*/
/**
* Generate a frequency set as the union of two input sets. If an
* element is contained in both sets, the value for the output will be
* the maximum of the two input values.
*
* @param a The first set.
* @param b The second set.
* @return The union of the two sets, with the maximum value chosen
* whenever both sets contain the same key.
*/
protected static FrequencySet<String> combineMax(FrequencySet<String> a, FrequencySet<String> b) {
FrequencySet<String> result = combineAndClip(a, b, 1);
for (Map.Entry<String, MutableInt> entry : a.entrySet()) {
result.get(entry.getKey()).v = entry.getValue().v;
}
for (Map.Entry<String, MutableInt> entry : b.entrySet()) {
MutableInt slot = result.get(entry.getKey());
slot.v = Math.max(slot.v, entry.getValue().v);
}
return result;
}
/**
* Generate a frequency set as the union of two input sets, with the
* values clipped to a specified maximum value. If an element is
* contained in both sets, the value for the output, prior to clipping,
* will be the sum of the two input values.
*
* @param a The first set.
* @param b The second set.
* @param clip The maximum value to allow for any output.
* @return The sum of the two sets, with the individual elements clipped
* to the maximum value gived by {@code clip}.
*/
protected static FrequencySet<String> combineAndClip(FrequencySet<String> a, FrequencySet<String> b, int clip) {
FrequencySet<String> result = new FrequencySet<String>();
for (Map.Entry<String, MutableInt> entry : a.entrySet()) {
for (int i = 0; i < entry.getValue().v; i++) {
result.add(entry.getKey());
}
}
for (Map.Entry<String, MutableInt> entry : b.entrySet()) {
for (int i = 0; i < entry.getValue().v; i++) {
result.add(entry.getKey());
}
}
for (Map.Entry<String, MutableInt> entry : result.entrySet()) {
entry.getValue().v = Math.min(entry.getValue().v, clip);
}
return result;
}
@Override
public void tokenRef(TerminalAST ref) {
frequencies.peek().add(ref.getText());
}
@Override
public void ruleRef(GrammarAST ref, ActionAST arg) {
frequencies.peek().add(ref.getText());
}
/*
* Parser rules
*/
@Override
protected void enterAlternative(AltAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitAlternative(AltAST tree) {
frequencies.push(combineMax(frequencies.pop(), frequencies.pop()));
}
@Override
protected void enterElement(GrammarAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitElement(GrammarAST tree) {
frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2));
}
@Override
protected void exitSubrule(GrammarAST tree) {
if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) {
for (Map.Entry<String, MutableInt> entry : frequencies.peek().entrySet()) {
entry.getValue().v = 2;
}
}
}
/*
* Lexer rules
*/
@Override
protected void enterLexerAlternative(GrammarAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitLexerAlternative(GrammarAST tree) {
frequencies.push(combineMax(frequencies.pop(), frequencies.pop()));
}
@Override
protected void enterLexerElement(GrammarAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitLexerElement(GrammarAST tree) {
frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2));
}
@Override
protected void exitLexerSubrule(GrammarAST tree) {
if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) {
for (Map.Entry<String, MutableInt> entry : frequencies.peek().entrySet()) {
entry.getValue().v = 2;
}
}
}
}

153
tool/src/org/antlr/v4/codegen/model/RuleFunction.java
View File

@ -32,7 +32,6 @@ package org.antlr.v4.codegen.model;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.TreeNodeStream;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.decl.AltLabelStructDecl;
import org.antlr.v4.codegen.model.decl.AttributeDecl;
@ -45,10 +44,8 @@ import org.antlr.v4.codegen.model.decl.ContextTokenListIndexedGetterDecl;
import org.antlr.v4.codegen.model.decl.Decl;
import org.antlr.v4.codegen.model.decl.StructDecl;
import org.antlr.v4.misc.FrequencySet;
import org.antlr.v4.misc.MutableInt;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.OrderedHashSet;
@ -59,12 +56,9 @@ import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.TerminalAST;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -292,151 +286,4 @@ public class RuleFunction extends OutputModelObject {
}
ruleCtx.addDecl(d); // stick in overall rule's ctx
}
protected static class ElementFrequenciesVisitor extends GrammarTreeVisitor {
final Deque<FrequencySet<String>> frequencies;
public ElementFrequenciesVisitor(TreeNodeStream input) {
super(input);
frequencies = new ArrayDeque<FrequencySet<String>>();
frequencies.push(new FrequencySet<String>());
}
/*
* Common
*/
/**
* Generate a frequency set as the union of two input sets. If an
* element is contained in both sets, the value for the output will be
* the maximum of the two input values.
*
* @param a The first set.
* @param b The second set.
* @return The union of the two sets, with the maximum value chosen
* whenever both sets contain the same key.
*/
protected static FrequencySet<String> combineMax(FrequencySet<String> a, FrequencySet<String> b) {
FrequencySet<String> result = combineAndClip(a, b, 1);
for (Map.Entry<String, MutableInt> entry : a.entrySet()) {
result.get(entry.getKey()).v = entry.getValue().v;
}
for (Map.Entry<String, MutableInt> entry : b.entrySet()) {
MutableInt slot = result.get(entry.getKey());
slot.v = Math.max(slot.v, entry.getValue().v);
}
return result;
}
/**
* Generate a frequency set as the union of two input sets, with the
* values clipped to a specified maximum value. If an element is
* contained in both sets, the value for the output, prior to clipping,
* will be the sum of the two input values.
*
* @param a The first set.
* @param b The second set.
* @param clip The maximum value to allow for any output.
* @return The sum of the two sets, with the individual elements clipped
* to the maximum value gived by {@code clip}.
*/
protected static FrequencySet<String> combineAndClip(FrequencySet<String> a, FrequencySet<String> b, int clip) {
FrequencySet<String> result = new FrequencySet<String>();
for (Map.Entry<String, MutableInt> entry : a.entrySet()) {
for (int i = 0; i < entry.getValue().v; i++) {
result.add(entry.getKey());
}
}
for (Map.Entry<String, MutableInt> entry : b.entrySet()) {
for (int i = 0; i < entry.getValue().v; i++) {
result.add(entry.getKey());
}
}
for (Map.Entry<String, MutableInt> entry : result.entrySet()) {
entry.getValue().v = Math.min(entry.getValue().v, clip);
}
return result;
}
@Override
public void tokenRef(TerminalAST ref) {
frequencies.peek().add(ref.getText());
}
@Override
public void ruleRef(GrammarAST ref, ActionAST arg) {
frequencies.peek().add(ref.getText());
}
/*
* Parser rules
*/
@Override
protected void enterAlternative(AltAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitAlternative(AltAST tree) {
frequencies.push(combineMax(frequencies.pop(), frequencies.pop()));
}
@Override
protected void enterElement(GrammarAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitElement(GrammarAST tree) {
frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2));
}
@Override
protected void exitSubrule(GrammarAST tree) {
if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) {
for (Map.Entry<String, MutableInt> entry : frequencies.peek().entrySet()) {
entry.getValue().v = 2;
}
}
}
/*
* Lexer rules
*/
@Override
protected void enterLexerAlternative(GrammarAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitLexerAlternative(GrammarAST tree) {
frequencies.push(combineMax(frequencies.pop(), frequencies.pop()));
}
@Override
protected void enterLexerElement(GrammarAST tree) {
frequencies.push(new FrequencySet<String>());
}
@Override
protected void exitLexerElement(GrammarAST tree) {
frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2));
}
@Override
protected void exitLexerSubrule(GrammarAST tree) {
if (tree.getType() == CLOSURE || tree.getType() == POSITIVE_CLOSURE) {
for (Map.Entry<String, MutableInt> entry : frequencies.peek().entrySet()) {
entry.getValue().v = 2;
}
}
}
}
}

5
tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g
View File

@ -76,6 +76,7 @@ options {
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.parse;
import org.antlr.v4.Tool;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import java.lang.reflect.Method;
@ -86,13 +87,14 @@ public String grammarName;
public GrammarAST currentRuleAST;
public String currentModeName = LexerGrammar.DEFAULT_MODE_NAME;
public String currentRuleName;
//public GrammarAST currentRuleBlock;
public GrammarAST currentOuterAltRoot;
public int currentOuterAltNumber = 1; // 1..n
public int rewriteEBNFLevel = 0;
public GrammarTreeVisitor() { this(null); }
// Should be abstract but can't make gen'd parser abstract;
// subclasses should implement else everything goes to stderr!
public ErrorManager getErrorManager() { return null; }
public void visitGrammar(GrammarAST t) { visit(t, "grammarSpec"); }
@ -742,6 +744,7 @@ lexerAtom
| WILDCARD
| LEXER_CHAR_SET
| range
| ruleref
;
actionElement

7
tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java
View File

@ -130,6 +130,9 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
this.errMgr = g.tool.errMgr;
}
@Override
public ErrorManager getErrorManager() { return errMgr; }
public void process() { visitGrammar(g.ast); }
// Routines to route visitor traffic to the checking routines
@ -376,8 +379,8 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
void checkInvalidRuleRef(Token ruleID) {
String fileName = ruleID.getInputStream().getSourceName();
if ( g.isLexer() && Character.isLowerCase(ruleID.getText().charAt(0)) ) {
g.tool.errMgr.grammarError(ErrorType.PARSER_RULES_NOT_ALLOWED,
fileName, ruleID, ruleID.getText());
g.tool.errMgr.grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE,
fileName, ruleID, ruleID.getText(), currentRuleName);
}
}

10
tool/src/org/antlr/v4/semantics/RuleCollector.java
View File

@ -36,6 +36,7 @@ import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.parse.ScopeParser;
import org.antlr.v4.tool.AttributeDict;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
@ -52,13 +53,20 @@ import java.util.Map;
public class RuleCollector extends GrammarTreeVisitor {
/** which grammar are we checking */
public Grammar g;
public ErrorManager errMgr;
// stuff to collect. this is the output
public OrderedHashMap<String, Rule> rules = new OrderedHashMap<String, Rule>();
public MultiMap<String,GrammarAST> ruleToAltLabels = new MultiMap<String, GrammarAST>();
public Map<String,String> altLabelToRuleName = new HashMap<String, String>();
public RuleCollector(Grammar g) { this.g = g; }
public RuleCollector(Grammar g) {
this.g = g;
this.errMgr = g.tool.errMgr;
}
@Override
public ErrorManager getErrorManager() { return errMgr; }
public void process(GrammarAST ast) { visitGrammar(ast); }

11
tool/src/org/antlr/v4/semantics/SymbolCollector.java
View File

@ -31,6 +31,7 @@
package org.antlr.v4.semantics;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.Rule;
@ -68,10 +69,18 @@ public class SymbolCollector extends GrammarTreeVisitor {
/** Track action name node in @parser::members {...} or @members {...} */
List<GrammarAST> namedActions = new ArrayList<GrammarAST>();
public ErrorManager errMgr;
// context
public Rule currentRule;
public SymbolCollector(Grammar g) { this.g = g; }
public SymbolCollector(Grammar g) {
this.g = g;
this.errMgr = g.tool.errMgr;
}
@Override
public ErrorManager getErrorManager() { return errMgr; }
public void process(GrammarAST ast) { visitGrammar(ast); }

6
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -224,6 +224,12 @@ public enum ErrorType {
* <p>reference to undefined rule: <em>rule</em></p>
*/
UNDEFINED_RULE_REF(56, "reference to undefined rule: <arg>", ErrorSeverity.ERROR),
/**
* Compiler Error 160.
*
* <p>reference to undefined rule: <em>rule</em></p>
*/
PARSER_RULE_REF_IN_LEXER_RULE(160, "reference to parser rule <arg> in lexer rule <arg2>", ErrorSeverity.ERROR),
/**
* Compiler Error 57.
*

3
tool/src/org/antlr/v4/tool/Grammar.java
View File

@ -30,7 +30,6 @@
package org.antlr.v4.tool;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.Tool;
import org.antlr.v4.analysis.LeftRecursiveRuleTransformer;
import org.antlr.v4.misc.CharSupport;
@ -1063,6 +1062,8 @@ public class Grammar implements AttributeResolver {
public void stringRef(TerminalAST ref) {
strings.add(ref.getText());
}
@Override
public ErrorManager getErrorManager() { return tool.errMgr; }
};
collector.visitGrammar(ast);
return strings;

21
tool/test/org/antlr/v4/test/TestATNConstruction.java
View File

@ -35,11 +35,14 @@ import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.junit.Test;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -397,24 +400,32 @@ public class TestATNConstruction extends BaseTest {
@Test public void testParserRuleRefInLexerRule() throws Exception {
boolean threwException = false;
ErrorQueue errorQueue = new ErrorQueue();
try {
String gstr =
"lexer grammar U;\n"+
"A : a;";
"grammar U;\n"+
"a : A;\n"+
"A : a;\n";
Tool tool = new Tool();
tool.removeListeners();
ErrorQueue errorQueue = new ErrorQueue();
tool.addListener(errorQueue);
System.out.println("errors:"+errorQueue.errors);
System.out.println("warns:"+errorQueue.warnings);
assertEquals(0, errorQueue.size());
GrammarRootAST grammarRootAST = tool.parseGrammarFromString(gstr);
assertEquals(0, errorQueue.size());
Grammar g = tool.createGrammar(grammarRootAST);
assertEquals(0, errorQueue.size());
g.fileName = "<string>";
tool.process(g, false);
}
catch (Exception e) {
threwException = true;
e.printStackTrace();
}
System.out.println(errorQueue);
assertEquals(1, errorQueue.errors.size());
assertEquals(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE, errorQueue.errors.get(0).getErrorType());
assertEquals("[a, A]", Arrays.toString(errorQueue.errors.get(0).getArgs()));
assertTrue(!threwException);
}