From c33b06fdde6de46ed304157f9ab5218580058881 Mon Sep 17 00:00:00 2001 From: parrt Date: Sun, 2 Oct 2011 15:21:32 -0800 Subject: [PATCH] bug fix; no lexer if no lex rules [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9095] --- .../antlr/v4/runtime/ANTLRParserListener.java | 13 ++-- .../org/antlr/v4/runtime/BaseRecognizer.java | 7 +- .../v4/runtime/NoViableAltException.java | 20 +++--- .../v4/runtime/RecognitionException.java | 71 ++++++++++++------- .../v4/runtime/atn/ParserATNSimulator.java | 5 +- .../org/antlr/v4/runtime/tree/BaseAST.java | 7 +- .../org/antlr/v4/runtime/tree/TreeParser.java | 2 +- tool/src/org/antlr/v4/tool/GrammarAST.java | 13 ++++ .../v4/tool/GrammarTransformPipeline.java | 15 ++-- .../org/antlr/v4/test/TestATNInterpreter.java | 16 ++--- .../org/antlr/v4/test/TestParseErrors.java | 3 +- 11 files changed, 93 insertions(+), 79 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRParserListener.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRParserListener.java index a72fffa5b..935e70322 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRParserListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRParserListener.java @@ -48,14 +48,10 @@ public interface ANTLRParserListener { * @param recognizer * What parser got the error. From this object, you * can access the context as well as the input stream. - * @param startTokenIndex - * The offending token index in the input token stream. - * If no viable alternative error, it's the token index + * @param offendingToken + * The offending token in the input token stream. + * If no viable alternative error, e has token * at which we started production for the decision. - * @param stopTokenIndex - * Normally a copy of the offending token index unless we - * found a no viable alternative error. In that case, - * this is the offending token index. * @param line * At what line in input to the error occur? This always refers to * stopTokenIndex @@ -70,8 +66,7 @@ public interface ANTLRParserListener { * surrounding rule. */ public void error(BaseRecognizer recognizer, - int startTokenIndex, - int stopTokenIndex, + Token offendingToken, int line, int charPositionInLine, String msg, diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java index 83a3144cd..e189432f7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java @@ -197,17 +197,12 @@ public abstract class BaseRecognizer extends Recognizer { { int line = offendingToken.getLine(); int charPositionInLine = offendingToken.getCharPositionInLine(); - int start = offendingToken.getTokenIndex(); - int stop = start; - if ( e instanceof NoViableAltException ) { - start = ((NoViableAltException)e).startToken.getTokenIndex(); - } if ( _listeners==null || _listeners.size()==0 ) { emitErrorMessage("line "+line+":"+charPositionInLine+" "+msg); return; } for (ANTLRParserListener pl : _listeners) { - pl.error(this, start, stop, line, charPositionInLine, msg, e); + pl.error(this, offendingToken, line, charPositionInLine, msg, e); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java index 7f919655e..885f2f631 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java @@ -66,14 +66,14 @@ public class NoViableAltException extends RecognitionException { this.offendingToken = offendingToken; } - public String toString() { - if ( recognizer!=null ) { - TokenStream tokens = ((Parser)recognizer).getTokenStream(); - String bad = tokens.toString(startToken.getTokenIndex(), - offendingToken.getTokenIndex()); - return "NoViableAltException(input=\""+bad+"\" last token type is "+ - getUnexpectedType()+")"; - } - return "NoViableAltException(last token type is "+getUnexpectedType()+")"; - } +// public String toString() { +// if ( recognizer!=null ) { +// TokenStream tokens = ((Parser)recognizer).getTokenStream(); +// String bad = tokens.toString(startToken.getTokenIndex(), +// offendingToken.getTokenIndex()); +// return "NoViableAltException(input=\""+bad+"\" last token type is "+ +// getUnexpectedType()+")"; +// } +// return "NoViableAltException(last token type is "+getUnexpectedType()+")"; +// } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java index 466a91cb3..aecd2b8ed 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java @@ -29,7 +29,6 @@ package org.antlr.v4.runtime; import org.antlr.v4.runtime.misc.IntervalSet; -import org.antlr.v4.runtime.tree.*; /** The root of the ANTLR exception hierarchy. In general, ANTLR tracks just * 3 kinds of errors: prediction errors, failed predicate errors, and @@ -39,15 +38,15 @@ import org.antlr.v4.runtime.tree.*; */ public class RecognitionException extends RuntimeException { /** Who threw the exception? */ - public BaseRecognizer recognizer; + protected BaseRecognizer recognizer; // TODO: make a dummy recognizer for the interpreter to use? // Next two (ctx,input) should be what is in recognizer, but // won't work when interpreting - public RuleContext ctx; + protected RuleContext ctx; - public IntStream input; + protected IntStream input; /** What is index of token/char were we looking at when the error occurred? */ // public int offendingTokenIndex; @@ -56,12 +55,14 @@ public class RecognitionException extends RuntimeException { * can retrieve the ith Token, we have to track the Token object. * For parsers. Even when it's a tree parser, token might be set. */ - public Token offendingToken; + protected Token offendingToken; /** If this is a tree parser exception, node is set to the node with * the problem. - */ public Object offendingNode; + */ + + protected int offendingState; /** If you are parsing a tree node stream, you will encounter som * imaginary nodes w/o line/col info. We now search backwards looking @@ -80,33 +81,49 @@ public class RecognitionException extends RuntimeException { this.recognizer = recognizer; this.input = input; this.ctx = ctx; - - //this.offendingTokenIndex = input.index(); -// if ( input instanceof TokenStream ) { -// this.offendingToken = ((TokenStream)input).LT(1); -// } -// else if ( input instanceof ASTNodeStream) { -// //extractInformationFromTreeNodeStream(input); -// } + this.offendingState = ctx.s; } + /** Where was the parser in the ATN when the error occurred? + * For No viable alternative exceptions, this is the decision state number. + * For others, it is the state whose emanating edge we couldn't match. + * This will help us tie into the grammar and syntax diagrams in + * ANTLRWorks v2. + */ + public int getOffendingState() { return offendingState; } + public IntervalSet getExpectedTokens() { if ( recognizer!=null ) return recognizer._interp.atn.nextTokens(ctx); return null; } - - /** Return the token type or char of the unexpected input element */ - public int getUnexpectedType() { - if ( recognizer==null ) return offendingToken.getType(); - if ( recognizer.getInputStream() instanceof TokenStream) { - return offendingToken.getType(); - } - else if ( recognizer.getInputStream() instanceof ASTNodeStream) { - ASTNodeStream nodes = (ASTNodeStream)recognizer.getInputStream(); - ASTAdaptor adaptor = nodes.getTreeAdaptor(); - return adaptor.getType(offendingNode); - } - return Token.INVALID_TYPE; + public RuleContext getCtx() { + return ctx; } + + public IntStream getInputStream() { + return input; + } + + public Token getOffendingToken() { + return offendingToken; + } + + public BaseRecognizer getRecognizer() { + return recognizer; + } + + // /** Return the token type or char of the unexpected input element */ +// public int getUnexpectedType() { +// if ( recognizer==null ) return offendingToken.getType(); +// if ( recognizer.getInputStream() instanceof TokenStream) { +// return offendingToken.getType(); +// } +// else if ( recognizer.getInputStream() instanceof ASTNodeStream) { +// ASTNodeStream nodes = (ASTNodeStream)recognizer.getInputStream(); +// ASTAdaptor adaptor = nodes.getTreeAdaptor(); +// return adaptor.getType(offendingNode); +// } +// return Token.INVALID_TYPE; +// } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index faa0abf04..62318a313 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -124,7 +124,10 @@ public class ParserATNSimulator extends ATNSimulator { try { alt = execATN(input, dfa, m, s0_closure, useContext); } - catch (NoViableAltException nvae) { dumpDeadEndConfigs(nvae); throw nvae; } + catch (NoViableAltException nvae) { + if ( debug ) dumpDeadEndConfigs(nvae); + throw nvae; + } finally { input.seek(m); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseAST.java b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseAST.java index b81cdff97..41c747e99 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseAST.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseAST.java @@ -29,8 +29,7 @@ package org.antlr.v4.runtime.tree; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /** A generic AST implementation with no payload. You must subclass to * actually have any user data. ANTLR v3 uses a list of children approach @@ -190,13 +189,13 @@ public abstract class BaseAST implements AST { /** Insert child t at child position i (0..n) by shifting children i+1..n-1 to the right one position. Set parent / indexes properly but does NOT collapse nil-rooted t's that come in here like addChild. - Set position i==n to add to end. + Set position i==n to add to end. Add @ 0 in empty list, adds 1st child. */ public void insertChild(int i, BaseAST t) { if (i < 0 || i > getChildCount()) { throw new IndexOutOfBoundsException(i+" out or range"); } - + if ( children==null ) children = createChildrenList(); children.add(i, t); // walk others to increment their child indexes // set index, parent of this one too diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java index 0cfb5cd40..23cf6ff90 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java @@ -97,7 +97,7 @@ public class TreeParser extends BaseRecognizer { { String tokenText = ""; - ASTAdaptor adaptor = ((ASTNodeStream)e.input).getTreeAdaptor(); + ASTAdaptor adaptor = ((ASTNodeStream)e.getInputStream()).getTreeAdaptor(); return adaptor.create(new CommonToken(expectedTokenType, tokenText)); } diff --git a/tool/src/org/antlr/v4/tool/GrammarAST.java b/tool/src/org/antlr/v4/tool/GrammarAST.java index 20738988c..872453326 100644 --- a/tool/src/org/antlr/v4/tool/GrammarAST.java +++ b/tool/src/org/antlr/v4/tool/GrammarAST.java @@ -113,6 +113,19 @@ public class GrammarAST extends CommonTree { return false; } + /** Fix bug */ + @Override + public void insertChild(int i, Object t) { + if (i < 0 || i > getChildCount()) { + throw new IndexOutOfBoundsException(i+" out or range"); + } + if ( children==null ) children = createChildrenList(); + children.add(i, t); + // walk others to increment their child indexes + // set index, parent of this one too + this.freshenParentAndChildIndexes(i); + } + // TODO: move to basetree when i settle on how runtime works // TODO: don't include this node!! // TODO: reuse other method diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index 556adccba..1669773cd 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -375,13 +375,6 @@ public class GrammarTransformPipeline { Map litAliases = Grammar.getStringLiteralAliasesFromLexerRules(lexerAST); - if ( nLexicalRules==0 && (litAliases==null||litAliases.size()==0) && - combinedGrammar.stringLiteralToTypeMap.size()==0 ) - { - // no rules, tokens{}, or 'literals' in grammar - return null; - } - Set stringLiterals = combinedGrammar.getStringLiterals(); // add strings from combined grammar (and imported grammars) into lexer // put them first as they are keywords; must resolve ambigs to these rules @@ -400,7 +393,8 @@ public class GrammarTransformPipeline { CommonToken idToken = new CommonToken(ANTLRParser.ID, rname); litRule.addChild(new TerminalAST(idToken)); litRule.addChild(blk); - lexerRulesRoot.getChildren().add(0, litRule); // add first + lexerRulesRoot.insertChild(0, litRule); // add first +// lexerRulesRoot.getChildren().add(0, litRule); lexerRulesRoot.freshenParentAndChildIndexes(); // reset indexes and set litRule parent } @@ -409,11 +403,10 @@ public class GrammarTransformPipeline { combinedAST.sanityCheckParentAndChildIndexes(); // System.out.println(combinedAST.toTokenString()); -// lexerAST.freshenParentAndChildIndexesDeeply(); -// combinedAST.freshenParentAndChildIndexesDeeply(); - System.out.println("after extract implicit lexer ="+combinedAST.toStringTree()); System.out.println("lexer ="+lexerAST.toStringTree()); + + if ( lexerRulesRoot.getChildCount()==0 ) return null; return lexerAST; } } diff --git a/tool/test/org/antlr/v4/test/TestATNInterpreter.java b/tool/test/org/antlr/v4/test/TestATNInterpreter.java index 409fd5e78..2351d7c87 100644 --- a/tool/test/org/antlr/v4/test/TestATNInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNInterpreter.java @@ -68,8 +68,8 @@ public class TestATNInterpreter extends BaseTest { checkMatchedAlt(lg, g, "ac", 1); } catch (NoViableAltException re) { - errorIndex = re.offendingToken.getTokenIndex(); - errorTokenType = re.offendingToken.getType(); + errorIndex = re.getOffendingToken().getTokenIndex(); + errorTokenType = re.getOffendingToken().getType(); } assertEquals(1, errorIndex); assertEquals(errorTokenType, 5); @@ -94,8 +94,8 @@ public class TestATNInterpreter extends BaseTest { checkMatchedAlt(lg, g, "abd", 1); } catch (NoViableAltException re) { - errorIndex = re.offendingToken.getTokenIndex(); - errorTokenType = re.offendingToken.getType(); + errorIndex = re.getOffendingToken().getTokenIndex(); + errorTokenType = re.getOffendingToken().getType(); } assertEquals(2, errorIndex); assertEquals(errorTokenType, 6); @@ -117,8 +117,8 @@ public class TestATNInterpreter extends BaseTest { checkMatchedAlt(lg, g, "abd", 1); } catch (NoViableAltException re) { - errorIndex = re.offendingToken.getTokenIndex(); - errorTokenType = re.offendingToken.getType(); + errorIndex = re.getOffendingToken().getTokenIndex(); + errorTokenType = re.getOffendingToken().getType(); } assertEquals(2, errorIndex); assertEquals(errorTokenType, 6); @@ -186,8 +186,8 @@ public class TestATNInterpreter extends BaseTest { checkMatchedAlt(lg, g, "abd", 1); } catch (NoViableAltException re) { - errorIndex = re.offendingToken.getTokenIndex(); - errorTokenType = re.offendingToken.getType(); + errorIndex = re.getOffendingToken().getTokenIndex(); + errorTokenType = re.getOffendingToken().getType(); } assertEquals(2, errorIndex); assertEquals(6, errorTokenType); diff --git a/tool/test/org/antlr/v4/test/TestParseErrors.java b/tool/test/org/antlr/v4/test/TestParseErrors.java index 32caa80f2..16b504822 100644 --- a/tool/test/org/antlr/v4/test/TestParseErrors.java +++ b/tool/test/org/antlr/v4/test/TestParseErrors.java @@ -38,8 +38,7 @@ public class TestParseErrors extends BaseTest { "a : 'a' 'b'" + " | 'a' 'c'" + ";\n" + - "q : 'e' ;\n" + - "WS : ' ' ;\n"; + "q : 'e' ;\n"; String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "ae", false); String expecting = "line 1:1 no viable alternative at input 'e'\n"; String result = stderrDuringParse;