bug fix; no lexer if no lex rules

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9095]

runtime/Java/src/org/antlr/v4/runtime/ANTLRParserListener.java

@@ -48,14 +48,10 @@ public interface ANTLRParserListener {
 	 * @param recognizer
 	 * 		  What parser got the error. From this object, you
 	 * 		  can access the context as well as the input stream.
-	 * @param startTokenIndex
-	 * 		  The offending token index in the input token stream.
-	 * 		  If no viable alternative error, it's the token index
+	 * @param offendingToken
+	 * 		  The offending token in the input token stream.
+	 * 		  If no viable alternative error, e has token
 	 * 		  at which we started production for the decision.
-	 * @param stopTokenIndex
-	 * 		  Normally a copy of the offending token index unless we
-	 * 		  found a no viable alternative error. In that case,
-	 * 		  this is the offending token index.
 	 * @param line
 	 * 		  At what line in input to the error occur? This always refers to
 	 * 		  stopTokenIndex
@@ -70,8 +66,7 @@ public interface ANTLRParserListener {
 	 *        surrounding rule.
 	 */
 	public void error(BaseRecognizer recognizer,
-					  int startTokenIndex,
-					  int stopTokenIndex,
+					  Token offendingToken,
 					  int line,
 					  int charPositionInLine,
 					  String msg,

runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java

@@ -197,17 +197,12 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 	{
 		int line = offendingToken.getLine();
 		int charPositionInLine = offendingToken.getCharPositionInLine();
-		int start = offendingToken.getTokenIndex();
-		int stop = start;
-		if ( e instanceof NoViableAltException ) {
-			start = ((NoViableAltException)e).startToken.getTokenIndex();
-		}
 		if ( _listeners==null || _listeners.size()==0 ) {
 			emitErrorMessage("line "+line+":"+charPositionInLine+" "+msg);
 			return;
 		}
 		for (ANTLRParserListener pl : _listeners) {
-			pl.error(this, start, stop, line, charPositionInLine, msg, e);
+			pl.error(this, offendingToken, line, charPositionInLine, msg, e);
 		}
 	}
 

runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java

@@ -66,14 +66,14 @@ public class NoViableAltException extends RecognitionException {
 		this.offendingToken = offendingToken;
 	}
 
-	public String toString() {
-		if ( recognizer!=null ) {
-			TokenStream tokens = ((Parser)recognizer).getTokenStream();
-			String bad = tokens.toString(startToken.getTokenIndex(),
-										 offendingToken.getTokenIndex());
-			return "NoViableAltException(input=\""+bad+"\" last token type is "+
-					getUnexpectedType()+")";
-		}
-		return "NoViableAltException(last token type is "+getUnexpectedType()+")";
-	}
+//	public String toString() {
+//		if ( recognizer!=null ) {
+//			TokenStream tokens = ((Parser)recognizer).getTokenStream();
+//			String bad = tokens.toString(startToken.getTokenIndex(),
+//										 offendingToken.getTokenIndex());
+//			return "NoViableAltException(input=\""+bad+"\" last token type is "+
+//					getUnexpectedType()+")";
+//		}
+//		return "NoViableAltException(last token type is "+getUnexpectedType()+")";
+//	}
 }

runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java

@@ -29,7 +29,6 @@
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.misc.IntervalSet;
-import org.antlr.v4.runtime.tree.*;
 
 /** The root of the ANTLR exception hierarchy. In general, ANTLR tracks just
  *  3 kinds of errors: prediction errors, failed predicate errors, and
@@ -39,15 +38,15 @@ import org.antlr.v4.runtime.tree.*;
  */
 public class RecognitionException extends RuntimeException {
 	/** Who threw the exception? */
-	public BaseRecognizer recognizer;
+	protected BaseRecognizer recognizer;
 
 	// TODO: make a dummy recognizer for the interpreter to use?
 	// Next two (ctx,input) should be what is in recognizer, but
 	// won't work when interpreting
 
-	public RuleContext ctx;
+	protected RuleContext ctx;
 
-	public IntStream input;
+	protected IntStream input;
 
 	/** What is index of token/char were we looking at when the error occurred? */
 //	public int offendingTokenIndex;
@@ -56,12 +55,14 @@ public class RecognitionException extends RuntimeException {
 	 *  can retrieve the ith Token, we have to track the Token object.
 	 *  For parsers.  Even when it's a tree parser, token might be set.
 	 */
-	public Token offendingToken;
+	protected Token offendingToken;
 
 	/** If this is a tree parser exception, node is set to the node with
 	 *  the problem.
-	 */
 	public Object offendingNode;
+	 */
+
+	protected int offendingState;
 
 	/** If you are parsing a tree node stream, you will encounter som
 	 *  imaginary nodes w/o line/col info.  We now search backwards looking
@@ -80,33 +81,49 @@ public class RecognitionException extends RuntimeException {
 		this.recognizer = recognizer;
 		this.input = input;
 		this.ctx = ctx;
-
-		//this.offendingTokenIndex = input.index();
-//		if ( input instanceof TokenStream ) {
-//			this.offendingToken = ((TokenStream)input).LT(1);
-//		}
-//		else if ( input instanceof ASTNodeStream) {
-//			//extractInformationFromTreeNodeStream(input);
-//		}
+		this.offendingState = ctx.s;
 	}
 
+	/** Where was the parser in the ATN when the error occurred?
+	 *  For No viable alternative exceptions, this is the decision state number.
+	 *  For others, it is the state whose emanating edge we couldn't match.
+	 *  This will help us tie into the grammar and syntax diagrams in
+	 *  ANTLRWorks v2.
+	 */
+	public int getOffendingState() { return offendingState; }
+
 	public IntervalSet getExpectedTokens() {
 		if ( recognizer!=null ) return recognizer._interp.atn.nextTokens(ctx);
 		return null;
 	}
 
-
-	/** Return the token type or char of the unexpected input element */
-	public int getUnexpectedType() {
-		if ( recognizer==null ) return offendingToken.getType();
-		if ( recognizer.getInputStream() instanceof TokenStream) {
-			return offendingToken.getType();
-		}
-		else if ( recognizer.getInputStream() instanceof ASTNodeStream) {
-			ASTNodeStream nodes = (ASTNodeStream)recognizer.getInputStream();
-			ASTAdaptor adaptor = nodes.getTreeAdaptor();
-			return adaptor.getType(offendingNode);
-		}
-		return Token.INVALID_TYPE;
+	public RuleContext getCtx() {
+		return ctx;
 	}
+
+	public IntStream getInputStream() {
+		return input;
+	}
+
+	public Token getOffendingToken() {
+		return offendingToken;
+	}
+
+	public BaseRecognizer getRecognizer() {
+		return recognizer;
+	}
+
+	//	/** Return the token type or char of the unexpected input element */
+//	public int getUnexpectedType() {
+//		if ( recognizer==null ) return offendingToken.getType();
+//		if ( recognizer.getInputStream() instanceof TokenStream) {
+//			return offendingToken.getType();
+//		}
+//		else if ( recognizer.getInputStream() instanceof ASTNodeStream) {
+//			ASTNodeStream nodes = (ASTNodeStream)recognizer.getInputStream();
+//			ASTAdaptor adaptor = nodes.getTreeAdaptor();
+//			return adaptor.getType(offendingNode);
+//		}
+//		return Token.INVALID_TYPE;
+//	}
 }

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -124,7 +124,10 @@ public class ParserATNSimulator extends ATNSimulator {
 		try {
 			alt = execATN(input, dfa, m, s0_closure, useContext);
 		}
-		catch (NoViableAltException nvae) {	dumpDeadEndConfigs(nvae); throw nvae; }
+		catch (NoViableAltException nvae) {
+			if ( debug ) dumpDeadEndConfigs(nvae);
+			throw nvae;
+		}
 		finally {
 			input.seek(m);
 		}

runtime/Java/src/org/antlr/v4/runtime/tree/BaseAST.java

@@ -29,8 +29,7 @@
 
 package org.antlr.v4.runtime.tree;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 /** A generic AST implementation with no payload.  You must subclass to
  *  actually have any user data.  ANTLR v3 uses a list of children approach
@@ -190,13 +189,13 @@ public abstract class BaseAST implements AST {
 	/** Insert child t at child position i (0..n) by shifting children
 		i+1..n-1 to the right one position. Set parent / indexes properly
 	 	but does NOT collapse nil-rooted t's that come in here like addChild.
-	    Set position i==n to add to end.
+	    Set position i==n to add to end. Add @ 0 in empty list, adds 1st child.
 	 */
 	public void insertChild(int i, BaseAST t) {
 		if (i < 0 || i > getChildCount()) {
 			throw new IndexOutOfBoundsException(i+" out or range");
 		}
-
+		if ( children==null ) children = createChildrenList();
 		children.add(i, t);
 		// walk others to increment their child indexes
 		// set index, parent of this one too

runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java

@@ -97,7 +97,7 @@ public class TreeParser extends BaseRecognizer {
 	{
 		String tokenText =
 			"<missing "+getTokenNames()[expectedTokenType]+">";
-        ASTAdaptor adaptor = ((ASTNodeStream)e.input).getTreeAdaptor();
+        ASTAdaptor adaptor = ((ASTNodeStream)e.getInputStream()).getTreeAdaptor();
         return adaptor.create(new CommonToken(expectedTokenType, tokenText));
 	}
 

tool/src/org/antlr/v4/tool/GrammarAST.java

@@ -113,6 +113,19 @@ public class GrammarAST extends CommonTree {
 		return false;
 	}
 
+	/** Fix bug */
+	@Override
+	public void insertChild(int i, Object t) {
+		if (i < 0 || i > getChildCount()) {
+			throw new IndexOutOfBoundsException(i+" out or range");
+		}
+		if ( children==null ) children = createChildrenList();
+		children.add(i, t);
+		// walk others to increment their child indexes
+		// set index, parent of this one too
+		this.freshenParentAndChildIndexes(i);
+	}
+
     // TODO: move to basetree when i settle on how runtime works
     // TODO: don't include this node!!
 	// TODO: reuse other method

tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java

@@ -375,13 +375,6 @@ public class GrammarTransformPipeline {
 		Map<String,String> litAliases =
 			Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
 
-		if ( nLexicalRules==0 && (litAliases==null||litAliases.size()==0) &&
-			 combinedGrammar.stringLiteralToTypeMap.size()==0 )
-		{
-			// no rules, tokens{}, or 'literals' in grammar
-			return null;
-		}
-
 		Set<String> stringLiterals = combinedGrammar.getStringLiterals();
 		// add strings from combined grammar (and imported grammars) into lexer
 		// put them first as they are keywords; must resolve ambigs to these rules
@@ -400,7 +393,8 @@ public class GrammarTransformPipeline {
 			CommonToken idToken = new CommonToken(ANTLRParser.ID, rname);
 			litRule.addChild(new TerminalAST(idToken));
 			litRule.addChild(blk);
-			lexerRulesRoot.getChildren().add(0, litRule); // add first
+			lexerRulesRoot.insertChild(0, litRule);        // add first
+//			lexerRulesRoot.getChildren().add(0, litRule);
 			lexerRulesRoot.freshenParentAndChildIndexes(); // reset indexes and set litRule parent
 		}
 
@@ -409,11 +403,10 @@ public class GrammarTransformPipeline {
 		combinedAST.sanityCheckParentAndChildIndexes();
 //		System.out.println(combinedAST.toTokenString());
 
-//		lexerAST.freshenParentAndChildIndexesDeeply();
-//		combinedAST.freshenParentAndChildIndexesDeeply();
-
 		System.out.println("after extract implicit lexer ="+combinedAST.toStringTree());
 		System.out.println("lexer ="+lexerAST.toStringTree());
+
+		if ( lexerRulesRoot.getChildCount()==0 )	return null;
 		return lexerAST;
 	}
 }

tool/test/org/antlr/v4/test/TestATNInterpreter.java

@@ -68,8 +68,8 @@ public class TestATNInterpreter extends BaseTest {
 			checkMatchedAlt(lg, g, "ac", 1);
 		}
 		catch (NoViableAltException re) {
-			errorIndex = re.offendingToken.getTokenIndex();
-			errorTokenType = re.offendingToken.getType();
+			errorIndex = re.getOffendingToken().getTokenIndex();
+			errorTokenType = re.getOffendingToken().getType();
 		}
 		assertEquals(1, errorIndex);
 		assertEquals(errorTokenType, 5);
@@ -94,8 +94,8 @@ public class TestATNInterpreter extends BaseTest {
 			checkMatchedAlt(lg, g, "abd", 1);
 		}
 		catch (NoViableAltException re) {
-			errorIndex = re.offendingToken.getTokenIndex();
-			errorTokenType = re.offendingToken.getType();
+			errorIndex = re.getOffendingToken().getTokenIndex();
+			errorTokenType = re.getOffendingToken().getType();
 		}
 		assertEquals(2, errorIndex);
 		assertEquals(errorTokenType, 6);
@@ -117,8 +117,8 @@ public class TestATNInterpreter extends BaseTest {
 			checkMatchedAlt(lg, g, "abd", 1);
 		}
 		catch (NoViableAltException re) {
-			errorIndex = re.offendingToken.getTokenIndex();
-			errorTokenType = re.offendingToken.getType();
+			errorIndex = re.getOffendingToken().getTokenIndex();
+			errorTokenType = re.getOffendingToken().getType();
 		}
 		assertEquals(2, errorIndex);
 		assertEquals(errorTokenType, 6);
@@ -186,8 +186,8 @@ public class TestATNInterpreter extends BaseTest {
 			checkMatchedAlt(lg, g, "abd", 1);
 		}
 		catch (NoViableAltException re) {
-			errorIndex = re.offendingToken.getTokenIndex();
-			errorTokenType = re.offendingToken.getType();
+			errorIndex = re.getOffendingToken().getTokenIndex();
+			errorTokenType = re.getOffendingToken().getType();
 		}
 		assertEquals(2, errorIndex);
 		assertEquals(6, errorTokenType);

tool/test/org/antlr/v4/test/TestParseErrors.java

@@ -38,8 +38,7 @@ public class TestParseErrors extends BaseTest {
 			"a : 'a' 'b'" +
 			"  | 'a' 'c'" +
 			";\n" +
-			"q : 'e' ;\n" +
-			"WS : ' ' ;\n";
+			"q : 'e' ;\n";
 		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "ae", false);
 		String expecting = "line 1:1 no viable alternative at input 'e'\n";
 		String result = stderrDuringParse;