added more dbg prints, turned on context sensitivity in test java lr.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9545]

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -46,6 +46,7 @@ import java.util.*;
 public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	public static boolean debug = false;
 	public static boolean dfa_debug = false;
+	public static boolean retry_debug = false;
 
 	public static int ATN_failover = 0;
 	public static int predict_calls = 0;
@@ -122,7 +123,6 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	}
 
 	public void reset() {
-		userWantsCtxSensitive = false;
 		outerContext = ParserRuleContext.EMPTY;
 		prevAccept = null;
 		prevAcceptIndex = -1;
@@ -161,7 +161,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 			input.seek(index);
 			input.release(m);
 		}
-		if ( debug ) System.out.println("DFA after predictATN: "+dfa.toString());
+		if ( debug ) System.out.println("DFA after predictATN: "+dfa.toString(parser.getTokenNames()));
 		return alt;
 	}
 
@@ -181,12 +181,12 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 					   @NotNull DFAState s0,
                        @Nullable ParserRuleContext outerContext)
     {
-//		dump(dfa);
 		if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
 		this.outerContext = outerContext;
 		if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
 											" exec LA(1)=="+ getLookaheadName(input) +
 											", outerContext="+outerContext.toString(parser));
+		if ( dfa_debug ) System.out.print(dfa.toString(parser.getTokenNames()));
 		DFAState prevAcceptState = null;
 		DFAState s = s0;
 		int t = input.LA(1);
@@ -243,7 +243,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 						}
 					}
 					if ( dfa_debug ) {
-						System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa);
+						System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames()));
 						//dump(dfa);
 					}
 					// action already executed
@@ -288,11 +288,15 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	}
 
 	public String getInputString(@NotNull SymbolStream<Symbol> input, int start) {
+		return getInputString(input, start, input.index());
+	}
+
+	public String getInputString(@NotNull SymbolStream<Symbol> input, int start, int stop) {
 		if ( input instanceof TokenStream ) {
-			return ((TokenStream)input).toString(start,input.index());
+			return ((TokenStream)input).toString(start,stop);
 		}
 		else if ( input instanceof ASTNodeStream) {
-			return ((ASTNodeStream<Symbol>)input).toString(input.get(start),input.LT(1));
+			return ((ASTNodeStream<Symbol>)input).toString(input.get(start),input.get(stop));
 		}
 		return "n/a";
 	}
@@ -355,6 +359,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 									   getInputString(input, startIndex));
 					System.out.println("REACH="+reach);
 				}
+//				System.out.println("AMBIG dec "+dfa.decision+" for alt "+ambigAlts+" upon "+
+//								   getInputString(input, startIndex));
+//				System.out.println("userWantsCtxSensitive="+userWantsCtxSensitive);
 
                 // can we resolve with predicates?
                 SemanticContext[] altToPred =
@@ -544,10 +551,11 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		retry_with_context++;
 		int old_k = input.index();
 		// retry using context, if any; if none, kill all but min as before
-		if ( debug ) System.out.println("RETRY "+ getInputString(input, startIndex) +
+		if ( retry_debug ) System.out.println("RETRY '"+ getInputString(input, startIndex) +
-										" with ctx="+ originalContext);
+										"' with ctx="+ originalContext);
 		int min = ambigAlts.getMinElement();
 		if ( originalContext==ParserRuleContext.EMPTY ) {
+			if ( retry_debug ) System.out.println("ctx empty; no need to retry");
 			// no point in retrying with ctx since it's same.
 			// this implies that we have a true ambiguity
 			reportAmbiguity(startIndex, input.index(), ambigAlts, reach);
@@ -558,10 +566,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		input.seek(startIndex); // rewind
 		DFA ctx_dfa = new DFA(dfa.atnStartState);
 		int ctx_alt = predictATN(ctx_dfa, input, originalContext, true);
-		if ( debug ) System.out.println("retry predicts "+ctx_alt+" vs "+ambigAlts.getMinElement()+
+		if ( retry_debug ) System.out.println("retry predicts "+ctx_alt+" vs "+ambigAlts.getMinElement()+
 										" with conflict="+ctx_dfa.conflict+
-										" dfa="+ctx_dfa);
+										" full ctx dfa="+ctx_dfa.toString(parser.getTokenNames()));
-
 
 		if ( ctx_dfa.conflict ) {
 //			System.out.println("retry gives ambig for "+input.toString(startIndex, input.index()));
@@ -571,7 +578,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 //			System.out.println("NO ambig for "+input.toString(startIndex, input.index()));
 //			System.out.println(ctx_dfa.toString(parser.getTokenNames()));
 			if ( old_k != input.index() ) {
-				System.out.println("ACK!!!!!!!! diff k; old="+(old_k-startIndex+1)+", new="+(input.index()-startIndex+1));
+				if ( retry_debug ) System.out.println("used diff amount of k; old="+(old_k-startIndex+1)+", new="+(input.index()-startIndex+1));
 			}
 			retry_with_context_indicates_no_conflict++;
 			reportContextSensitivity(startIndex, input.index(), ambigAlts, reach);
@@ -583,11 +590,16 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		int predictedAlt = ctx_alt;
 		if ( buildDFA ) {
 			DFAState reachTarget = addDFAEdge(dfa, closure, t, reach);
+			makeAcceptState(reachTarget, ctx_alt);
 			reachTarget.isCtxSensitive = true;
 			if ( reachTarget.ctxToPrediction==null ) {
 				reachTarget.ctxToPrediction = new LinkedHashMap<RuleContext, Integer>();
 			}
 			reachTarget.ctxToPrediction.put(originalContext, predictedAlt);
+			if ( retry_debug ) {
+				System.out.println("adding edge upon "+getTokenName(t));
+				System.out.println("DFA is "+dfa.toString(parser.getTokenNames()));
+			}
 		}
 //					System.out.println("RESOLVE to "+predictedAlt);
 		//System.out.println(reachTarget.ctxToPrediction.size()+" size of ctx map");
@@ -815,8 +827,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 							   @NotNull IntervalSet alts,
 							   @NotNull OrderedHashSet<ATNConfig> configs)
 	{
-		if ( debug ) {
+		if ( debug || retry_debug ) {
-			System.out.println("reportConflict "+alts+":"+configs);
+			System.out.println("reportConflict "+alts+":"+configs+
+							   ", input="+getInputString(parser.getInputStream(), startIndex, stopIndex));
 		}
 		if ( parser!=null ) parser.reportConflict(startIndex, stopIndex, alts, configs);
 	}
@@ -825,6 +838,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 										 @NotNull IntervalSet alts,
 										 @NotNull OrderedHashSet<ATNConfig> configs)
 	{
+		if ( debug || retry_debug ) {
+			System.out.println("reportContextSensitivity "+alts+":"+configs+
+							   ", input="+getInputString(parser.getInputStream(), startIndex, stopIndex));
+		}
 		if ( parser!=null ) parser.reportContextSensitivity(startIndex, stopIndex, alts, configs);
 	}
 
@@ -833,9 +850,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 								@NotNull IntervalSet ambigAlts,
 								@NotNull OrderedHashSet<ATNConfig> configs)
 	{
-		if ( debug ) {
+		if ( debug || retry_debug ) {
 			System.out.println("reportAmbiguity "+
-							   ambigAlts+":"+configs);
+							   ambigAlts+":"+configs+
+							   ", input="+getInputString(parser.getInputStream(), startIndex, stopIndex));
 		}
 		if ( parser!=null ) parser.reportAmbiguity(startIndex, stopIndex, ambigAlts, configs);
 	}
@@ -845,9 +863,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 											 @NotNull SemanticContext[] altToPred,
 											 @NotNull OrderedHashSet<ATNConfig> configs)
 	{
-		if ( debug ) {
+		if ( debug || retry_debug ) {
 			System.out.println("reportInsufficientPredicates "+
-								   ambigAlts+":"+Arrays.toString(altToPred));
+							   ambigAlts+":"+Arrays.toString(altToPred)+
+							   getInputString(parser.getInputStream(), startIndex, stopIndex));
 		}
 		if ( parser!=null ) {
 			parser.reportInsufficientPredicates(startIndex, stopIndex, ambigAlts,
@@ -1081,6 +1100,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		DFAState to = addDFAState(dfa, q);
         if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
 		addDFAEdge(from, t, to);
+		if ( debug ) System.out.println("DFA=\n"+dfa.toString(parser.getTokenNames()));
 		return to;
 	}
 

runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java

@@ -28,11 +28,12 @@
  */
 package org.antlr.v4.runtime.dfa;
 
+import org.antlr.v4.runtime.atn.ATNState;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
-import org.antlr.v4.runtime.atn.ATNState;
 
-import java.util.*;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 public class DFA {
 	/** A set of all DFA states. Use Map so we can get old state back
@@ -69,4 +70,5 @@ public class DFA {
 		DFASerializer serializer = new LexerDFASerializer(this);
 		return serializer.toString();
 	}
+
 }

runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java

@@ -32,6 +32,8 @@ package org.antlr.v4.runtime.dfa;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
 
+import java.util.Map;
+
 /** A DFA walker that knows how to dump them to serialized strings. */
 public class DFASerializer {
 	@NotNull
@@ -48,7 +50,9 @@ public class DFASerializer {
 	public String toString() {
 		if ( dfa.s0==null ) return null;
 		StringBuilder buf = new StringBuilder();
-		for (DFAState s : dfa.states.values()) {
+		Map<DFAState,DFAState> states = dfa.states;
+		if ( states!=null ) {
+			for (DFAState s : states.values()) {
 				int n = 0;
 				if ( s.edges!=null ) n = s.edges.length;
 				for (int i=0; i<n; i++) {
@@ -60,6 +64,7 @@ public class DFASerializer {
 					}
 				}
 			}
+		}
 		String output = buf.toString();
 		//return Utils.sortLinesInString(output);
 		return output;

tool/playground/MyUListener.java

@@ -1,60 +0,0 @@
-/*
- [The "BSD license"]
- Copyright (c) 2011 Terence Parr
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- 1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
- 3. The name of the author may not be used to endorse or promote products
-    derived from this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.Token;
-
-public class MyUListener extends BlankUListener {
-    UParser parser;
-    public MyUListener(UParser parser) { this.parser = parser; }
-    @Override
-    public void enterRule(UParser.aContext ctx) {
-        System.out.println("enter Rule"+ctx.toInfoString(parser));
-    }
-
-    @Override
-    public void exitRule(UParser.aContext ctx) {
-        System.out.println("exitRule "+ctx.toInfoString(parser));
-    }
-
-    @Override
-    public void visitTerminal(Token token) {
-        System.out.println("visitTerminal "+token);
-    }
-
-    @Override
-    public void enterEveryRule(ParserRuleContext<Token> ctx) {
-        System.out.println("enterEvery Rule"+ctx.toInfoString(parser));
-    }
-
-    @Override
-    public void exitEveryRule(ParserRuleContext<Token> ctx) {
-        System.out.println("exitEvery Rule"+ctx.toInfoString(parser));
-    }
-}

tool/playground/T.g

@@ -1,5 +1,8 @@
 grammar T;
-s : (ID | ID ID?) ';' ;
+s : FOO;
+a : e B ;
+b : e A B ;
+e : A | ;
 ID : 'a'..'z'+ ;
 INT : '0'..'9'+;
 WS : (' '|'\n') {skip();} ;

tool/playground/TestJavaLR.java

@@ -131,6 +131,7 @@ class TestJavaLR {
 				if ( parser==null ) {
 					parser = new JavaLRParser(null);
 					if ( showTree ) parser.setBuildParseTree(true);
+					parser.getInterpreter().setContextSensitive(true);
 //                    parser.setErrorHandler(new BailErrorStrategy<Token>());
 //					parser.getInterpreter().setContextSensitive(true);
 				}

tool/playground/TestU.java

@@ -1,14 +1,15 @@
 import org.antlr.v4.runtime.ANTLRFileStream;
 import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.ParserRuleContext;
 
 public class TestU {
 	public static void main(String[] args) throws Exception {
 		ULexer t = new ULexer(new ANTLRFileStream(args[0]));
 		CommonTokenStream tokens = new CommonTokenStream(t);
 		UParser p = new UParser(tokens);
-        p.setListener(new MyUListener(p));
+		p.getInterpreter().setContextSensitive(true);
-        p.s();
+		p.setBuildParseTree(true);
-//		p.setBuildParseTree(true);
+		ParserRuleContext r = p.s();
-//		UParser.aContext ctx = p.a();
+		System.out.println(r.toStringTree(p));
 	}
 }

tool/playground/U.g

@@ -1,6 +1,11 @@
 grammar U;
-s : a ;
+s : '{' stat* '}' ;
-a : ID (',' ID)* ';' ;
+// if x then break else if y then return else break
+// still ambig: if x then if y then break else break
+stat: 'if' ID 'then' stat ('else' stat)?
+    | 'break'
+    | 'return'
+    ;
 
 INT : '0'..'9'+ ;
 ID : 'a'..'z'+ ;