From 63168c5577a9ef15203d36bec5d6bd81bcef8f8d Mon Sep 17 00:00:00 2001 From: parrt Date: Wed, 7 Dec 2011 17:47:08 -0800 Subject: [PATCH] added more dbg prints, turned on context sensitivity in test java lr. [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9545] --- .../v4/runtime/atn/ParserATNSimulator.java | 58 ++++++++++++------ .../src/org/antlr/v4/runtime/dfa/DFA.java | 8 ++- .../antlr/v4/runtime/dfa/DFASerializer.java | 23 ++++--- tool/playground/MyUListener.java | 60 ------------------- tool/playground/T.g | 5 +- tool/playground/TestJavaLR.java | 1 + tool/playground/TestU.java | 9 +-- tool/playground/U.g | 9 ++- 8 files changed, 75 insertions(+), 98 deletions(-) delete mode 100644 tool/playground/MyUListener.java diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 5073ea089..b99a68253 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -46,6 +46,7 @@ import java.util.*; public class ParserATNSimulator extends ATNSimulator { public static boolean debug = false; public static boolean dfa_debug = false; + public static boolean retry_debug = false; public static int ATN_failover = 0; public static int predict_calls = 0; @@ -122,7 +123,6 @@ public class ParserATNSimulator extends ATNSimulator { } public void reset() { - userWantsCtxSensitive = false; outerContext = ParserRuleContext.EMPTY; prevAccept = null; prevAcceptIndex = -1; @@ -161,7 +161,7 @@ public class ParserATNSimulator extends ATNSimulator { input.seek(index); input.release(m); } - if ( debug ) System.out.println("DFA after predictATN: "+dfa.toString()); + if ( debug ) System.out.println("DFA after predictATN: "+dfa.toString(parser.getTokenNames())); return alt; } @@ -181,12 +181,12 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull DFAState s0, @Nullable ParserRuleContext outerContext) { -// dump(dfa); if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; this.outerContext = outerContext; if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ " exec LA(1)=="+ getLookaheadName(input) + ", outerContext="+outerContext.toString(parser)); + if ( dfa_debug ) System.out.print(dfa.toString(parser.getTokenNames())); DFAState prevAcceptState = null; DFAState s = s0; int t = input.LA(1); @@ -243,7 +243,7 @@ public class ParserATNSimulator extends ATNSimulator { } } if ( dfa_debug ) { - System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa); + System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames())); //dump(dfa); } // action already executed @@ -288,11 +288,15 @@ public class ParserATNSimulator extends ATNSimulator { } public String getInputString(@NotNull SymbolStream input, int start) { + return getInputString(input, start, input.index()); + } + + public String getInputString(@NotNull SymbolStream input, int start, int stop) { if ( input instanceof TokenStream ) { - return ((TokenStream)input).toString(start,input.index()); + return ((TokenStream)input).toString(start,stop); } else if ( input instanceof ASTNodeStream) { - return ((ASTNodeStream)input).toString(input.get(start),input.LT(1)); + return ((ASTNodeStream)input).toString(input.get(start),input.get(stop)); } return "n/a"; } @@ -355,6 +359,9 @@ public class ParserATNSimulator extends ATNSimulator { getInputString(input, startIndex)); System.out.println("REACH="+reach); } +// System.out.println("AMBIG dec "+dfa.decision+" for alt "+ambigAlts+" upon "+ +// getInputString(input, startIndex)); +// System.out.println("userWantsCtxSensitive="+userWantsCtxSensitive); // can we resolve with predicates? SemanticContext[] altToPred = @@ -544,10 +551,11 @@ public class ParserATNSimulator extends ATNSimulator { retry_with_context++; int old_k = input.index(); // retry using context, if any; if none, kill all but min as before - if ( debug ) System.out.println("RETRY "+ getInputString(input, startIndex) + - " with ctx="+ originalContext); + if ( retry_debug ) System.out.println("RETRY '"+ getInputString(input, startIndex) + + "' with ctx="+ originalContext); int min = ambigAlts.getMinElement(); if ( originalContext==ParserRuleContext.EMPTY ) { + if ( retry_debug ) System.out.println("ctx empty; no need to retry"); // no point in retrying with ctx since it's same. // this implies that we have a true ambiguity reportAmbiguity(startIndex, input.index(), ambigAlts, reach); @@ -558,10 +566,9 @@ public class ParserATNSimulator extends ATNSimulator { input.seek(startIndex); // rewind DFA ctx_dfa = new DFA(dfa.atnStartState); int ctx_alt = predictATN(ctx_dfa, input, originalContext, true); - if ( debug ) System.out.println("retry predicts "+ctx_alt+" vs "+ambigAlts.getMinElement()+ + if ( retry_debug ) System.out.println("retry predicts "+ctx_alt+" vs "+ambigAlts.getMinElement()+ " with conflict="+ctx_dfa.conflict+ - " dfa="+ctx_dfa); - + " full ctx dfa="+ctx_dfa.toString(parser.getTokenNames())); if ( ctx_dfa.conflict ) { // System.out.println("retry gives ambig for "+input.toString(startIndex, input.index())); @@ -571,7 +578,7 @@ public class ParserATNSimulator extends ATNSimulator { // System.out.println("NO ambig for "+input.toString(startIndex, input.index())); // System.out.println(ctx_dfa.toString(parser.getTokenNames())); if ( old_k != input.index() ) { - System.out.println("ACK!!!!!!!! diff k; old="+(old_k-startIndex+1)+", new="+(input.index()-startIndex+1)); + if ( retry_debug ) System.out.println("used diff amount of k; old="+(old_k-startIndex+1)+", new="+(input.index()-startIndex+1)); } retry_with_context_indicates_no_conflict++; reportContextSensitivity(startIndex, input.index(), ambigAlts, reach); @@ -581,13 +588,18 @@ public class ParserATNSimulator extends ATNSimulator { // TODO: if ambig, why turn on ctx sensitive? int predictedAlt = ctx_alt; - if ( buildDFA) { + if ( buildDFA ) { DFAState reachTarget = addDFAEdge(dfa, closure, t, reach); + makeAcceptState(reachTarget, ctx_alt); reachTarget.isCtxSensitive = true; if ( reachTarget.ctxToPrediction==null ) { reachTarget.ctxToPrediction = new LinkedHashMap(); } reachTarget.ctxToPrediction.put(originalContext, predictedAlt); + if ( retry_debug ) { + System.out.println("adding edge upon "+getTokenName(t)); + System.out.println("DFA is "+dfa.toString(parser.getTokenNames())); + } } // System.out.println("RESOLVE to "+predictedAlt); //System.out.println(reachTarget.ctxToPrediction.size()+" size of ctx map"); @@ -815,8 +827,9 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull IntervalSet alts, @NotNull OrderedHashSet configs) { - if ( debug ) { - System.out.println("reportConflict "+alts+":"+configs); + if ( debug || retry_debug ) { + System.out.println("reportConflict "+alts+":"+configs+ + ", input="+getInputString(parser.getInputStream(), startIndex, stopIndex)); } if ( parser!=null ) parser.reportConflict(startIndex, stopIndex, alts, configs); } @@ -825,6 +838,10 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull IntervalSet alts, @NotNull OrderedHashSet configs) { + if ( debug || retry_debug ) { + System.out.println("reportContextSensitivity "+alts+":"+configs+ + ", input="+getInputString(parser.getInputStream(), startIndex, stopIndex)); + } if ( parser!=null ) parser.reportContextSensitivity(startIndex, stopIndex, alts, configs); } @@ -833,9 +850,10 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull IntervalSet ambigAlts, @NotNull OrderedHashSet configs) { - if ( debug ) { + if ( debug || retry_debug ) { System.out.println("reportAmbiguity "+ - ambigAlts+":"+configs); + ambigAlts+":"+configs+ + ", input="+getInputString(parser.getInputStream(), startIndex, stopIndex)); } if ( parser!=null ) parser.reportAmbiguity(startIndex, stopIndex, ambigAlts, configs); } @@ -845,9 +863,10 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull SemanticContext[] altToPred, @NotNull OrderedHashSet configs) { - if ( debug ) { + if ( debug || retry_debug ) { System.out.println("reportInsufficientPredicates "+ - ambigAlts+":"+Arrays.toString(altToPred)); + ambigAlts+":"+Arrays.toString(altToPred)+ + getInputString(parser.getInputStream(), startIndex, stopIndex)); } if ( parser!=null ) { parser.reportInsufficientPredicates(startIndex, stopIndex, ambigAlts, @@ -1081,6 +1100,7 @@ public class ParserATNSimulator extends ATNSimulator { DFAState to = addDFAState(dfa, q); if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t)); addDFAEdge(from, t, to); + if ( debug ) System.out.println("DFA=\n"+dfa.toString(parser.getTokenNames())); return to; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java index a3a874975..4d05a1e7e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java @@ -28,18 +28,19 @@ */ package org.antlr.v4.runtime.dfa; +import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; -import org.antlr.v4.runtime.atn.ATNState; -import java.util.*; +import java.util.LinkedHashMap; +import java.util.Map; public class DFA { /** A set of all DFA states. Use Map so we can get old state back * (Set only allows you to see if it's there). */ @NotNull - public final Map states = new LinkedHashMap(); + public final Map states = new LinkedHashMap(); @Nullable public DFAState s0; public int decision; @@ -69,4 +70,5 @@ public class DFA { DFASerializer serializer = new LexerDFASerializer(this); return serializer.toString(); } + } diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java index 48e44b34a..6a7e86969 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java @@ -32,6 +32,8 @@ package org.antlr.v4.runtime.dfa; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; +import java.util.Map; + /** A DFA walker that knows how to dump them to serialized strings. */ public class DFASerializer { @NotNull @@ -48,15 +50,18 @@ public class DFASerializer { public String toString() { if ( dfa.s0==null ) return null; StringBuilder buf = new StringBuilder(); - for (DFAState s : dfa.states.values()) { - int n = 0; - if ( s.edges!=null ) n = s.edges.length; - for (int i=0; i"+ getStateString(t)+'\n'); + Map states = dfa.states; + if ( states!=null ) { + for (DFAState s : states.values()) { + int n = 0; + if ( s.edges!=null ) n = s.edges.length; + for (int i=0; i"+ getStateString(t)+'\n'); + } } } } diff --git a/tool/playground/MyUListener.java b/tool/playground/MyUListener.java deleted file mode 100644 index 9e2ccd33c..000000000 --- a/tool/playground/MyUListener.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.Token; - -public class MyUListener extends BlankUListener { - UParser parser; - public MyUListener(UParser parser) { this.parser = parser; } - @Override - public void enterRule(UParser.aContext ctx) { - System.out.println("enter Rule"+ctx.toInfoString(parser)); - } - - @Override - public void exitRule(UParser.aContext ctx) { - System.out.println("exitRule "+ctx.toInfoString(parser)); - } - - @Override - public void visitTerminal(Token token) { - System.out.println("visitTerminal "+token); - } - - @Override - public void enterEveryRule(ParserRuleContext ctx) { - System.out.println("enterEvery Rule"+ctx.toInfoString(parser)); - } - - @Override - public void exitEveryRule(ParserRuleContext ctx) { - System.out.println("exitEvery Rule"+ctx.toInfoString(parser)); - } -} diff --git a/tool/playground/T.g b/tool/playground/T.g index adc99e194..93e6ab88b 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,5 +1,8 @@ grammar T; -s : (ID | ID ID?) ';' ; +s : FOO; +a : e B ; +b : e A B ; +e : A | ; ID : 'a'..'z'+ ; INT : '0'..'9'+; WS : (' '|'\n') {skip();} ; diff --git a/tool/playground/TestJavaLR.java b/tool/playground/TestJavaLR.java index 3cc1ed320..7804f654d 100644 --- a/tool/playground/TestJavaLR.java +++ b/tool/playground/TestJavaLR.java @@ -131,6 +131,7 @@ class TestJavaLR { if ( parser==null ) { parser = new JavaLRParser(null); if ( showTree ) parser.setBuildParseTree(true); + parser.getInterpreter().setContextSensitive(true); // parser.setErrorHandler(new BailErrorStrategy()); // parser.getInterpreter().setContextSensitive(true); } diff --git a/tool/playground/TestU.java b/tool/playground/TestU.java index 0952d29fa..5e1379adf 100644 --- a/tool/playground/TestU.java +++ b/tool/playground/TestU.java @@ -1,14 +1,15 @@ import org.antlr.v4.runtime.ANTLRFileStream; import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; public class TestU { public static void main(String[] args) throws Exception { ULexer t = new ULexer(new ANTLRFileStream(args[0])); CommonTokenStream tokens = new CommonTokenStream(t); UParser p = new UParser(tokens); - p.setListener(new MyUListener(p)); - p.s(); -// p.setBuildParseTree(true); -// UParser.aContext ctx = p.a(); + p.getInterpreter().setContextSensitive(true); + p.setBuildParseTree(true); + ParserRuleContext r = p.s(); + System.out.println(r.toStringTree(p)); } } diff --git a/tool/playground/U.g b/tool/playground/U.g index 3f12212f8..66c87ef11 100644 --- a/tool/playground/U.g +++ b/tool/playground/U.g @@ -1,6 +1,11 @@ grammar U; -s : a ; -a : ID (',' ID)* ';' ; +s : '{' stat* '}' ; +// if x then break else if y then return else break +// still ambig: if x then if y then break else break +stat: 'if' ID 'then' stat ('else' stat)? + | 'break' + | 'return' + ; INT : '0'..'9'+ ; ID : 'a'..'z'+ ;