forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

added more dbg prints, turned on context sensitivity in test java lr. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9545]
This commit is contained in:
parrt 2011-12-07 17:47:08 -08:00
parent 9173367a1e
commit 63168c5577
8 changed files with 75 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -46,6 +46,7 @@ import java.util.*;
public class ParserATNSimulator<Symbol> extends ATNSimulator {
public static boolean debug = false;
public static boolean dfa_debug = false;
public static boolean retry_debug = false;
public static int ATN_failover = 0;
public static int predict_calls = 0;
@ -122,7 +123,6 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
}
public void reset() {
userWantsCtxSensitive = false;
outerContext = ParserRuleContext.EMPTY;
prevAccept = null;
prevAcceptIndex = -1;
@ -161,7 +161,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
input.seek(index);
input.release(m);
}
if ( debug ) System.out.println("DFA after predictATN: "+dfa.toString());
if ( debug ) System.out.println("DFA after predictATN: "+dfa.toString(parser.getTokenNames()));
return alt;
}
@ -181,12 +181,12 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull DFAState s0,
@Nullable ParserRuleContext outerContext)
{
// dump(dfa);
if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
this.outerContext = outerContext;
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" exec LA(1)=="+ getLookaheadName(input) +
", outerContext="+outerContext.toString(parser));
if ( dfa_debug ) System.out.print(dfa.toString(parser.getTokenNames()));
DFAState prevAcceptState = null;
DFAState s = s0;
int t = input.LA(1);
@ -243,7 +243,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
}
}
if ( dfa_debug ) {
System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa);
System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa.toString(parser.getTokenNames()));
//dump(dfa);
}
// action already executed
@ -288,11 +288,15 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
}
public String getInputString(@NotNull SymbolStream<Symbol> input, int start) {
return getInputString(input, start, input.index());
}
public String getInputString(@NotNull SymbolStream<Symbol> input, int start, int stop) {
if ( input instanceof TokenStream ) {
return ((TokenStream)input).toString(start,input.index());
return ((TokenStream)input).toString(start,stop);
}
else if ( input instanceof ASTNodeStream) {
return ((ASTNodeStream<Symbol>)input).toString(input.get(start),input.LT(1));
return ((ASTNodeStream<Symbol>)input).toString(input.get(start),input.get(stop));
}
return "n/a";
}
@ -355,6 +359,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
getInputString(input, startIndex));
System.out.println("REACH="+reach);
}
// System.out.println("AMBIG dec "+dfa.decision+" for alt "+ambigAlts+" upon "+
// getInputString(input, startIndex));
// System.out.println("userWantsCtxSensitive="+userWantsCtxSensitive);
// can we resolve with predicates?
SemanticContext[] altToPred =
@ -544,10 +551,11 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
retry_with_context++;
int old_k = input.index();
// retry using context, if any; if none, kill all but min as before
if ( debug ) System.out.println("RETRY "+ getInputString(input, startIndex) +
" with ctx="+ originalContext);
if ( retry_debug ) System.out.println("RETRY '"+ getInputString(input, startIndex) +
"' with ctx="+ originalContext);
int min = ambigAlts.getMinElement();
if ( originalContext==ParserRuleContext.EMPTY ) {
if ( retry_debug ) System.out.println("ctx empty; no need to retry");
// no point in retrying with ctx since it's same.
// this implies that we have a true ambiguity
reportAmbiguity(startIndex, input.index(), ambigAlts, reach);
@ -558,10 +566,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
input.seek(startIndex); // rewind
DFA ctx_dfa = new DFA(dfa.atnStartState);
int ctx_alt = predictATN(ctx_dfa, input, originalContext, true);
if ( debug ) System.out.println("retry predicts "+ctx_alt+" vs "+ambigAlts.getMinElement()+
if ( retry_debug ) System.out.println("retry predicts "+ctx_alt+" vs "+ambigAlts.getMinElement()+
" with conflict="+ctx_dfa.conflict+
" dfa="+ctx_dfa);
" full ctx dfa="+ctx_dfa.toString(parser.getTokenNames()));
if ( ctx_dfa.conflict ) {
// System.out.println("retry gives ambig for "+input.toString(startIndex, input.index()));
@ -571,7 +578,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// System.out.println("NO ambig for "+input.toString(startIndex, input.index()));
// System.out.println(ctx_dfa.toString(parser.getTokenNames()));
if ( old_k != input.index() ) {
System.out.println("ACK!!!!!!!! diff k; old="+(old_k-startIndex+1)+", new="+(input.index()-startIndex+1));
if ( retry_debug ) System.out.println("used diff amount of k; old="+(old_k-startIndex+1)+", new="+(input.index()-startIndex+1));
}
retry_with_context_indicates_no_conflict++;
reportContextSensitivity(startIndex, input.index(), ambigAlts, reach);
@ -581,13 +588,18 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// TODO: if ambig, why turn on ctx sensitive?
int predictedAlt = ctx_alt;
if ( buildDFA) {
if ( buildDFA ) {
DFAState reachTarget = addDFAEdge(dfa, closure, t, reach);
makeAcceptState(reachTarget, ctx_alt);
reachTarget.isCtxSensitive = true;
if ( reachTarget.ctxToPrediction==null ) {
reachTarget.ctxToPrediction = new LinkedHashMap<RuleContext, Integer>();
}
reachTarget.ctxToPrediction.put(originalContext, predictedAlt);
if ( retry_debug ) {
System.out.println("adding edge upon "+getTokenName(t));
System.out.println("DFA is "+dfa.toString(parser.getTokenNames()));
}
}
// System.out.println("RESOLVE to "+predictedAlt);
//System.out.println(reachTarget.ctxToPrediction.size()+" size of ctx map");
@ -815,8 +827,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull IntervalSet alts,
@NotNull OrderedHashSet<ATNConfig> configs)
{
if ( debug ) {
System.out.println("reportConflict "+alts+":"+configs);
if ( debug || retry_debug ) {
System.out.println("reportConflict "+alts+":"+configs+
", input="+getInputString(parser.getInputStream(), startIndex, stopIndex));
}
if ( parser!=null ) parser.reportConflict(startIndex, stopIndex, alts, configs);
}
@ -825,6 +838,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull IntervalSet alts,
@NotNull OrderedHashSet<ATNConfig> configs)
{
if ( debug || retry_debug ) {
System.out.println("reportContextSensitivity "+alts+":"+configs+
", input="+getInputString(parser.getInputStream(), startIndex, stopIndex));
}
if ( parser!=null ) parser.reportContextSensitivity(startIndex, stopIndex, alts, configs);
}
@ -833,9 +850,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull IntervalSet ambigAlts,
@NotNull OrderedHashSet<ATNConfig> configs)
{
if ( debug ) {
if ( debug || retry_debug ) {
System.out.println("reportAmbiguity "+
ambigAlts+":"+configs);
ambigAlts+":"+configs+
", input="+getInputString(parser.getInputStream(), startIndex, stopIndex));
}
if ( parser!=null ) parser.reportAmbiguity(startIndex, stopIndex, ambigAlts, configs);
}
@ -845,9 +863,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull SemanticContext[] altToPred,
@NotNull OrderedHashSet<ATNConfig> configs)
{
if ( debug ) {
if ( debug || retry_debug ) {
System.out.println("reportInsufficientPredicates "+
ambigAlts+":"+Arrays.toString(altToPred));
ambigAlts+":"+Arrays.toString(altToPred)+
getInputString(parser.getInputStream(), startIndex, stopIndex));
}
if ( parser!=null ) {
parser.reportInsufficientPredicates(startIndex, stopIndex, ambigAlts,
@ -1081,6 +1100,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
DFAState to = addDFAState(dfa, q);
if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
addDFAEdge(from, t, to);
if ( debug ) System.out.println("DFA=\n"+dfa.toString(parser.getTokenNames()));
return to;
}

8
runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java
View File

@ -28,18 +28,19 @@
*/
package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.atn.ATNState;
import java.util.*;
import java.util.LinkedHashMap;
import java.util.Map;
public class DFA {
/** A set of all DFA states. Use Map so we can get old state back
* (Set only allows you to see if it's there).
*/
@NotNull
public final Map<DFAState, DFAState> states = new LinkedHashMap<DFAState, DFAState>();
public final Map<DFAState, DFAState> states = new LinkedHashMap<DFAState, DFAState>();
@Nullable
public DFAState s0;
public int decision;
@ -69,4 +70,5 @@ public class DFA {
DFASerializer serializer = new LexerDFASerializer(this);
return serializer.toString();
}
}

23
runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java
View File

@ -32,6 +32,8 @@ package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.Map;
/** A DFA walker that knows how to dump them to serialized strings. */
public class DFASerializer {
@NotNull
@ -48,15 +50,18 @@ public class DFASerializer {
public String toString() {
if ( dfa.s0==null ) return null;
StringBuilder buf = new StringBuilder();
for (DFAState s : dfa.states.values()) {
int n = 0;
if ( s.edges!=null ) n = s.edges.length;
for (int i=0; i<n; i++) {
DFAState t = s.edges[i];
if ( t!=null && t.stateNumber != Integer.MAX_VALUE ) {
buf.append(getStateString(s));
String label = getEdgeLabel(i);
buf.append("-"+label+"->"+ getStateString(t)+'\n');
Map<DFAState,DFAState> states = dfa.states;
if ( states!=null ) {
for (DFAState s : states.values()) {
int n = 0;
if ( s.edges!=null ) n = s.edges.length;
for (int i=0; i<n; i++) {
DFAState t = s.edges[i];
if ( t!=null && t.stateNumber != Integer.MAX_VALUE ) {
buf.append(getStateString(s));
String label = getEdgeLabel(i);
buf.append("-"+label+"->"+ getStateString(t)+'\n');
}
}
}
}

60
tool/playground/MyUListener.java
View File

@ -1,60 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
public class MyUListener extends BlankUListener {
UParser parser;
public MyUListener(UParser parser) { this.parser = parser; }
@Override
public void enterRule(UParser.aContext ctx) {
System.out.println("enter Rule"+ctx.toInfoString(parser));
}
@Override
public void exitRule(UParser.aContext ctx) {
System.out.println("exitRule "+ctx.toInfoString(parser));
}
@Override
public void visitTerminal(Token token) {
System.out.println("visitTerminal "+token);
}
@Override
public void enterEveryRule(ParserRuleContext<Token> ctx) {
System.out.println("enterEvery Rule"+ctx.toInfoString(parser));
}
@Override
public void exitEveryRule(ParserRuleContext<Token> ctx) {
System.out.println("exitEvery Rule"+ctx.toInfoString(parser));
}
}

5
tool/playground/T.g
View File

@ -1,5 +1,8 @@
grammar T;
s : (ID | ID ID?) ';' ;
s : FOO;
a : e B ;
b : e A B ;
e : A | ;
ID : 'a'..'z'+ ;
INT : '0'..'9'+;
WS : (' '|'\n') {skip();} ;

1
tool/playground/TestJavaLR.java
View File

@ -131,6 +131,7 @@ class TestJavaLR {
if ( parser==null ) {
parser = new JavaLRParser(null);
if ( showTree ) parser.setBuildParseTree(true);
parser.getInterpreter().setContextSensitive(true);
// parser.setErrorHandler(new BailErrorStrategy<Token>());
// parser.getInterpreter().setContextSensitive(true);
}

9
tool/playground/TestU.java
View File

@ -1,14 +1,15 @@
import org.antlr.v4.runtime.ANTLRFileStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
public class TestU {
public static void main(String[] args) throws Exception {
ULexer t = new ULexer(new ANTLRFileStream(args[0]));
CommonTokenStream tokens = new CommonTokenStream(t);
UParser p = new UParser(tokens);
p.setListener(new MyUListener(p));
p.s();
// p.setBuildParseTree(true);
// UParser.aContext ctx = p.a();
p.getInterpreter().setContextSensitive(true);
p.setBuildParseTree(true);
ParserRuleContext r = p.s();
System.out.println(r.toStringTree(p));
}
}

9
tool/playground/U.g
View File

@ -1,6 +1,11 @@
grammar U;
s : a ;
a : ID (',' ID)* ';' ;
s : '{' stat* '}' ;
// if x then break else if y then return else break
// still ambig: if x then if y then break else break
stat: 'if' ID 'then' stat ('else' stat)?
| 'break'
| 'return'
;
INT : '0'..'9'+ ;
ID : 'a'..'z'+ ;