diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java index 4cd7f3f43..28f253af9 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java @@ -122,6 +122,7 @@ public class ATNConfigSet implements Set { * We use (s,i,pi) as key */ public boolean add(ATNConfig config, @Nullable PredictionContextCache contextCache) { + //contextCache = null; // TODO: costs time to cache and saves essentially no RAM Key key = new Key(config); ATNConfig existing = configToContext.get(key); if ( existing==null ) { // nothing there yet; easy, just add diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java index adedf8602..78302d291 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java @@ -145,6 +145,7 @@ public class ArrayPredictionContext extends PredictionContext { else { next = new SingletonPredictionContext(this.parents[i], this.invokingStates[i]); + if ( contextCache!=null ) next = contextCache.add(next); } boolean rootIsWildcard = fullCtx; newCtx = merge(newCtx, next, contextCache, rootIsWildcard); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 47e0f1ac6..0809298dc 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -267,11 +267,17 @@ public class ParserATNSimulator extends ATNSimulator { * to {@code false} enables additional internal optimizations which may lose * this information. */ + // TODO: not sure we need public boolean reportAmbiguities = true; /** Do only local context prediction (SLL(k) style). */ public boolean SLL = false; + // LAME globals to avoid parameter during experiment!!!!! + protected TokenStream _input; + protected int _startIndex; + protected ParserRuleContext _outerContext; + /** Testing only! */ public ParserATNSimulator(@NotNull ATN atn) { this(null, atn); @@ -295,6 +301,14 @@ public class ParserATNSimulator extends ATNSimulator { public int adaptivePredict(@NotNull TokenStream input, int decision, @Nullable ParserRuleContext outerContext) { + if ( debug || debug_list_atn_decisions ) { + System.out.println("adaptivePredict decision "+decision+ + " exec LA(1)=="+ getLookaheadName(input)+ + " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); + } + _input = input; + _startIndex = input.index(); + _outerContext = outerContext; predict_calls++; DFA dfa = decisionToDFA[decision]; if ( dfa==null || dfa.s0==null ) { @@ -324,7 +338,7 @@ public class ParserATNSimulator extends ATNSimulator { contextCache = new PredictionContextCache("predict ctx cache"); if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( debug || debug_list_atn_decisions ) { - System.out.println("ATN decision "+dfa.decision+ + System.out.println("predictATN decision "+dfa.decision+ " exec LA(1)=="+ getLookaheadName(input) + ", outerContext="+outerContext.toString(parser)); } @@ -364,7 +378,7 @@ public class ParserATNSimulator extends ATNSimulator { { if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( dfa_debug ) { - System.out.println("DFA decision "+dfa.decision+ + System.out.println("execDFA decision "+dfa.decision+ " exec LA(1)=="+ getLookaheadName(input) + ", outerContext="+outerContext.toString(parser)); } @@ -534,7 +548,7 @@ public class ParserATNSimulator extends ATNSimulator { if ( debug || debug_list_atn_decisions) { System.out.println("execATN decision "+dfa.decision+ " exec LA(1)=="+ getLookaheadName(input)+ - "line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); + " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); } ATN_failover++; @@ -551,7 +565,12 @@ public class ParserATNSimulator extends ATNSimulator { while (true) { // while more work boolean loopsSimulateTailRecursion = false; - ATNConfigSet reach = computeReachSet(previous, t, greedy, +// System.out.println("REACH "+getLookaheadName(input)); + ATNConfigSet reach = computeReachSet(previous, t, + input, + startIndex, + outerContext, + greedy, loopsSimulateTailRecursion, false); if ( reach==null ) throw noViableAlt(input, outerContext, previous, startIndex); @@ -676,13 +695,18 @@ public class ParserATNSimulator extends ATNSimulator { retry_with_context++; reportAttemptingFullContext(dfa, s0, startIndex, input.index()); - if ( debug ) System.out.println("execATNWithFullContext "+s0+", greedy="+greedy); + if ( debug || debug_list_atn_decisions ) { + System.out.println("execATNWithFullContext "+s0+", greedy="+greedy); + } ATNConfigSet reach = null; ATNConfigSet previous = s0; input.seek(startIndex); int t = input.LA(1); while (true) { // while more work - reach = computeReachSet(previous, t, greedy, true, true); +// System.out.println("LL REACH "+getLookaheadName(input)+ +// " from configs.size="+previous.size()+ +// " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); + reach = computeReachSet(previous, t, input, startIndex, outerContext, greedy, true, true); if ( reach==null ) { throw noViableAlt(input, outerContext, previous, startIndex); } @@ -743,6 +767,9 @@ public class ParserATNSimulator extends ATNSimulator { } protected ATNConfigSet computeReachSet(ATNConfigSet closure, int t, + @NotNull TokenStream input, + int startIndex, + ParserRuleContext outerContext, boolean greedy, boolean loopsSimulateTailRecursion, boolean fullCtx) @@ -778,8 +805,44 @@ public class ParserATNSimulator extends ATNSimulator { reach.addAll(intermediate); } else { + // Ok, no unique alt, but now strip out all false preds to + // reduce load. This helps a lot in predicated configs from + // left-recursive expression rules. + + //------------- +// IntervalSet alts = evalSemanticContext(predPredictions, outerContext, reportAmbiguities); +// D.prediction = ATN.INVALID_ALT_NUMBER; +// switch (alts.size()) { +// case 0: +// throw noViableAlt(input, outerContext, D.configs, startIndex); +// +// case 1: +// return alts.getMinElement(); +// +// default: +// // report ambiguity after predicate evaluation to make sure the correct +// // set of ambig alts is reported. +// if (reportAmbiguities) { +// reportAmbiguity(dfa, D, startIndex, stopIndex, alts, D.configs); +// } +// +// return alts.getMinElement(); +// } + //--------------- + for (ATNConfig c : intermediate) { +// if ( c.semanticContext!=null && c.semanticContext!=SemanticContext.NONE ) { +// int currentPosition = input.index(); +// input.seek(startIndex); +// boolean predSucceeds = c.semanticContext.eval(parser, outerContext); +// input.seek(currentPosition); +// if ( !predSucceeds ) continue; +// } + // pred evaluated to true, must compute closure (will eval 2nd time after this func) + long start = System.currentTimeMillis(); closure(c, reach, closureBusy, false, greedy, loopsSimulateTailRecursion); + long stop = System.currentTimeMillis(); + //System.out.println("CLOSURE duration "+(stop-start)+": "+c); } } @@ -950,12 +1013,12 @@ public class ParserATNSimulator extends ATNSimulator { continue; } - boolean evaluatedResult = pair.pred.eval(parser, outerContext); + boolean predicateEvaluationResult = pair.pred.eval(parser, outerContext); if ( debug || dfa_debug ) { - System.out.println("eval pred "+pair+"="+evaluatedResult); + System.out.println("eval pred "+pair+"="+predicateEvaluationResult); } - if ( evaluatedResult ) { + if ( predicateEvaluationResult ) { if ( debug || dfa_debug ) System.out.println("PREDICT "+pair.alt); predictions.add(pair.alt); if (!complete) { @@ -1163,13 +1226,26 @@ public class ParserATNSimulator extends ATNSimulator { } } - ATNConfig c; - if ( collectPredicates && + ATNConfig c = null; + if ( collectPredicates && (!pt.isCtxDependent || (pt.isCtxDependent&&inContext)) ) { - SemanticContext newSemCtx = SemanticContext.and(config.semanticContext, pt.getPredicate()); - c = new ATNConfig(config, pt.target, newSemCtx); - } + SemanticContext newSemCtx = SemanticContext.and(config.semanticContext, pt.getPredicate()); + if ( SLL ) { + c = new ATNConfig(config, pt.target, newSemCtx); + } + else { + int currentPosition = _input.index(); + _input.seek(_startIndex); + boolean predSucceeds = pt.getPredicate().eval(parser, _outerContext); + _input.seek(currentPosition); + if ( predSucceeds ) { + // TODO: ignore semctx in config now? actually can only ignore in full LL mode + // REMOVE pred eval chk above? Actually no preds so it's a no-op + c = new ATNConfig(config, pt.target); // no pred context + } + } + } else { c = new ATNConfig(config, pt.target); } @@ -1261,6 +1337,7 @@ public class ParserATNSimulator extends ATNSimulator { @Nullable public IntervalSet getConflictingAlts(@NotNull ATNConfigSet configs, boolean fullCtx) { if ( debug ) System.out.println("### check ambiguous "+configs); +// System.out.println("getConflictingAlts; set size="+configs.size()); // First get a list of configurations for each state. // Most of the time, each state will have one associated configuration. MultiMap stateToConfigListMap = new MultiMap(); diff --git a/tool/playground/Foo.java b/tool/playground/Foo.java index 4c38e23b7..59fac88a3 100644 --- a/tool/playground/Foo.java +++ b/tool/playground/Foo.java @@ -4,7 +4,7 @@ class Foo { if (ch < 256 && !(3==4 && 5==6 && (ch == 0xff || ch == 0xb5 || - ch == 0x49 || ch == 0x69 || //I and i + ch == 0x49 || ch == 0x69 || //I and i // cmt out and it continues! ch == 0x53 || ch == 0x73 || //S and s ch == 0x4b || ch == 0x6b || //K and k ch == 0xc5 || ch == 0xe5))) //A+ring diff --git a/tool/playground/TestJavaLR.java b/tool/playground/TestJavaLR.java index 194920695..28ba63b77 100644 --- a/tool/playground/TestJavaLR.java +++ b/tool/playground/TestJavaLR.java @@ -28,6 +28,7 @@ */ import org.antlr.v4.runtime.ANTLRFileStream; +import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.DiagnosticErrorListener; import org.antlr.v4.runtime.ParserRuleContext; @@ -46,6 +47,7 @@ class TestJavaLR { public static boolean printTree = false; public static boolean SLL = false; public static boolean diag = false; + public static boolean bail = false; public static void main(String[] args) { doAll(args); @@ -62,6 +64,7 @@ class TestJavaLR { if ( args[i].equals("-tree") ) showTree = true; else if ( args[i].equals("-ptree") ) printTree = true; else if ( args[i].equals("-SLL") ) SLL = true; + else if ( args[i].equals("-bail") ) bail = true; else if ( args[i].equals("-diag") ) diag = true; doFile(new File(args[i])); // parse it } @@ -132,13 +135,12 @@ class TestJavaLR { // Create a parser that reads from the scanner if ( parser==null ) { parser = new JavaLRParser(null); -// parser.setErrorHandler(new BailErrorStrategy()); -// parser.getInterpreter().setContextSensitive(true); + if ( diag ) parser.addErrorListener(new DiagnosticErrorListener()); + if ( bail ) parser.setErrorHandler(new BailErrorStrategy()); + if ( SLL ) parser.getInterpreter().SLL = true; } parser.setTokenStream(tokens); - if ( diag ) parser.addErrorListener(new DiagnosticErrorListener()); - if ( SLL ) parser.getInterpreter().SLL = true; // start parsing at the compilationUnit rule ParserRuleContext tree = parser.compilationUnit(); if ( showTree ) tree.inspect(parser);