diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java index 97dd1636e..9998dd000 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java @@ -71,6 +71,7 @@ public class ATN { // runtime for lexer only public int[] ruleToTokenType; public int[] ruleToActionIndex; + @NotNull public final List modeToStartState = new ArrayList(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java index 0f56db603..1211bbc4c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java @@ -64,16 +64,18 @@ public abstract class ATNSimulator { * more time I think and doesn't save on the overall footprint * so it's not worth the complexity. */ - protected final PredictionContextCache sharedContextCache = - new PredictionContextCache("shared DFA state context cache"); + protected final PredictionContextCache sharedContextCache; static { ERROR = new DFAState(new ATNConfigSet()); ERROR.stateNumber = Integer.MAX_VALUE; } - public ATNSimulator(@NotNull ATN atn) { + public ATNSimulator(@NotNull ATN atn, + @NotNull PredictionContextCache sharedContextCache) + { this.atn = atn; + this.sharedContextCache = sharedContextCache; } public abstract void reset(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 710615180..8742d8a3e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -101,7 +101,7 @@ public class LexerATNSimulator extends ATNSimulator { protected int charPositionInLine = 0; @NotNull - public final DFA[] dfa; + public final DFA[] decisionToDFA; protected int mode = Lexer.DEFAULT_MODE; /** Used during DFA/ATN exec to record the most recent accept configuration info */ @@ -111,17 +111,26 @@ public class LexerATNSimulator extends ATNSimulator { public static int ATN_failover = 0; public static int match_calls = 0; - public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) { - this(null, atn, decisionToDFA); + public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA, + @NotNull PredictionContextCache sharedContextCache) + { + this(null, atn, decisionToDFA,sharedContextCache); } public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn, - @NotNull DFA[] decisionToDFA) + @NotNull DFA[] decisionToDFA, + @NotNull PredictionContextCache sharedContextCache) { - super(atn); - dfa = new DFA[atn.modeToStartState.size()]; - for (int i=0; i extends ATNSimulator { protected ParserRuleContext _outerContext; /** Testing only! */ - public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) { - this(null, atn, decisionToDFA); + public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA, + @NotNull PredictionContextCache sharedContextCache) + { + this(null, atn, decisionToDFA, sharedContextCache); } - public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) { - super(atn); + public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn, + @NotNull DFA[] decisionToDFA, + @NotNull PredictionContextCache sharedContextCache) + { + super(atn,sharedContextCache); this.parser = parser; this.decisionToDFA = decisionToDFA; // DOTGenerator dot = new DOTGenerator(null); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java index 0b12e9888..ee5da0a6a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java @@ -13,14 +13,15 @@ public class PredictionContextCache { protected Map cache = new HashMap(); - public PredictionContextCache(String name) { + public PredictionContextCache() { this.name = name; } /** Add a context to the cache and return it. If the context already exists, * return that one instead and do not add a new context to the cache. + * Protect shared cache from unsafe thread access. */ - public PredictionContext add(PredictionContext ctx) { + public synchronized PredictionContext add(PredictionContext ctx) { if ( ctx==PredictionContext.EMPTY ) return PredictionContext.EMPTY; PredictionContext existing = cache.get(ctx); if ( existing!=null ) { @@ -31,11 +32,11 @@ public class PredictionContextCache { return ctx; } - public PredictionContext get(PredictionContext ctx) { + public synchronized PredictionContext get(PredictionContext ctx) { return cache.get(ctx); } - public int size() { + public synchronized int size() { return cache.size(); } } diff --git a/tool/playground/TestJavaLR.java b/tool/playground/TestJavaLR.java index c86b00b2e..540fb3285 100644 --- a/tool/playground/TestJavaLR.java +++ b/tool/playground/TestJavaLR.java @@ -49,6 +49,8 @@ class TestJavaLR { public static boolean SLL = false; public static boolean diag = false; public static boolean bail = false; + public static boolean x2 = false; + public static boolean threaded = false; public static void main(String[] args) { doAll(args); @@ -68,6 +70,8 @@ class TestJavaLR { else if ( args[i].equals("-SLL") ) SLL = true; else if ( args[i].equals("-bail") ) bail = true; else if ( args[i].equals("-diag") ) diag = true; + else if ( args[i].equals("-2x") ) x2 = true; + else if ( args[i].equals("-threaded") ) threaded = true; doFile(new File(args[i])); // parse it } } @@ -151,6 +155,7 @@ class TestJavaLR { if ( printTree ) System.out.println(t.toStringTree(parser)); //System.err.println("finished "+f); // System.out.println("cache size = "+DefaultErrorStrategy.cache.size()); + lexer=null; parser=null; // force rebuild } } catch (Exception e) { diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index fab5251c1..ed83cce4b 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -119,6 +119,8 @@ Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= << @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public abstract class extends { protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); public static final int =}; separator=", ", wrap, anchor>; @@ -191,7 +193,7 @@ case : return _sempred(()_localctx, predIndex); parser_ctor(p) ::= << public (TokenStream input) { super(input); - _interp = new ParserATNSimulator\(this,_ATN,_decisionToDFA); + _interp = new ParserATNSimulator\(this,_ATN,_decisionToDFA,_sharedContextCache); } >> @@ -716,6 +718,8 @@ Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= << @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public abstract class extends Lexer { protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); public static final int =}; separator=", ", wrap, anchor>; = ;}; separator="\n"> @@ -735,7 +739,7 @@ public abstract class extends public (CharStream input) { super(input); - _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA); + _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); } @Override diff --git a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java index 9ae4544fa..f48e72944 100644 --- a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java +++ b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java @@ -56,7 +56,7 @@ public class LexerInterpreter implements TokenSource { public LexerInterpreter(LexerGrammar g) { Tool antlr = new Tool(); antlr.process(g,false); - interp = new LexerATNSimulator(g.atn,null); + interp = new LexerATNSimulator(g.atn,null,null); } public void setInput(String inputString) { diff --git a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java index 796e4694b..70e3b089e 100644 --- a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java +++ b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java @@ -38,6 +38,7 @@ import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.DecisionState; import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; @@ -46,6 +47,9 @@ import org.antlr.v4.tool.Grammar; public class ParserInterpreter { public static class DummyParser extends Parser { public final DFA[] decisionToDFA; // not shared for interp + public final PredictionContextCache sharedContextCache = + new PredictionContextCache(); + public Grammar g; public DummyParser(Grammar g, TokenStream input) { super(input); @@ -89,7 +93,9 @@ public class ParserInterpreter { Tool antlr = new Tool(); antlr.process(g,false); parser = new DummyParser(g, input); - atnSimulator = new ParserATNSimulator(parser, g.atn, parser.decisionToDFA); + atnSimulator = + new ParserATNSimulator(parser, g.atn, parser.decisionToDFA, + parser.sharedContextCache); } public synchronized int predictATN(@NotNull DFA dfa, @NotNull TokenStream input, diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index 19d3a7f39..7c319e081 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -211,7 +211,7 @@ public abstract class BaseTest { CharStream input, boolean adaptive) { - LexerATNSimulator interp = new LexerATNSimulator(atn,null); + LexerATNSimulator interp = new LexerATNSimulator(atn,null,null); List tokenTypes = new ArrayList(); int ttype; boolean hitEOF = false; diff --git a/tool/test/org/antlr/v4/test/TestATNInterpreter.java b/tool/test/org/antlr/v4/test/TestATNInterpreter.java index cd429aac1..8861c0506 100644 --- a/tool/test/org/antlr/v4/test/TestATNInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNInterpreter.java @@ -273,7 +273,7 @@ public class TestATNInterpreter extends BaseTest { int expected) { ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 5e5db1edc..0f5a98af1 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -479,7 +479,7 @@ public class TestATNParserPrediction extends BaseTest { { Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); @@ -531,7 +531,7 @@ public class TestATNParserPrediction extends BaseTest { // sync to ensure multiple tests don't race on dfa access Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null); semanticProcess(lg); g.importVocab(lg); @@ -546,7 +546,7 @@ public class TestATNParserPrediction extends BaseTest { // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); ParserATNSimulator interp = - new ParserATNSimulator(atn, new DFA[atn.getNumberOfDecisions()+1]); + new ParserATNSimulator(atn, new DFA[atn.getNumberOfDecisions()],null); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); TokenStream input = new IntTokenStream(types); @@ -568,7 +568,7 @@ public class TestATNParserPrediction extends BaseTest { { // Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null); semanticProcess(lg); g.importVocab(lg); diff --git a/tool/test/org/antlr/v4/test/TestGraphNodes.java b/tool/test/org/antlr/v4/test/TestGraphNodes.java index 82a5027d4..0e00bd55d 100644 --- a/tool/test/org/antlr/v4/test/TestGraphNodes.java +++ b/tool/test/org/antlr/v4/test/TestGraphNodes.java @@ -13,7 +13,7 @@ public class TestGraphNodes extends TestCase { @Override protected void setUp() throws Exception { PredictionContext.globalNodeCount = 1; - contextCache = new PredictionContextCache("testing"); + contextCache = new PredictionContextCache(); } public boolean rootIsWildcard() { return true; } diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java index ad76835e0..726613dfa 100644 --- a/tool/test/org/antlr/v4/test/TestPerformance.java +++ b/tool/test/org/antlr/v4/test/TestPerformance.java @@ -317,7 +317,7 @@ public class TestPerformance extends BaseTest { System.currentTimeMillis() - startTime); final LexerATNSimulator lexerInterpreter = sharedLexer.getInterpreter(); - final DFA[] modeToDFA = lexerInterpreter.dfa; + final DFA[] modeToDFA = lexerInterpreter.decisionToDFA; if (SHOW_DFA_STATE_STATS) { int states = 0; int configs = 0;