synchronized lexer. just as fast now to build new parser/lexer each iter in TestJavaLR

runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java

@@ -71,6 +71,7 @@ public class ATN {
 	// runtime for lexer only
 	public int[] ruleToTokenType;
 	public int[] ruleToActionIndex;
+
 	@NotNull
 	public final List<TokensStartState> modeToStartState = new ArrayList<TokensStartState>();
 

runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java

@@ -64,16 +64,18 @@ public abstract class ATNSimulator {
 	 *  more time I think and doesn't save on the overall footprint
 	 *  so it's not worth the complexity.
  	 */
-	protected final PredictionContextCache sharedContextCache =
+	protected final PredictionContextCache sharedContextCache;
-		new PredictionContextCache("shared DFA state context cache");
 
 	static {
 		ERROR = new DFAState(new ATNConfigSet());
 		ERROR.stateNumber = Integer.MAX_VALUE;
 	}
 
-	public ATNSimulator(@NotNull ATN atn) {
+	public ATNSimulator(@NotNull ATN atn,
+						@NotNull PredictionContextCache sharedContextCache)
+	{
 		this.atn = atn;
+		this.sharedContextCache = sharedContextCache;
 	}
 
 	public abstract void reset();

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -101,7 +101,7 @@ public class LexerATNSimulator extends ATNSimulator {
 	protected int charPositionInLine = 0;
 
 	@NotNull
-	public final DFA[] dfa;
+	public final DFA[] decisionToDFA;
 	protected int mode = Lexer.DEFAULT_MODE;
 
 	/** Used during DFA/ATN exec to record the most recent accept configuration info */
@@ -111,17 +111,26 @@ public class LexerATNSimulator extends ATNSimulator {
 	public static int ATN_failover = 0;
 	public static int match_calls = 0;
 
-	public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
+	public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA,
-		this(null, atn, decisionToDFA);
+							 @NotNull PredictionContextCache sharedContextCache)
+	{
+		this(null, atn, decisionToDFA,sharedContextCache);
 	}
 
 	public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn,
-							 @NotNull DFA[] decisionToDFA)
+							 @NotNull DFA[] decisionToDFA,
+							 @NotNull PredictionContextCache sharedContextCache)
 	{
-		super(atn);
+		super(atn,sharedContextCache);
-		dfa = new DFA[atn.modeToStartState.size()];
+		this.decisionToDFA = decisionToDFA;
-		for (int i=0; i<atn.modeToStartState.size(); i++) {
+		if ( decisionToDFA[Lexer.DEFAULT_MODE]==null ) { // create all mode dfa
-			dfa[i] = new DFA(atn.modeToStartState.get(i));
+			synchronized (this.decisionToDFA) {
+				if ( decisionToDFA[Lexer.DEFAULT_MODE]==null ) { // create all mode dfa
+					for (int i=0; i<atn.modeToStartState.size(); i++) {
+						this.decisionToDFA[i] = new DFA(atn.modeToStartState.get(i));
+					}
+				}
+			}
 		}
 		this.recog = recog;
 	}
@@ -153,14 +162,19 @@ public class LexerATNSimulator extends ATNSimulator {
 		int mark = input.mark();
 		traceBeginMatch(input, mode);
 		try {
-			if ( dfa[mode].s0==null ) {
+			synchronized (decisionToDFA[mode]) {
-				return matchATN(input);
+				// Synchronizing on the mode DFA causes a lot of contention
-			}
+				// between shared lexers in multiple threads, but is the
-			else {
+				// simplest safety measure. We can start with it.
-				return execDFA(input, dfa[mode].s0);
+				if ( decisionToDFA[mode].s0==null ) {
+					return matchATN(input);
+				}
+				else {
+					return execDFA(input, decisionToDFA[mode].s0);
+				}
 			}
 		}
-        finally {
+		finally {
 			traceEndMatch();
 			input.release(mark);
 		}
@@ -188,11 +202,11 @@ public class LexerATNSimulator extends ATNSimulator {
 
 		ATNConfigSet s0_closure = computeStartState(input, startState);
 		int old_mode = mode;
-		dfa[mode].s0 = addDFAState(s0_closure);
+		decisionToDFA[mode].s0 = addDFAState(s0_closure);
-		int predict = execATN(input, s0_closure, dfa[mode].s0);
+		int predict = execATN(input, s0_closure, decisionToDFA[mode].s0);
 
 		if ( debug ) {
-			System.out.format("DFA after matchATN: %s\n", dfa[old_mode].toLexerString());
+			System.out.format("DFA after matchATN: %s\n", decisionToDFA[old_mode].toLexerString());
 		}
 
 		tracePredict(predict);
@@ -642,7 +656,7 @@ public class LexerATNSimulator extends ATNSimulator {
 
 		if ( dfa_debug ) {
 			System.out.format("back from DFA update, ttype=%d, dfa[mode %d]=\n%s\n",
-				ttype, mode, dfa[mode].toLexerString());
+							  ttype, mode, decisionToDFA[mode].toLexerString());
 		}
 
 		// action already executed by ATN
@@ -714,7 +728,7 @@ public class LexerATNSimulator extends ATNSimulator {
 		if ( configs.hasSemanticContext ) return null;
 
 		DFAState proposed = new DFAState(configs);
-		DFAState existing = dfa[mode].states.get(proposed);
+		DFAState existing = decisionToDFA[mode].states.get(proposed);
 		if ( existing!=null ) return existing;
 
 		DFAState newState = proposed;
@@ -735,16 +749,16 @@ public class LexerATNSimulator extends ATNSimulator {
 			newState.prediction = atn.ruleToTokenType[newState.lexerRuleIndex];
 		}
 
-		newState.stateNumber = dfa[mode].states.size();
+		newState.stateNumber = decisionToDFA[mode].states.size();
 		configs.setReadonly(true);
 		newState.configs = configs;
-		dfa[mode].states.put(newState, newState);
+		decisionToDFA[mode].states.put(newState, newState);
 		return newState;
 	}
 
 	@Nullable
 	public DFA getDFA(int mode) {
-		return dfa[mode];
+		return decisionToDFA[mode];
 	}
 
 	/** Get the text of the current token from an *action* in lexer not

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -150,12 +150,17 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	protected ParserRuleContext<?> _outerContext;
 
 	/** Testing only! */
-	public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
+	public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA,
-		this(null, atn, decisionToDFA);
+							  @NotNull PredictionContextCache sharedContextCache)
+	{
+		this(null, atn, decisionToDFA, sharedContextCache);
 	}
 
-	public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
+	public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn,
-		super(atn);
+							  @NotNull DFA[] decisionToDFA,
+							  @NotNull PredictionContextCache sharedContextCache)
+	{
+		super(atn,sharedContextCache);
 		this.parser = parser;
 		this.decisionToDFA = decisionToDFA;
 		//		DOTGenerator dot = new DOTGenerator(null);

runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java

@@ -13,14 +13,15 @@ public class PredictionContextCache {
 	protected Map<PredictionContext, PredictionContext> cache =
 		new HashMap<PredictionContext, PredictionContext>();
 
-	public PredictionContextCache(String name) {
+	public PredictionContextCache() {
 		this.name = name;
 	}
 
 	/** Add a context to the cache and return it. If the context already exists,
 	 *  return that one instead and do not add a new context to the cache.
+	 *  Protect shared cache from unsafe thread access.
 	 */
-	public PredictionContext add(PredictionContext ctx) {
+	public synchronized PredictionContext add(PredictionContext ctx) {
 		if ( ctx==PredictionContext.EMPTY ) return PredictionContext.EMPTY;
 		PredictionContext existing = cache.get(ctx);
 		if ( existing!=null ) {
@@ -31,11 +32,11 @@ public class PredictionContextCache {
 		return ctx;
 	}
 
-	public PredictionContext get(PredictionContext ctx) {
+	public synchronized PredictionContext get(PredictionContext ctx) {
 		return cache.get(ctx);
 	}
 
-	public int size() {
+	public synchronized int size() {
 		return cache.size();
 	}
 }

tool/playground/TestJavaLR.java

@@ -49,6 +49,8 @@ class TestJavaLR {
 	public static boolean SLL = false;
 	public static boolean diag = false;
 	public static boolean bail = false;
+	public static boolean x2 = false;
+	public static boolean threaded = false;
 
 	public static void main(String[] args) {
 		doAll(args);
@@ -68,6 +70,8 @@ class TestJavaLR {
 					else if ( args[i].equals("-SLL") ) SLL = true;
 					else if ( args[i].equals("-bail") ) bail = true;
 					else if ( args[i].equals("-diag") ) diag = true;
+					else if ( args[i].equals("-2x") ) x2 = true;
+					else if ( args[i].equals("-threaded") ) threaded = true;
 					doFile(new File(args[i])); // parse it
 				}
 			}
@@ -151,6 +155,7 @@ class TestJavaLR {
 				if ( printTree ) System.out.println(t.toStringTree(parser));
 				//System.err.println("finished "+f);
 //                System.out.println("cache size = "+DefaultErrorStrategy.cache.size());
+				lexer=null; parser=null; // force rebuild
 			}
 		}
 		catch (Exception e) {

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -119,6 +119,8 @@ Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= <<
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
 public <if(parser.abstractRecognizer)>abstract <endif>class <parser.name> extends <superclass> {
 	protected static final DFA[] _decisionToDFA;
+	protected static final PredictionContextCache _sharedContextCache =
+		new PredictionContextCache();
 	<if(parser.tokens)>
 	public static final int
 		<parser.tokens:{k | <k>=<parser.tokens.(k)>}; separator=", ", wrap, anchor>;
@@ -191,7 +193,7 @@ case <f.ruleIndex> : return <f.name>_sempred((<f.ctxType>)_localctx, predIndex);
 parser_ctor(p) ::= <<
 public <p.name>(TokenStream input) {
 	super(input);
-	_interp = new ParserATNSimulator\<Token>(this,_ATN,_decisionToDFA);
+	_interp = new ParserATNSimulator\<Token>(this,_ATN,_decisionToDFA,_sharedContextCache);
 }
 >>
 
@@ -716,6 +718,8 @@ Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= <<
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
 public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends Lexer {
 	protected static final DFA[] _decisionToDFA;
+	protected static final PredictionContextCache _sharedContextCache =
+		new PredictionContextCache();
 	public static final int
 		<lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>;
 	<rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n">
@@ -735,7 +739,7 @@ public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends
 
 	public <lexer.name>(CharStream input) {
 		super(input);
-		_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA);
+		_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
 	}
 
 	@Override

tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java

@@ -56,7 +56,7 @@ public class LexerInterpreter implements TokenSource {
 	public LexerInterpreter(LexerGrammar g) {
 		Tool antlr = new Tool();
 		antlr.process(g,false);
-		interp = new LexerATNSimulator(g.atn,null);
+		interp = new LexerATNSimulator(g.atn,null,null);
 	}
 
 	public void setInput(String inputString) {

tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java

@@ -38,6 +38,7 @@ import org.antlr.v4.runtime.atn.ATN;
 import org.antlr.v4.runtime.atn.ATNState;
 import org.antlr.v4.runtime.atn.DecisionState;
 import org.antlr.v4.runtime.atn.ParserATNSimulator;
+import org.antlr.v4.runtime.atn.PredictionContextCache;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
@@ -46,6 +47,9 @@ import org.antlr.v4.tool.Grammar;
 public class ParserInterpreter {
 	public static class DummyParser extends Parser {
 		public final DFA[] decisionToDFA; // not shared for interp
+		public final PredictionContextCache sharedContextCache =
+			new PredictionContextCache();
+
 		public Grammar g;
 		public DummyParser(Grammar g, TokenStream input) {
 			super(input);
@@ -89,7 +93,9 @@ public class ParserInterpreter {
 		Tool antlr = new Tool();
 		antlr.process(g,false);
 		parser = new DummyParser(g, input);
-		atnSimulator = new ParserATNSimulator<Token>(parser, g.atn, parser.decisionToDFA);
+		atnSimulator =
+			new ParserATNSimulator<Token>(parser, g.atn, parser.decisionToDFA,
+										  parser.sharedContextCache);
 	}
 
 	public synchronized int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,

tool/test/org/antlr/v4/test/BaseTest.java

@@ -211,7 +211,7 @@ public abstract class BaseTest {
 									  CharStream input,
 									  boolean adaptive)
 	{
-		LexerATNSimulator interp = new LexerATNSimulator(atn,null);
+		LexerATNSimulator interp = new LexerATNSimulator(atn,null,null);
 		List<String> tokenTypes = new ArrayList<String>();
 		int ttype;
 		boolean hitEOF = false;

tool/test/org/antlr/v4/test/TestATNInterpreter.java

@@ -273,7 +273,7 @@ public class TestATNInterpreter extends BaseTest {
 								int expected)
 	{
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
 		List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
 		System.out.println(types);
 

tool/test/org/antlr/v4/test/TestATNParserPrediction.java

@@ -479,7 +479,7 @@ public class TestATNParserPrediction extends BaseTest {
 	{
 		Tool.internalOption_ShowATNConfigsInDFA = true;
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
 		List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
 		System.out.println(types);
 
@@ -531,7 +531,7 @@ public class TestATNParserPrediction extends BaseTest {
 		// sync to ensure multiple tests don't race on dfa access
 		Tool.internalOption_ShowATNConfigsInDFA = true;
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
 
 		semanticProcess(lg);
 		g.importVocab(lg);
@@ -546,7 +546,7 @@ public class TestATNParserPrediction extends BaseTest {
 //		System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));
 
 		ParserATNSimulator<Token> interp =
-			new ParserATNSimulator<Token>(atn, new DFA[atn.getNumberOfDecisions()+1]);
+			new ParserATNSimulator<Token>(atn, new DFA[atn.getNumberOfDecisions()],null);
 		List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
 		System.out.println(types);
 		TokenStream input = new IntTokenStream(types);
@@ -568,7 +568,7 @@ public class TestATNParserPrediction extends BaseTest {
 	{
 //		Tool.internalOption_ShowATNConfigsInDFA = true;
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
 
 		semanticProcess(lg);
 		g.importVocab(lg);

tool/test/org/antlr/v4/test/TestGraphNodes.java

@@ -13,7 +13,7 @@ public class TestGraphNodes extends TestCase {
 	@Override
 	protected void setUp() throws Exception {
 		PredictionContext.globalNodeCount = 1;
-		contextCache = new PredictionContextCache("testing");
+		contextCache = new PredictionContextCache();
 	}
 
 	public boolean rootIsWildcard() { return true; }

tool/test/org/antlr/v4/test/TestPerformance.java

@@ -317,7 +317,7 @@ public class TestPerformance extends BaseTest {
                           System.currentTimeMillis() - startTime);
 
 		final LexerATNSimulator lexerInterpreter = sharedLexer.getInterpreter();
-		final DFA[] modeToDFA = lexerInterpreter.dfa;
+		final DFA[] modeToDFA = lexerInterpreter.decisionToDFA;
 		if (SHOW_DFA_STATE_STATS) {
 			int states = 0;
 			int configs = 0;