Merge pull request #360 from parrt/master

mv to runtime. damn git! Seems that it might delete but not move files. hmm...

tool/src/org/antlr/v4/tool/Grammar.java

@@ -41,6 +41,8 @@ import org.antlr.v4.parse.GrammarTreeVisitor;
 import org.antlr.v4.parse.TokenVocabParser;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.LexerInterpreter;
+import org.antlr.v4.runtime.ParserInterpreter;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenStream;
 import org.antlr.v4.runtime.atn.ATN;
@@ -57,8 +59,6 @@ import org.antlr.v4.tool.ast.GrammarASTWithOptions;
 import org.antlr.v4.tool.ast.GrammarRootAST;
 import org.antlr.v4.tool.ast.PredAST;
 import org.antlr.v4.tool.ast.TerminalAST;
-import org.antlr.v4.tool.interp.LexerInterpreter;
-import org.antlr.v4.tool.interp.ParserInterpreter;
 
 import java.io.IOException;
 import java.util.ArrayList;

tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java

@@ -1,99 +0,0 @@
-/*
- * [The "BSD license"]
- *  Copyright (c) 2012 Terence Parr
- *  Copyright (c) 2012 Sam Harwell
- *  All rights reserved.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *  1. Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *  2. Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *  3. The name of the author may not be used to endorse or promote products
- *     derived from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package org.antlr.v4.tool.interp;
-
-import org.antlr.v4.runtime.CharStream;
-import org.antlr.v4.runtime.Lexer;
-import org.antlr.v4.runtime.atn.ATN;
-import org.antlr.v4.runtime.atn.ATNType;
-import org.antlr.v4.runtime.atn.LexerATNSimulator;
-import org.antlr.v4.runtime.atn.PredictionContextCache;
-import org.antlr.v4.runtime.dfa.DFA;
-
-import java.util.Collection;
-
-public class LexerInterpreter extends Lexer {
-	protected final String grammarFileName;
-	protected final ATN atn;
-
-	protected final String[] tokenNames;
-	protected final String[] ruleNames;
-	protected final String[] modeNames;
-
-	protected final DFA[] _decisionToDFA;
-	protected final PredictionContextCache _sharedContextCache =
-		new PredictionContextCache();
-
-	public LexerInterpreter(String grammarFileName, Collection<String> tokenNames, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) {
-		super(input);
-
-		if (atn.grammarType != ATNType.LEXER) {
-			throw new IllegalArgumentException("The ATN must be a lexer ATN.");
-		}
-
-		this.grammarFileName = grammarFileName;
-		this.atn = atn;
-		this.tokenNames = tokenNames.toArray(new String[tokenNames.size()]);
-		this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]);
-		this.modeNames = modeNames.toArray(new String[modeNames.size()]);
-
-		this._decisionToDFA = new DFA[atn.getNumberOfDecisions()];
-		for (int i = 0; i < _decisionToDFA.length; i++) {
-			_decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
-		}
-		this._interp = new LexerATNSimulator(atn,_decisionToDFA,_sharedContextCache);
-	}
-
-	@Override
-	public ATN getATN() {
-		return atn;
-	}
-
-	@Override
-	public String getGrammarFileName() {
-		return grammarFileName;
-	}
-
-	@Override
-	public String[] getTokenNames() {
-		return tokenNames;
-	}
-
-	@Override
-	public String[] getRuleNames() {
-		return ruleNames;
-	}
-
-	@Override
-	public String[] getModeNames() {
-		return modeNames;
-	}
-}

tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java

@@ -1,278 +0,0 @@
-/*
- * [The "BSD license"]
- * Copyright (c) 2013 Terence Parr
- * Copyright (c) 2013 Sam Harwell
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote products
- *    derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package org.antlr.v4.tool.interp;
-
-import org.antlr.v4.runtime.FailedPredicateException;
-import org.antlr.v4.runtime.InterpreterRuleContext;
-import org.antlr.v4.runtime.Parser;
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.TokenStream;
-import org.antlr.v4.runtime.atn.ATN;
-import org.antlr.v4.runtime.atn.ATNState;
-import org.antlr.v4.runtime.atn.ActionTransition;
-import org.antlr.v4.runtime.atn.AtomTransition;
-import org.antlr.v4.runtime.atn.DecisionState;
-import org.antlr.v4.runtime.atn.LoopEndState;
-import org.antlr.v4.runtime.atn.ParserATNSimulator;
-import org.antlr.v4.runtime.atn.PrecedencePredicateTransition;
-import org.antlr.v4.runtime.atn.PredicateTransition;
-import org.antlr.v4.runtime.atn.PredictionContextCache;
-import org.antlr.v4.runtime.atn.RuleStartState;
-import org.antlr.v4.runtime.atn.RuleStopState;
-import org.antlr.v4.runtime.atn.RuleTransition;
-import org.antlr.v4.runtime.atn.StarLoopEntryState;
-import org.antlr.v4.runtime.atn.Transition;
-import org.antlr.v4.runtime.dfa.DFA;
-import org.antlr.v4.runtime.misc.Pair;
-
-import java.util.ArrayDeque;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Deque;
-
-/** A parser simulator that mimics what ANTLR's generated
- *  parser code does. A ParserATNSimulator is used to make
- *  predictions via adaptivePredict but this class moves a pointer through the
- *  ATN to simulate parsing. ParserATNSimulator just
- *  makes us efficient rather than having to backtrack, for example.
- *
- *  This properly creates parse trees even for left recursive rules.
- *
- *  We rely on the left recursive rule invocation and special predicate
- *  transitions to make left recursive rules work.
- *
- *  See TestParserInterpreter for examples.
- */
-public class ParserInterpreter extends Parser {
-	protected final String grammarFileName;
-	protected final ATN atn;
-	protected final BitSet pushRecursionContextStates;
-
-	protected final DFA[] decisionToDFA; // not shared like it is for generated parsers
-	protected final PredictionContextCache sharedContextCache =
-		new PredictionContextCache();
-
-	protected final String[] tokenNames;
-	protected final String[] ruleNames;
-
-	protected final Deque<Pair<ParserRuleContext, Integer>> _parentContextStack = new ArrayDeque<Pair<ParserRuleContext, Integer>>();
-	protected final Deque<InterpreterRuleContext> _contextStack = new ArrayDeque<InterpreterRuleContext>();
-
-	public ParserInterpreter(String grammarFileName, Collection<String> tokenNames, Collection<String> ruleNames, ATN atn, TokenStream input) {
-		super(input);
-		this.grammarFileName = grammarFileName;
-		this.atn = atn;
-		this.tokenNames = tokenNames.toArray(new String[tokenNames.size()]);
-		this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]);
-		this.decisionToDFA = new DFA[atn.getNumberOfDecisions()];
-		for (int i = 0; i < decisionToDFA.length; i++) {
-			decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
-		}
-
-		// identify the ATN states where pushNewRecursionContext must be called
-		this.pushRecursionContextStates = new BitSet(atn.states.size());
-		for (ATNState state : atn.states) {
-			if (!(state instanceof StarLoopEntryState)) {
-				continue;
-			}
-
-			RuleStartState ruleStartState = atn.ruleToStartState[state.ruleIndex];
-			if (!ruleStartState.isPrecedenceRule) {
-				continue;
-			}
-
-			ATNState maybeLoopEndState = state.transition(state.getNumberOfTransitions() - 1).target;
-			if (!(maybeLoopEndState instanceof LoopEndState)) {
-				continue;
-			}
-
-			if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) {
-				this.pushRecursionContextStates.set(state.stateNumber);
-			}
-		}
-
-		// get atn simulator that knows how to do predictions
-		setInterpreter(new ParserATNSimulator(this, atn,
-											  decisionToDFA,
-											  sharedContextCache));
-	}
-
-	@Override
-	public ATN getATN() {
-		return atn;
-	}
-
-	@Override
-	public String[] getTokenNames() {
-		return tokenNames;
-	}
-
-	@Override
-	public String[] getRuleNames() {
-		return ruleNames;
-	}
-
-	@Override
-	public String getGrammarFileName() {
-		return grammarFileName;
-	}
-
-	/** Begin parsing at startRuleIndex */
-	public ParserRuleContext parse(int startRuleIndex) {
-		RuleStartState startRuleStartState = atn.ruleToStartState[startRuleIndex];
-
-		InterpreterRuleContext rootContext = new InterpreterRuleContext(null, ATNState.INVALID_STATE_NUMBER, startRuleIndex);
-		if (startRuleStartState.isPrecedenceRule) {
-			enterRecursionRule(rootContext, startRuleIndex, 0);
-		}
-		else {
-			enterRule(rootContext, startRuleStartState.stateNumber, startRuleIndex);
-		}
-
-		while ( true ) {
-			ATNState p = getATNState();
-			switch ( p.getStateType() ) {
-			case ATNState.RULE_STOP :
-				// pop; return from rule
-				if ( _ctx.isEmpty() ) {
-					exitRule();
-					return rootContext;
-				}
-
-				visitRuleStopState(p);
-				break;
-
-			default :
-				visitState(p);
-				break;
-			}
-		}
-	}
-
-	@Override
-	public void enterRecursionRule(ParserRuleContext localctx, int ruleIndex, int precedence) {
-		_parentContextStack.push(new Pair<ParserRuleContext, Integer>(_ctx, localctx.invokingState));
-		super.enterRecursionRule(localctx, ruleIndex, precedence);
-	}
-
-	protected ATNState getATNState() {
-		return atn.states.get(getState());
-	}
-
-	protected void visitState(ATNState p) {
-		int edge;
-		if (p.getNumberOfTransitions() > 1) {
-			edge = getInterpreter().adaptivePredict(_input, ((DecisionState)p).decision, _ctx);
-		}
-		else {
-			edge = 1;
-		}
-
-		Transition transition = p.transition(edge - 1);
-		switch (transition.getSerializationType()) {
-		case Transition.EPSILON:
-			if (pushRecursionContextStates.get(p.stateNumber) && !(transition.target instanceof LoopEndState)) {
-				InterpreterRuleContext ctx = new InterpreterRuleContext(_parentContextStack.peek().a, _parentContextStack.peek().b, _ctx.getRuleIndex());
-				pushNewRecursionContext(ctx, atn.ruleToStartState[p.ruleIndex].stateNumber, _ctx.getRuleIndex());
-			}
-			break;
-
-		case Transition.ATOM:
-			match(((AtomTransition)transition).label);
-			break;
-
-		case Transition.RANGE:
-		case Transition.SET:
-		case Transition.NOT_SET:
-			if (!transition.matches(_input.LA(1), Token.MIN_USER_TOKEN_TYPE, 65535)) {
-				_errHandler.recoverInline(this);
-			}
-			matchWildcard();
-			break;
-
-		case Transition.WILDCARD:
-			matchWildcard();
-			break;
-
-		case Transition.RULE:
-			RuleStartState ruleStartState = (RuleStartState)transition.target;
-			int ruleIndex = ruleStartState.ruleIndex;
-			InterpreterRuleContext ctx = new InterpreterRuleContext(_ctx, p.stateNumber, ruleIndex);
-			if (ruleStartState.isPrecedenceRule) {
-				enterRecursionRule(ctx, ruleIndex, ((RuleTransition)transition).precedence);
-			}
-			else {
-				enterRule(ctx, transition.target.stateNumber, ruleIndex);
-			}
-			break;
-
-		case Transition.PREDICATE:
-			PredicateTransition predicateTransition = (PredicateTransition)transition;
-			if (!sempred(_ctx, predicateTransition.ruleIndex, predicateTransition.predIndex)) {
-				throw new FailedPredicateException(this);
-			}
-
-			break;
-
-		case Transition.ACTION:
-			ActionTransition actionTransition = (ActionTransition)transition;
-			action(_ctx, actionTransition.ruleIndex, actionTransition.actionIndex);
-			break;
-
-		case Transition.PRECEDENCE:
-			if (!precpred(_ctx, ((PrecedencePredicateTransition)transition).precedence)) {
-				throw new FailedPredicateException(this, String.format("precpred(_ctx, %d)", ((PrecedencePredicateTransition)transition).precedence));
-			}
-			break;
-
-		default:
-			throw new UnsupportedOperationException("Unrecognized ATN transition type.");
-		}
-
-		setState(transition.target.stateNumber);
-	}
-
-	protected void visitRuleStopState(ATNState p) {
-		RuleStartState ruleStartState = atn.ruleToStartState[p.ruleIndex];
-		if (ruleStartState.isPrecedenceRule) {
-			Pair<ParserRuleContext, Integer> parentContext = _parentContextStack.pop();
-			unrollRecursionContexts(parentContext.a);
-			setState(parentContext.b);
-		}
-		else {
-			exitRule();
-		}
-
-		RuleTransition ruleTransition = (RuleTransition)atn.states.get(getState()).transition(0);
-		setState(ruleTransition.followState.stateNumber);
-	}
-}

tool/test/org/antlr/v4/test/TestBufferedTokenStream.java

@@ -33,14 +33,14 @@ package org.antlr.v4.test;
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.BufferedTokenStream;
 import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.LexerInterpreter;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.TokenStream;
 import org.antlr.v4.tool.LexerGrammar;
-import org.antlr.v4.tool.interp.LexerInterpreter;
 import org.junit.Test;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 
 public class TestBufferedTokenStream extends BaseTest {
 

tool/test/org/antlr/v4/test/TestParserInterpreter.java

@@ -32,18 +32,15 @@ package org.antlr.v4.test;
 
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.LexerInterpreter;
+import org.antlr.v4.runtime.ParserInterpreter;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.LexerGrammar;
-import org.antlr.v4.tool.interp.LexerInterpreter;
-import org.antlr.v4.tool.interp.ParserInterpreter;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 
-import java.util.Arrays;
-
 
 public class TestParserInterpreter extends BaseTest {
 	@Test public void testA() throws Exception {

tool/test/org/antlr/v4/test/TestPerformance.java

@@ -40,6 +40,7 @@ import org.antlr.v4.runtime.DefaultErrorStrategy;
 import org.antlr.v4.runtime.DiagnosticErrorListener;
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.Parser;
+import org.antlr.v4.runtime.ParserInterpreter;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
@@ -64,7 +65,6 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeListener;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 import org.antlr.v4.runtime.tree.TerminalNode;
-import org.antlr.v4.tool.interp.ParserInterpreter;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -104,8 +104,9 @@ import java.util.logging.Logger;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
-import static org.hamcrest.CoreMatchers.*;
+import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 public class TestPerformance extends BaseTest {
     /**

tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java

@@ -31,13 +31,14 @@ package org.antlr.v4.test;
 
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.LexerInterpreter;
 import org.antlr.v4.runtime.TokenStreamRewriter;
 import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.tool.LexerGrammar;
-import org.antlr.v4.tool.interp.LexerInterpreter;
 import org.junit.Test;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 public class TestTokenStreamRewriter extends BaseTest {
 

tool/test/org/antlr/v4/test/TestUnbufferedCharStream.java

@@ -34,16 +34,16 @@ import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CommonTokenFactory;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.IntStream;
+import org.antlr.v4.runtime.LexerInterpreter;
 import org.antlr.v4.runtime.UnbufferedCharStream;
 import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.tool.LexerGrammar;
-import org.antlr.v4.tool.interp.LexerInterpreter;
 import org.junit.Test;
 
 import java.io.Reader;
 import java.io.StringReader;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 
 public class TestUnbufferedCharStream extends BaseTest {
 	@Test public void testNoChar() throws Exception {

tool/test/org/antlr/v4/test/TestUnbufferedTokenStream.java

@@ -32,12 +32,12 @@ package org.antlr.v4.test;
 
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.LexerInterpreter;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.TokenStream;
 import org.antlr.v4.runtime.UnbufferedTokenStream;
 import org.antlr.v4.tool.LexerGrammar;
-import org.antlr.v4.tool.interp.LexerInterpreter;
 import org.junit.Test;
 
 import java.io.StringReader;
@@ -45,7 +45,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 
 public class TestUnbufferedTokenStream extends BaseTest {
 	@Test public void testLookahead() throws Exception {