forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

reorg snapshot; separate LexerSharedState; pass ctx as arg to funcctions 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6874]
This commit is contained in:
parrt 2010-05-18 14:10:31 -08:00
parent e1ccf08680
commit b315dd4ae4
19 changed files with 349 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java
View File

@ -45,31 +45,27 @@ import java.util.Map;
public abstract class BaseRecognizer {
public static final int MEMO_RULE_FAILED = -2;
public static final int MEMO_RULE_UNKNOWN = -1;
public static final int INITIAL_FOLLOW_STACK_SIZE = 100;
public static final String NEXT_TOKEN_RULE_NAME = "nextToken";
public IntStream input;
/** State of a lexer, parser, or tree parser are collected into a state
* object so the state can be shared. This sharing is needed to
* have one grammar import others and share same error variables
* and other state variables. It's a kind of explicit multiple
* inheritance via delegation of methods and shared state.
*/
protected RecognizerSharedState state;
public RecognizerSharedState state;
public BaseRecognizer(IntStream input) {
this.input = input;
state = new RecognizerSharedState();
this(input, new RecognizerSharedState());
}
public BaseRecognizer(IntStream input, RecognizerSharedState state) {
this.input = input;
if ( state==null ) {
state = new RecognizerSharedState();
}
this.state = state;
state.input = input;
}
/** reset the parser's state; subclasses must rewinds the input stream */
@ -78,7 +74,7 @@ public abstract class BaseRecognizer {
if ( state==null ) {
return; // no shared state work to do
}
state._fsp = -1;
state.ctx.clear();
state.errorRecovery = false;
state.lastErrorIndex = -1;
state.failed = false;
@ -105,10 +101,10 @@ public abstract class BaseRecognizer {
public Object match(int ttype, LABitSet follow)
throws RecognitionException
{
System.out.println("match "+((TokenStream)input).LT(1)+" vs expected "+ttype);
System.out.println("match "+((TokenStream)state.input).LT(1)+" vs expected "+ttype);
Object matchedSymbol = getCurrentInputSymbol();
if ( input.LA(1)==ttype ) {
input.consume();
if ( state.input.LA(1)==ttype ) {
state.input.consume();
state.errorRecovery = false;
state.failed = false;
return matchedSymbol;
@ -122,14 +118,14 @@ public abstract class BaseRecognizer {
}
/** Match the wildcard: in a symbol */
public void matchAny(IntStream input) {
public void matchAny() {
state.errorRecovery = false;
state.failed = false;
input.consume();
state.input.consume();
}
public boolean mismatchIsUnwantedToken(int ttype) {
return input.LA(2)==ttype;
return state.input.LA(2)==ttype;
}
public boolean mismatchIsMissingToken(LABitSet follow) {
@ -142,7 +138,7 @@ public abstract class BaseRecognizer {
if ( follow.member(Token.EOR_TOKEN_TYPE) ) {
LABitSet viableTokensFollowingThisRule = computeContextSensitiveRuleFOLLOW();
follow = follow.or(viableTokensFollowingThisRule);
if ( state._fsp>=0 ) { // remove EOR if we're not the start symbol
if ( state.ctx.sp>=0 ) { // remove EOR if we're not the start symbol
follow.remove(Token.EOR_TOKEN_TYPE);
}
}
@ -151,13 +147,13 @@ public abstract class BaseRecognizer {
// "insert" the missing token
//System.out.println("viable tokens="+follow.toString(getTokenNames()));
//System.out.println("LT(1)="+((TokenStream)input).LT(1));
//System.out.println("LT(1)="+((TokenStream)state.input).LT(1));
// LABitSet cannot handle negative numbers like -1 (EOF) so I leave EOR
// in follow set to indicate that the fall of the start symbol is
// in the set (EOF can follow).
if ( follow.member(input.LA(1)) || follow.member(Token.EOR_TOKEN_TYPE) ) {
//System.out.println("LT(1)=="+((TokenStream)input).LT(1)+" is consistent with what follows; inserting...");
if ( follow.member(state.input.LA(1)) || follow.member(Token.EOR_TOKEN_TYPE) ) {
//System.out.println("LT(1)=="+((TokenStream)state.input).LT(1)+" is consistent with what follows; inserting...");
return true;
}
return false;
@ -353,14 +349,14 @@ public abstract class BaseRecognizer {
* token that the match() routine could not recover from.
*/
public void recover(RecognitionException re) {
if ( state.lastErrorIndex==input.index() ) {
if ( state.lastErrorIndex==state.input.index() ) {
// uh oh, another error at same token index; must be a case
// where LT(1) is in the recovery token set so nothing is
// consumed; consume a single token so at least to prevent
// an infinite loop; this is a failsafe.
input.consume();
state.input.consume();
}
state.lastErrorIndex = input.index();
state.lastErrorIndex = state.input.index();
LABitSet followSet = computeErrorRecoverySet();
beginResync();
consumeUntil(followSet);
@ -527,15 +523,15 @@ public abstract class BaseRecognizer {
return combineFollows(true);
}
// what is exact? it seems to only add sets from above on stack
// TODO: what is exact? it seems to only add sets from above on stack
// if EOR is in set i. When it sees a set w/o EOR, it stops adding.
// Why would we ever want them all? Maybe no viable alt instead of
// mismatched token?
protected LABitSet combineFollows(boolean exact) {
int top = state._fsp;
int top = state.ctx.sp;
LABitSet followSet = new LABitSet();
for (int i=top; i>=0; i--) {
LABitSet localFollowSet = (LABitSet)state.following[i];
LABitSet localFollowSet = (LABitSet)state.ctx.get(i).follow;
/*
System.out.println("local follow depth "+i+"="+
localFollowSet.toString(getTokenNames())+")");
@ -593,30 +589,30 @@ public abstract class BaseRecognizer {
RecognitionException e = null;
// if next token is what we are looking for then "delete" this token
if ( mismatchIsUnwantedToken(ttype) ) {
e = new UnwantedTokenException(ttype, input);
e = new UnwantedTokenException(ttype, state.input);
/*
System.err.println("recoverFromMismatchedToken deleting "+
((TokenStream)input).LT(1)+
" since "+((TokenStream)input).LT(2)+" is what we want");
*/
beginResync();
input.consume(); // simply delete extra token
state.input.consume(); // simply delete extra token
endResync();
reportError(e); // report after consuming so AW sees the token in the exception
// we want to return the token we're actually matching
Object matchedSymbol = getCurrentInputSymbol();
input.consume(); // move past ttype token as if all were ok
state.input.consume(); // move past ttype token as if all were ok
return matchedSymbol;
}
// can't recover with single token deletion, try insertion
if ( mismatchIsMissingToken(follow) ) {
Object inserted = getMissingSymbol(e, ttype, follow);
e = new MissingTokenException(ttype, input, inserted);
e = new MissingTokenException(ttype, state.input, inserted);
reportError(e); // report after inserting so AW sees the token in the exception
return inserted;
}
// even that didn't work; must throw the exception
e = new MismatchedTokenException(ttype, input);
e = new MismatchedTokenException(ttype, state.input);
throw e;
}
@ -672,34 +668,24 @@ public abstract class BaseRecognizer {
public void consumeUntil(int tokenType) {
//System.out.println("consumeUntil "+tokenType);
int ttype = input.LA(1);
int ttype = state.input.LA(1);
while (ttype != Token.EOF && ttype != tokenType) {
input.consume();
ttype = input.LA(1);
state.input.consume();
ttype = state.input.LA(1);
}
}
/** Consume tokens until one matches the given token set */
public void consumeUntil(LABitSet set) {
//System.out.println("consumeUntil("+set.toString(getTokenNames())+")");
int ttype = input.LA(1);
int ttype = state.input.LA(1);
while (ttype != Token.EOF && !set.member(ttype) ) {
//System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);
input.consume();
ttype = input.LA(1);
//System.out.println("consume during recover LA(1)="+getTokenNames()[state.input.LA(1)]);
state.input.consume();
ttype = state.input.LA(1);
}
}
/** Push a rule's follow set using our own hardcoded stack */
protected void pushFollow(LABitSet fset) {
if ( (state._fsp +1)>=state.following.length ) {
LABitSet[] f = new LABitSet[state.following.length*2];
System.arraycopy(state.following, 0, f, 0, state.following.length);
state.following = f;
}
state.following[++state._fsp] = fset;
}
/** Return List<String> of the rules in your parser instance
* leading up to a call to this method. You could override if
* you want more details such as the file/line info of where
@ -809,8 +795,8 @@ public abstract class BaseRecognizer {
* this rule and successfully parsed before, then seek ahead to
* 1 past the stop token matched for this rule last time.
*/
public boolean alreadyParsedRule(IntStream input, int ruleIndex) {
int stopIndex = getRuleMemoization(ruleIndex, input.index());
public boolean alreadyParsedRule(int ruleIndex) {
int stopIndex = getRuleMemoization(ruleIndex, state.input.index());
if ( stopIndex==MEMO_RULE_UNKNOWN ) {
return false;
}
@ -820,7 +806,7 @@ public abstract class BaseRecognizer {
}
else {
//System.out.println("seen rule "+ruleIndex+" before; skipping ahead to @"+(stopIndex+1)+" failed="+state.failed);
input.seek(stopIndex+1); // jump to one past stop token
state.input.seek(stopIndex+1); // jump to one past stop token
}
return true;
}

80
runtime/Java/src/org/antlr/v4/runtime/Lexer.java
View File

@ -28,6 +28,7 @@
package org.antlr.v4.runtime;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.IntStream;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenSource;
import org.antlr.v4.runtime.pda.PDA;
@ -37,26 +38,31 @@ import org.antlr.v4.runtime.pda.PDA;
* uses simplified match() and error recovery mechanisms in the interest
* of speed.
*/
public abstract class Lexer extends BaseRecognizer implements TokenSource {
public abstract class Lexer /* extends BaseRecognizer */ implements TokenSource {
public static final int DEFAULT_MODE = 0;
public LexerSharedState state;
public int _mode = DEFAULT_MODE;
public static PDA[] modeToPDA;
public Lexer(CharStream input) {
super(input);
public Lexer(IntStream input) {
this(input, new LexerSharedState());
}
public Lexer(CharStream input, RecognizerSharedState state) {
super(input, state);
public Lexer(IntStream input, LexerSharedState state) {
if ( state==null ) {
state = new LexerSharedState();
}
this.state = state;
state.input = input;
}
public void reset() {
super.reset(); // reset all recognizer state variables
// wack Lexer state variables
if ( input!=null ) {
input.seek(0); // rewind the input
if ( state.input!=null ) {
state.input.seek(0); // rewind the input
}
if ( state==null ) {
return; // no shared state work to do
@ -77,20 +83,20 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
while (true) {
state.token = null;
state.channel = Token.DEFAULT_CHANNEL;
state.tokenStartCharIndex = input.index();
state.tokenStartCharPositionInLine = ((CharStream)input).getCharPositionInLine();
state.tokenStartLine = ((CharStream)input).getLine();
state.tokenStartCharIndex = state.input.index();
state.tokenStartCharPositionInLine = ((CharStream)state.input).getCharPositionInLine();
state.tokenStartLine = ((CharStream)state.input).getLine();
state.text = null;
if ( input.LA(1)==CharStream.EOF ) {
Token eof = new CommonToken((CharStream)input,Token.EOF,
if ( state.input.LA(1)==CharStream.EOF ) {
Token eof = new CommonToken((CharStream)state.input,Token.EOF,
Token.DEFAULT_CHANNEL,
input.index(),input.index());
state.input.index(),state.input.index());
eof.setLine(getLine());
eof.setCharPositionInLine(getCharPositionInLine());
return eof;
}
{
state.type = modeToPDA[_mode].execThompson(input);
state.type = modeToPDA[_mode].execThompson(state.input);
if ( state.token==null ) {
emit();
}
@ -122,17 +128,17 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
/** Set the char stream and reset the lexer */
public void setCharStream(CharStream input) {
this.input = null;
this.state.input = null;
reset();
this.input = input;
this.state.input = input;
}
public CharStream getCharStream() {
return ((CharStream)input);
return ((CharStream)state.input);
}
public String getSourceName() {
return input.getSourceName();
return state.input.getSourceName();
}
/** Currently does not support multiple emits per nextToken invocation
@ -154,7 +160,7 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
* Parser or TreeParser.getMissingSymbol().
*/
public Token emit() {
Token t = new CommonToken(((CharStream)input), state.type,
Token t = new CommonToken(((CharStream)state.input), state.type,
state.channel, state.tokenStartCharIndex,
getCharIndex()-1);
t.setLine(state.tokenStartLine);
@ -165,16 +171,16 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
}
public int getLine() {
return ((CharStream)input).getLine();
return ((CharStream)state.input).getLine();
}
public int getCharPositionInLine() {
return ((CharStream)input).getCharPositionInLine();
return ((CharStream)state.input).getCharPositionInLine();
}
/** What is the index of the current character of lookahead? */
public int getCharIndex() {
return input.index();
return state.input.index();
}
/** Return the text matched so far for the current token or any
@ -184,7 +190,7 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
if ( state.text!=null ) {
return state.text;
}
return ((CharStream)input).substring(state.tokenStartCharIndex,getCharIndex()-1);
return ((CharStream)state.input).substring(state.tokenStartCharIndex,getCharIndex()-1);
}
/** Set the complete text of this token; it wipes any previous
@ -206,7 +212,15 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
errorRecovery = true;
*/
displayRecognitionError(this.getTokenNames(), e);
//displayRecognitionError(this.getTokenNames(), e);
}
/** Used to print out token names like ID during debugging and
* error reporting. The generated parsers implement a method
* that overrides this to point to their String[] tokenNames.
*/
public String[] getTokenNames() {
return null;
}
public String getErrorMessage(RecognitionException e, String[] tokenNames) {
@ -241,7 +255,7 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
getCharErrorDisplay(mre.a)+".."+getCharErrorDisplay(mre.b);
}
else {
msg = super.getErrorMessage(e, tokenNames);
//msg = super.getErrorMessage(e, tokenNames);
}
return msg;
}
@ -271,18 +285,8 @@ public abstract class Lexer extends BaseRecognizer implements TokenSource {
* to do sophisticated error recovery if you are in a fragment rule.
*/
public void recover(RecognitionException re) {
//System.out.println("consuming char "+(char)input.LA(1)+" during recovery");
//System.out.println("consuming char "+(char)state.input.LA(1)+" during recovery");
//re.printStackTrace();
input.consume();
}
public void traceIn(String ruleName, int ruleIndex) {
String inputSymbol = ((char)((CharStream)input).LT(1))+" line="+getLine()+":"+getCharPositionInLine();
super.traceIn(ruleName, ruleIndex, inputSymbol);
}
public void traceOut(String ruleName, int ruleIndex) {
String inputSymbol = ((char)((CharStream)input).LT(1))+" line="+getLine()+":"+getCharPositionInLine();
super.traceOut(ruleName, ruleIndex, inputSymbol);
state.input.consume();
}
}

48
runtime/Java/src/org/antlr/v4/runtime/LexerSharedState.java Normal file
View File

@ -0,0 +1,48 @@
package org.antlr.v4.runtime;
import org.antlr.runtime.IntStream;
import org.antlr.runtime.Token;
public class LexerSharedState {
public IntStream input;
/** The goal of all lexer rules/methods is to create a token object.
* This is an instance variable as multiple rules may collaborate to
* create a single token. nextToken will return this object after
* matching lexer rule(s). If you subclass to allow multiple token
* emissions, then set this to the last token to be matched or
* something nonnull so that the auto token emit mechanism will not
* emit another token.
*/
public Token token;
/** What character index in the stream did the current token start at?
* Needed, for example, to get the text for current token. Set at
* the start of nextToken.
*/
public int tokenStartCharIndex = -1;
/** The line on which the first character of the token resides */
public int tokenStartLine;
/** The character position of first character within the line */
public int tokenStartCharPositionInLine;
/** The channel number for the current token */
public int channel;
/** The token type for the current token */
public int type;
/** You can set the text for the current token to override what is in
* the input char buffer. Use setText() or can set this instance var.
*/
public String text;
public LexerSharedState() {
}
public LexerSharedState(LexerSharedState state) {
this.token = state.token;
this.tokenStartCharIndex = state.tokenStartCharIndex;
this.tokenStartLine = state.tokenStartLine;
this.tokenStartCharPositionInLine = state.tokenStartCharPositionInLine;
this.channel = state.channel;
this.type = state.type;
this.text = state.text;
}
}

26
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -27,7 +27,6 @@
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.IntStream;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenStream;
@ -47,17 +46,16 @@ public class Parser extends BaseRecognizer {
public void reset() {
super.reset(); // reset all recognizer state variables
if ( input!=null ) {
input.seek(0); // rewind the input
if ( state.input!=null ) {
state.input.seek(0); // rewind the input
}
}
protected Object getCurrentInputSymbol() {
return ((TokenStream)input).LT(1);
return ((TokenStream)state.input).LT(1);
}
protected Object getMissingSymbol(IntStream input,
RecognitionException e,
protected Object getMissingSymbol(RecognitionException e,
int expectedTokenType,
LABitSet follow)
{
@ -65,9 +63,9 @@ public class Parser extends BaseRecognizer {
if ( expectedTokenType== Token.EOF ) tokenText = "<missing EOF>";
else tokenText = "<missing "+getTokenNames()[expectedTokenType]+">";
CommonToken t = new CommonToken(expectedTokenType, tokenText);
Token current = ((TokenStream)input).LT(1);
Token current = ((TokenStream)state.input).LT(1);
if ( current.getType() == Token.EOF ) {
current = ((TokenStream)input).LT(-1);
current = ((TokenStream)state.input).LT(-1);
}
t.line = current.getLine();
t.charPositionInLine = current.getCharPositionInLine();
@ -77,24 +75,24 @@ public class Parser extends BaseRecognizer {
/** Set the token stream and reset the parser */
public void setTokenStream(TokenStream input) {
this.input = null;
this.state.input = null;
reset();
this.input = input;
this.state.input = input;
}
public TokenStream getTokenStream() {
return (TokenStream)input;
return (TokenStream)state.input;
}
public String getSourceName() {
return input.getSourceName();
return state.input.getSourceName();
}
public void traceIn(String ruleName, int ruleIndex) {
super.traceIn(ruleName, ruleIndex, ((TokenStream)input).LT(1));
super.traceIn(ruleName, ruleIndex, ((TokenStream)state.input).LT(1));
}
public void traceOut(String ruleName, int ruleIndex) {
super.traceOut(ruleName, ruleIndex, ((TokenStream)input).LT(1));
super.traceOut(ruleName, ruleIndex, ((TokenStream)state.input).LT(1));
}
}

57
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java Normal file
View File

@ -0,0 +1,57 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.Token;
import org.antlr.v4.runtime.misc.LABitSet;
/** Rules that return more than a single value must return an object
* containing all the values. Besides the properties defined in
* RuleLabelScope.predefinedRulePropertiesScope there may be user-defined
* return values. This class simply defines the minimum properties that
* are always defined and methods to access the others that might be
* available depending on output option such as template and tree.
*
* Note text is not an actual property of the return value, it is computed
* from start and stop using the input stream's toString() method. I
* could add a ctor to this so that we can pass in and store the input
* stream, but I'm not sure we want to do that. It would seem to be undefined
* to get the .text property anyway if the rule matches tokens from multiple
* input streams.
*
* I do not use getters for fields of objects that are used simply to
* group values such as this aggregate. The getters/setters are there to
* satisfy the superclass interface.
*/
public class ParserRuleContext extends RuleContext {
public Token start, stop;
public Object getStart() { return start; }
public Object getStop() { return stop; }
public ParserRuleContext() {;}
public ParserRuleContext(LABitSet follow) { super(follow); }
}

72
runtime/Java/src/org/antlr/v4/runtime/RecognizerSharedState.java
View File

@ -27,25 +27,19 @@
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.Token;
import org.antlr.v4.runtime.misc.LABitSet;
import org.antlr.runtime.IntStream;
import org.antlr.v4.runtime.misc.QStack;
import java.util.Map;
/** The set of fields needed by an abstract recognizer to recognize input
* and recover from errors etc... As a separate state object, it can be
* shared among multiple grammars; e.g., when one grammar imports another.
*
* These fields are publically visible but the actual state pointer per
* parser is protected.
*/
public class RecognizerSharedState {
/** Track the set of token types that can follow any rule invocation.
* Stack grows upwards. When it hits the max, it grows 2x in size
* and keeps going.
*/
public LABitSet[] following = new LABitSet[BaseRecognizer.INITIAL_FOLLOW_STACK_SIZE];
public int _fsp = -1;
public IntStream input;
public QStack<RuleContext> ctx;
/** This is true when we see an error and before having successfully
* matched a token. Prevents generation of more than one error message
@ -83,53 +77,13 @@ public class RecognizerSharedState {
*/
public Map[] ruleMemo;
// LEXER FIELDS (must be in same state object to avoid casting
// constantly in generated code and Lexer object) :(
/** The goal of all lexer rules/methods is to create a token object.
* This is an instance variable as multiple rules may collaborate to
* create a single token. nextToken will return this object after
* matching lexer rule(s). If you subclass to allow multiple token
* emissions, then set this to the last token to be matched or
* something nonnull so that the auto token emit mechanism will not
* emit another token.
*/
public Token token;
/** What character index in the stream did the current token start at?
* Needed, for example, to get the text for current token. Set at
* the start of nextToken.
*/
public int tokenStartCharIndex = -1;
/** The line on which the first character of the token resides */
public int tokenStartLine;
/** The character position of first character within the line */
public int tokenStartCharPositionInLine;
/** The channel number for the current token */
public int channel;
/** The token type for the current token */
public int type;
/** You can set the text for the current token to override what is in
* the input char buffer. Use setText() or can set this instance var.
*/
public String text;
public RecognizerSharedState() {;}
public RecognizerSharedState() {
this.ctx = new QStack<RuleContext>();
}
public RecognizerSharedState(RecognizerSharedState state) {
if ( this.following.length < state.following.length ) {
this.following = new LABitSet[state.following.length];
}
System.arraycopy(state.following, 0, this.following, 0, state.following.length);
this._fsp = state._fsp;
this.errorRecovery = state.errorRecovery;
this.ctx = state.ctx;
this.errorRecovery = state.errorRecovery;
this.lastErrorIndex = state.lastErrorIndex;
this.failed = state.failed;
this.syntaxErrors = state.syntaxErrors;
@ -138,11 +92,5 @@ public class RecognizerSharedState {
this.ruleMemo = new Map[state.ruleMemo.length];
System.arraycopy(state.ruleMemo, 0, this.ruleMemo, 0, state.ruleMemo.length);
}
this.token = state.token;
this.tokenStartCharIndex = state.tokenStartCharIndex;
this.tokenStartCharPositionInLine = state.tokenStartCharPositionInLine;
this.channel = state.channel;
this.type = state.type;
this.text = state.text;
}
}

53
runtime/Java/src/org/antlr/v4/runtime/RuleContext.java Normal file
View File

@ -0,0 +1,53 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.LABitSet;
/** Rules can return start/stop info as well as possible trees and templates */
public class RuleContext {
/** Track the set of token types that can follow any rule invocation. */
public LABitSet follow;
/** Return the start token or tree */
public Object getStart() { return null; }
/** Return the stop token or tree */
public Object getStop() { return null; }
/** Has a value potentially if output=AST; */
public Object getTree() { return null; }
/** Has a value potentially if output=template; Don't use StringTemplate
* type as it then causes a dependency with ST lib.
*/
public Object getTemplate() { return null; }
public RuleContext() {;}
public RuleContext(LABitSet follow) { this.follow = follow; }
}

20
runtime/Java/src/org/antlr/v4/runtime/misc/QStack.java
View File

@ -5,7 +5,17 @@ import java.util.EmptyStackException;
/** A quicker stack than Stack */
public class QStack<T> {
Object[] elements;
int sp = -1;
public int sp = -1;
public QStack() {
elements = new Object[10];
}
public QStack(QStack s) {
elements = new Object[s.elements.length];
System.arraycopy(s.elements, 0, elements, 0, s.elements.length);
this.sp = s.sp;
}
public void push(T fset) {
if ( (sp+1)>=elements.length ) {
@ -21,8 +31,16 @@ public class QStack<T> {
return (T)elements[sp];
}
public T get(int i) {
if ( i<0 ) throw new IllegalArgumentException("i<0");
if ( i>sp ) throw new IllegalArgumentException("i>"+sp);
return (T)elements[sp];
}
public T pop() {
if ( sp<0 ) throw new EmptyStackException();
return (T)elements[sp--];
}
public void clear() { sp = -1; }
}

38
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -17,6 +17,7 @@ ParserFile(file, parser, dfaDecls, bitSetDecls) ::= <<
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.RecognizerSharedState;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.misc.*;
import org.antlr.runtime.*;
@ -36,7 +37,6 @@ public class <p.name> extends Parser {
}
!>
<p.tokens.keys:{k | public static final int <k>=<p.tokens.(k)>;}; separator="\n">
public QStack\<ParserRuleReturnScope> _ctx = new QStack\<ParserRuleReturnScope>();
<p:ctor()>
<funcs; separator="\n">
<dfaDecls; separator="\n">
@ -60,20 +60,24 @@ public <p.name>(TokenStream input, RecognizerSharedState state) {
S.b_return retval = new S.b_return();
*/
RuleFunction(f,code,decls,context) ::= <<
RuleFunction(f,code,decls,context,scope) ::= <<
<context>
<scope>
<if(f.modifiers)><f.modifiers:{f | <f> }><else>public final <endif><if(f.retType)><f.retType><else>void<endif> <f.name>(<f.args:{a|<a>, }>LABitSet _follow) throws RecognitionException {
<! <if(f.scope)><f.scope.name>.push(new <f.scope.name>);<endif> !>
<if(f.retType)>
_ctx.push(new <f.context.name>(<f.args:{a|<a.name>}; separator=",">));
<if(f.modifiers)><f.modifiers:{f | <f> }><else>public final <endif><if(f.ctxType)><f.ctxType><else>void<endif> <f.name>(<f.ctxType> _ctx) throws RecognitionException {
<if(f.scope)>
<f.scope.name>_stack.push(new <f.scope.name>());
<endif>
<if(f.ctxType)>
state.ctx.push(_ctx);
<endif>
<decls; separator="\n">
try {
<code>
}
finally {
<if(f.retType)>return (<f.retType>)_ctx.pop();<endif>
<if(f.scope)><f.scope.name>_stack.pop();<endif>
<if(f.ctxType)>return (<f.ctxType>)state.ctx.pop();<endif>
}
}
>>
@ -166,7 +170,7 @@ cases(ttypes) ::= <<
>>
InvokeRule(r) ::= <<
<if(r.label)><r.label> = <endif><r.name>(<r.argExprs:{a|<a>, }><r.follow.name>);
<if(r.label)><r.label> = <endif><r.name>(new <r.ctxName>(<r.argExprs:{e|<e>,}><r.follow.name>));
>>
MatchToken(m) ::= <<
@ -189,21 +193,21 @@ RuleContextDecl(r) ::= "<r.ctxName> <r.name>;"
CaptureNextToken(d) ::= "<d.varName> = input.LA(1);"
StructDecl(s,attrs) ::= <<
public static class <s.name> extends ParserRuleReturnScope {
public static class <s.name> extends ParserRuleContext {
<attrs:{a | <a>;}; separator="\n">
<if(s.ctorAttrs)>
<if(s.ctorAttrs)>
public <s.name>(<s.ctorAttrs; separator=", ">) {
<s.ctorAttrs:{a | this.<a.name> = <a.name>;}; separator="\n">
}
<endif>
<endif>
};
>>
DynamicScopeStruct(rv,attrs) ::= <<
public static class <rv.name> {
DynamicScopeStruct(d,attrs) ::= <<
public static class <d.name> {
<attrs:{a | <a>;}; separator="\n">
};
QStack b_stack = new QStack();
public QStack\<<f.scope.name>\> <f.scope.name>_stack = new QStack\<<f.scope.name>\>();
>>
AttributeDecl(d) ::= "<d.decl>"
@ -219,7 +223,7 @@ public static final LABitSet <b.name>=new LABitSet(new long[]{<b.fset.bits:{<it>
LexerFile(fileName, lexer) ::= <<
// $ANTLR ANTLRVersion> <fileName> generatedTimestamp>
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RecognizerSharedState;
import org.antlr.v4.runtime.LexerSharedState;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.pda.*;
@ -235,9 +239,9 @@ public class <lexerName> extends Lexer {
<modes:{m| public static final int <m> = <i0>;}; separator="\n">
public <lexerName>(CharStream input) {
this(input, new RecognizerSharedState());
this(input, new LexerSharedState());
}
public <lexerName>(CharStream input, RecognizerSharedState state) {
public <lexerName>(CharStream input, LexerSharedState state) {
super(input,state);
}
public String getGrammarFileName() { return "<fileName>"; }

12
tool/src/org/antlr/v4/codegen/OutputModelFactory.java
View File

@ -86,18 +86,6 @@ public abstract class OutputModelFactory {
// outputModel.dfaDefs.add(d);
}
public String getLoopLabel(GrammarAST ast) {
return "loop"+ ast.token.getTokenIndex();
}
public String getLoopCounter(GrammarAST ast) {
return "cnt"+ ast.token.getTokenIndex();
}
public String getListLabel(String label) { return label+"_list"; }
public String getRuleFunctionContextStructName(String ruleName) { return ruleName+"_ctx"; }
public String getDynamicScopeStructName(String ruleName) { return ruleName+"_scope"; }
public BitSetDecl createFollowBitSet(GrammarAST ast, IntervalSet set) {
String inRuleName = ast.nfaState.rule.name;
String elementName = ast.getText(); // assume rule ref

4
tool/src/org/antlr/v4/codegen/ParserFactory.java
View File

@ -33,7 +33,7 @@ public class ParserFactory extends OutputModelFactory {
InvokeRule r = new InvokeRule(this, ID, label);
AddToList a = null;
if ( label!=null && label.parent.getType()== ANTLRParser.PLUS_ASSIGN ) {
a = new AddToList(this, getListLabel(r.label), r);
a = new AddToList(this, gen.target.getListLabel(r.label), r);
}
return Utils.list(r, a);
}
@ -43,7 +43,7 @@ public class ParserFactory extends OutputModelFactory {
MatchToken m = new MatchToken(this, (TerminalAST) ID, label);
AddToList a = null;
if ( label!=null && label.parent.getType()== ANTLRParser.PLUS_ASSIGN ) {
a = new AddToList(this, getListLabel(m.label), m);
a = new AddToList(this, gen.target.getListLabel(m.label), m);
}
return Utils.list(m, a);
}

17
tool/src/org/antlr/v4/codegen/Target.java
View File

@ -3,6 +3,8 @@ package org.antlr.v4.codegen;
import org.antlr.v4.automata.Label;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarAST;
import org.antlr.v4.tool.Rule;
import org.stringtemplate.v4.ST;
import java.io.IOException;
@ -120,5 +122,20 @@ public class Target {
return buf.toString();
}
public String getLoopLabel(GrammarAST ast) {
return "loop"+ ast.token.getTokenIndex();
}
public String getLoopCounter(GrammarAST ast) {
return "cnt"+ ast.token.getTokenIndex();
}
public String getListLabel(String label) { return label+"_list"; }
public String getRuleFunctionContextStructName(Rule r) {
if ( r.args==null && r.retvals==null ) return "ParserRuleContext";
return r.name+"_ctx";
}
public String getDynamicScopeStructName(String ruleName) { return ruleName+"_scope"; }
public int getInlineTestsVsBitsetThreshold() { return 20; }
}

8
tool/src/org/antlr/v4/codegen/src/DynamicScopeStruct.java
View File

@ -1,11 +1,13 @@
package org.antlr.v4.codegen.src;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.Attribute;
import java.util.Collection;
/** */
public class DynamicScopeStruct extends StructDecl {
public DynamicScopeStruct(OutputModelFactory factory, Rule r) {
super(factory, factory.getDynamicScopeStructName(r.name), r.scope.attributes.values());
public DynamicScopeStruct(OutputModelFactory factory, String name, Collection<Attribute> attrList) {
super(factory, name, attrList);
}
}

4
tool/src/org/antlr/v4/codegen/src/InvokeRule.java
View File

@ -21,9 +21,7 @@ public class InvokeRule extends SrcOp implements LabeledOp {
this.ast = ast;
this.name = ast.getText();
Rule r = factory.g.getRule(name);
if ( r.retvals!=null || r.args!=null ) {
ctxName = factory.getRuleFunctionContextStructName(r.name);
}
ctxName = factory.gen.target.getRuleFunctionContextStructName(r);
if ( labelAST!=null ) {
label = labelAST.getText();

4
tool/src/org/antlr/v4/codegen/src/LL1PlusBlock.java
View File

@ -22,7 +22,7 @@ public class LL1PlusBlock extends LL1Loop {
IntervalSet exitLook = dfa.startState.edge(1).label;
this.exitLook = factory.gen.target.getTokenTypesAsTargetLabels(factory.g, exitLook.toArray());
loopLabel = factory.getLoopLabel(blkAST);
loopCounterVar = factory.getLoopCounter(blkAST);
loopLabel = factory.gen.target.getLoopLabel(blkAST);
loopCounterVar = factory.gen.target.getLoopCounter(blkAST);
}
}

2
tool/src/org/antlr/v4/codegen/src/LL1StarBlock.java
View File

@ -25,7 +25,7 @@ public class LL1StarBlock extends LL1Loop {
IntervalSet exitLook = dfa.startState.edge(1).label;
this.exitLook = factory.gen.target.getTokenTypesAsTargetLabels(factory.g, exitLook.toArray());
loopLabel = factory.getLoopLabel(blkAST);
loopLabel = factory.gen.target.getLoopLabel(blkAST);
}
}

2
tool/src/org/antlr/v4/codegen/src/MatchToken.java
View File

@ -22,7 +22,7 @@ public class MatchToken extends SrcOp implements LabeledOp {
TokenDecl d = new TokenDecl(factory, label);
factory.currentRule.peek().addDecl(d);
if ( labelAST.parent.getType() == ANTLRParser.PLUS_ASSIGN ) {
TokenListDecl l = new TokenListDecl(factory, factory.getListLabel(label));
TokenListDecl l = new TokenListDecl(factory, factory.gen.target.getListLabel(label));
factory.currentRule.peek().addDecl(l);
}
}

34
tool/src/org/antlr/v4/codegen/src/RuleFunction.java
View File

@ -7,6 +7,7 @@ import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.ScopeParser;
import org.antlr.v4.tool.Attribute;
import org.antlr.v4.tool.GrammarAST;
import org.antlr.v4.tool.Rule;
@ -19,7 +20,7 @@ import java.util.List;
public class RuleFunction extends OutputModelObject {
public String name;
public List<String> modifiers;
public String retType;
public String ctxType;
public List<String> globalScopesUsed;
public Collection<String> ruleLabels;
public Collection<String> tokenLabels;
@ -29,6 +30,7 @@ public class RuleFunction extends OutputModelObject {
public boolean isStartRule;
public StructDecl context;
public DynamicScopeStruct scope;
public Collection<Attribute> args = null;
public OrderedHashSet<Decl> decls;
@ -44,24 +46,32 @@ public class RuleFunction extends OutputModelObject {
}
modifiers = Utils.nodesToStrings(r.modifiers);
List<Attribute> argsDynScopeAndReturnValues = new ArrayList<Attribute>();
ctxType = factory.gen.target.getRuleFunctionContextStructName(r);
List<Attribute> argsAndReturnValues = new ArrayList<Attribute>();
List<Attribute> ctorAttrs = new ArrayList<Attribute>();
if ( r.args!=null ) {
this.args = r.args.attributes.values();
argsDynScopeAndReturnValues.addAll(r.args.attributes.values());
argsAndReturnValues.addAll(r.args.attributes.values());
args = r.args.attributes.values();
ctorAttrs.addAll(args);
}
if ( r.retvals!=null ) {
retType = factory.getRuleFunctionContextStructName(r.name);
argsDynScopeAndReturnValues.addAll(r.retvals.attributes.values());
argsAndReturnValues.addAll(r.retvals.attributes.values());
}
if ( r.scope!=null ) {
argsDynScopeAndReturnValues.addAll(r.scope.attributes.values());
scope = new DynamicScopeStruct(factory, factory.gen.target.getDynamicScopeStructName(r.name),
r.scope.attributes.values());
}
if ( argsDynScopeAndReturnValues.size()>0 ) {
context = new StructDecl(factory, factory.getRuleFunctionContextStructName(r.name),
argsDynScopeAndReturnValues);
context.ctorAttrs = args;
if ( argsAndReturnValues.size()>0 ) {
context = new StructDecl(factory, factory.gen.target.getRuleFunctionContextStructName(r),
argsAndReturnValues);
ctorAttrs.add(ScopeParser.parseAttributeDef("LABitSet follow"));
context.ctorAttrs = ctorAttrs;
}
ruleLabels = r.getLabelNames();
tokenLabels = r.getTokenRefs();
exceptions = Utils.nodesToStrings(r.exceptionActions);
@ -91,7 +101,7 @@ public class RuleFunction extends OutputModelObject {
final List<String> sup = super.getChildren();
return new ArrayList<String>() {{
if ( sup!=null ) addAll(sup);
add("context"); add("decls"); add("code");
add("context"); add("scope"); add("decls"); add("code");
}};
}
}

2
tool/src/org/antlr/v4/parse/ScopeParser.java
View File

@ -76,7 +76,7 @@ public class ScopeParser {
* but if the separator is ',' you cannot use ',' in the initvalue
* unless you escape use "\," escape.
*/
protected static Attribute parseAttributeDef(String decl) {
public static Attribute parseAttributeDef(String decl) {
if ( decl==null ) return null;
Attribute attr = new Attribute();
boolean inID = false;