From 3fe028ecc4e7671c1ed3e5c74cb0fdb98cc1cc61 Mon Sep 17 00:00:00 2001 From: parrt Date: Wed, 26 May 2010 13:08:20 -0800 Subject: [PATCH] got mode methods in [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6896] --- .../Java/src/org/antlr/v4/runtime/Lexer.java | 40 +++++++++++-------- .../antlr/v4/runtime/LexerSharedState.java | 10 +++++ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index 0df69d74e..072673ced 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -31,22 +31,23 @@ import org.antlr.runtime.CharStream; import org.antlr.runtime.IntStream; import org.antlr.runtime.Token; import org.antlr.runtime.TokenSource; +import org.antlr.v4.runtime.misc.QStack; import org.antlr.v4.runtime.pda.PDA; +import java.util.EmptyStackException; + /** A lexer is recognizer that draws input symbols from a character stream. * lexer grammars result in a subclass of this object. A Lexer object * uses simplified match() and error recovery mechanisms in the interest * of speed. */ -public abstract class Lexer /* extends BaseRecognizer */ implements TokenSource { +public abstract class Lexer implements TokenSource { public static final int DEFAULT_MODE = 0; public static final int MORE = -2; public static final int SKIP = -3; public LexerSharedState state; - public int _mode = DEFAULT_MODE; - public static PDA[] modeToPDA; public Lexer(IntStream input) { @@ -100,25 +101,14 @@ public abstract class Lexer /* extends BaseRecognizer */ implements TokenSource eof.setCharPositionInLine(getCharPositionInLine()); return eof; } - int ttype = modeToPDA[_mode].execThompson(state.input); + int ttype = modeToPDA[state.mode].execThompson(state.input); if ( state.type == Token.INVALID_TOKEN_TYPE ) state.type = ttype; if ( state.type==SKIP ) { continue outer; } -// if ( state.token==null ) { -// emit(); -// } } while ( state.type==MORE ); - emit(); + if ( state.token==null ) emit(); return state.token; -// catch (NoViableAltException nva) { -// reportError(nva); -// recover(nva); // throw out current char and try again -// } -// catch (RecognitionException re) { -// reportError(re); -// // match() routine has already called recover() -// } } } @@ -129,7 +119,23 @@ public abstract class Lexer /* extends BaseRecognizer */ implements TokenSource * and emits it. */ public void skip() { - state.token = Token.SKIP_TOKEN; + state.type = SKIP; + } + + public void more() { + state.type = MORE; + } + + public void mode(int m) { state.mode = m; } + public void pushMode(int m) { + if ( state.modeStack==null ) state.modeStack = new QStack(); + state.modeStack.push(state.mode); + state.mode = m; + } + public int popMode() { + if ( state.modeStack==null ) throw new EmptyStackException(); + state.mode = state.modeStack.pop(); + return state.mode; } /** Set the char stream and reset the lexer */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerSharedState.java b/runtime/Java/src/org/antlr/v4/runtime/LexerSharedState.java index 23ccaab53..a6ed5f99f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerSharedState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerSharedState.java @@ -2,6 +2,7 @@ package org.antlr.v4.runtime; import org.antlr.runtime.IntStream; import org.antlr.runtime.Token; +import org.antlr.v4.runtime.misc.QStack; public class LexerSharedState { public IntStream input; @@ -15,19 +16,28 @@ public class LexerSharedState { * emit another token. */ public Token token; + /** What character index in the stream did the current token start at? * Needed, for example, to get the text for current token. Set at * the start of nextToken. */ public int tokenStartCharIndex = -1; + /** The line on which the first character of the token resides */ public int tokenStartLine; + /** The character position of first character within the line */ public int tokenStartCharPositionInLine; + /** The channel number for the current token */ public int channel; + /** The token type for the current token */ public int type; + + public QStack modeStack; + public int mode = Lexer.DEFAULT_MODE; + /** You can set the text for the current token to override what is in * the input char buffer. Use setText() or can set this instance var. */