forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Move CharStream.EOF to IntStream.EOF

This commit is contained in:
Sam Harwell 2012-10-01 14:09:11 -05:00
parent fa7015f798
commit fa3e6e5867
10 changed files with 35 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java
View File

@ -159,13 +159,13 @@ public class ANTLRInputStream implements CharStream {
if ( i<0 ) {
i++; // e.g., translate LA(-1) to use offset i=0; then data[p+0-1]
if ( (p+i-1) < 0 ) {
return CharStream.EOF; // invalid; no char before first char
return IntStream.EOF; // invalid; no char before first char
}
}
if ( (p+i-1) >= n ) {
//System.out.println("char LA("+i+")=EOF; p="+p);
return CharStream.EOF;
return IntStream.EOF;
}
//System.out.println("char LA("+i+")="+(char)data[p+i-1]+"; p="+p);
//System.out.println("LA("+i+"); p="+p+" n="+n+" data.length="+data.length);

1
runtime/Java/src/org/antlr/v4/runtime/CharStream.java
View File

@ -32,7 +32,6 @@ import org.antlr.v4.runtime.misc.Interval;
/** A source of characters for an ANTLR lexer */
public interface CharStream extends IntStream {
public static final int EOF = -1;
public static final int MIN_CHAR = Character.MIN_VALUE;
public static final int MAX_CHAR = Character.MAX_VALUE-1; // FFFE is max

6
runtime/Java/src/org/antlr/v4/runtime/IntStream.java
View File

@ -32,6 +32,12 @@ package org.antlr.v4.runtime;
* or token type sequence (such as interpretation).
*/
public interface IntStream {
/**
* The value returned by {@link #LA LA()} when the end of the stream is
* reached.
*/
public static final int EOF = -1;
void consume();
/** Get int at current input pointer + i ahead where i=1 is next int.

2
runtime/Java/src/org/antlr/v4/runtime/Lexer.java
View File

@ -165,7 +165,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
recover(e);
ttype = SKIP;
}
if ( _input.LA(1)==CharStream.EOF ) {
if ( _input.LA(1)==IntStream.EOF ) {
_hitEOF = true;
}
if ( _type == Token.INVALID_TYPE ) _type = ttype;

2
runtime/Java/src/org/antlr/v4/runtime/Token.java
View File

@ -43,7 +43,7 @@ public interface Token {
public static final int MIN_USER_TOKEN_TYPE = 1;
public static final int EOF = CharStream.EOF;
public static final int EOF = IntStream.EOF;
/** All tokens go to the parser (unless skip() is called in that rule)
* on a particular "channel". The parser tunes to a particular channel

4
runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java
View File

@ -161,9 +161,9 @@ public class UnbufferedCharStream implements CharStream {
sync(i);
int index = p + i - 1;
if ( index < 0 ) throw new IndexOutOfBoundsException();
if ( index > n ) return CharStream.EOF;
if ( index > n ) return IntStream.EOF;
int c = data[index];
if ( c==(char)CharStream.EOF ) return CharStream.EOF;
if ( c==(char)IntStream.EOF ) return IntStream.EOF;
return c;
}

12
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -229,7 +229,7 @@ public class LexerATNSimulator extends ATNSimulator {
}
// if no edge, pop over to ATN interpreter, update DFA and return
if ( s.edges == null || t >= s.edges.length || t <= CharStream.EOF ||
if ( s.edges == null || t >= s.edges.length || t <= IntStream.EOF ||
s.edges[t] == null )
{
ATN_failover++;
@ -248,7 +248,7 @@ public class LexerATNSimulator extends ATNSimulator {
captureSimState(prevAccept, input, s);
// keep going unless we're at EOF; check if something else could match
// EOF never in DFA
if ( t==CharStream.EOF ) break;
if ( t==IntStream.EOF ) break;
}
consume(input);
@ -297,7 +297,7 @@ public class LexerATNSimulator extends ATNSimulator {
DFAState target = null;
ATNConfigSet reach = null;
if (s != null) {
if ( s.edges != null && t < s.edges.length && t > CharStream.EOF ) {
if ( s.edges != null && t < s.edges.length && t > IntStream.EOF ) {
closure = s.configs;
target = s.edges[t];
if (target == ERROR) {
@ -374,7 +374,7 @@ public class LexerATNSimulator extends ATNSimulator {
}
else {
// if no accept and EOF is first char, return EOF
if ( t==CharStream.EOF && input.index()==startIndex ) {
if ( t==IntStream.EOF && input.index()==startIndex ) {
return Token.EOF;
}
@ -503,7 +503,7 @@ public class LexerATNSimulator extends ATNSimulator {
case Transition.NOT_SET:
NotSetTransition nst = (NotSetTransition)trans;
if (!nst.set.contains(t) && t!=CharStream.EOF) // ~set doesn't not match EOF
if (!nst.set.contains(t) && t!=IntStream.EOF) // ~set doesn't not match EOF
{
if ( debug ) {
System.out.format("match ~set %s\n", nst.set.toString(true));
@ -515,7 +515,7 @@ public class LexerATNSimulator extends ATNSimulator {
return null;
case Transition.WILDCARD:
if (t != CharStream.EOF) {
if (t != IntStream.EOF) {
return trans.target;
}

4
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -33,7 +33,7 @@ import org.antlr.runtime.CommonToken;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.misc.CharSupport;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ActionTransition;
@ -271,7 +271,7 @@ public class LexerATNFactory extends ParserATNFactory {
if ( node.getText().equals("EOF") ) {
ATNState left = newState(node);
ATNState right = newState(node);
left.addTransition(new AtomTransition(right, CharStream.EOF));
left.addTransition(new AtomTransition(right, IntStream.EOF));
return new Handle(left, right);
}
return _ruleRef(node);

3
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -39,6 +39,7 @@ import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
@ -232,7 +233,7 @@ public abstract class BaseTest {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if ( t==CharStream.EOF ) {
if ( t==IntStream.EOF ) {
hitEOF = true;
}
} while ( ttype!=Token.EOF );

25
tool/test/org/antlr/v4/test/TestUnbufferedCharStream.java
View File

@ -32,6 +32,7 @@ package org.antlr.v4.test;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonTokenFactory;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.UnbufferedCharStream;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.interp.LexerInterpreter;
@ -45,11 +46,11 @@ public class TestUnbufferedCharStream extends BaseTest {
CharStream input = new TestingUnbufferedCharStream(
new StringReader("")
);
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
input.consume();
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
input.consume();
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
}
@Test public void test1Char() throws Exception {
@ -58,7 +59,7 @@ public class TestUnbufferedCharStream extends BaseTest {
);
assertEquals('x', input.LA(1));
input.consume();
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
String r = input.getRemainingBuffer();
assertEquals("\uFFFF", r); // shouldn't include x
assertEquals("x\uFFFF", input.getBuffer()); // whole buffer
@ -74,7 +75,7 @@ public class TestUnbufferedCharStream extends BaseTest {
assertEquals("y", input.getRemainingBuffer()); // shouldn't include x
assertEquals("xy", input.getBuffer());
input.consume();
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
}
@Test public void test2CharAhead() throws Exception {
@ -83,7 +84,7 @@ public class TestUnbufferedCharStream extends BaseTest {
);
assertEquals('x', input.LA(1));
assertEquals('y', input.LA(2));
assertEquals(CharStream.EOF, input.LA(3));
assertEquals(IntStream.EOF, input.LA(3));
}
@Test public void testBufferExpand() throws Exception {
@ -97,7 +98,7 @@ public class TestUnbufferedCharStream extends BaseTest {
assertEquals('3', input.LA(4));
assertEquals('4', input.LA(5));
assertEquals("01234", input.getBuffer());
assertEquals(CharStream.EOF, input.LA(6));
assertEquals(IntStream.EOF, input.LA(6));
}
@Test public void testBufferWrapSize1() throws Exception {
@ -115,7 +116,7 @@ public class TestUnbufferedCharStream extends BaseTest {
input.consume();
assertEquals('4', input.LA(1));
input.consume();
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
}
@Test public void testBufferWrapSize2() throws Exception {
@ -133,7 +134,7 @@ public class TestUnbufferedCharStream extends BaseTest {
input.consume();
assertEquals('4', input.LA(1));
input.consume();
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
}
@Test public void test1Mark() throws Exception {
@ -145,7 +146,7 @@ public class TestUnbufferedCharStream extends BaseTest {
assertEquals('y', input.LA(2));
assertEquals('z', input.LA(3));
input.release(m);
assertEquals(CharStream.EOF, input.LA(4));
assertEquals(IntStream.EOF, input.LA(4));
assertEquals("xyz\uFFFF", input.getBuffer());
}
@ -157,7 +158,7 @@ public class TestUnbufferedCharStream extends BaseTest {
input.consume(); // x, moves to y
input.consume(); // y
input.consume(); // z, moves to EOF
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
assertEquals("xyz\uFFFF", input.getBuffer());
input.release(m); // wipes buffer
assertEquals("\uFFFF", input.getBuffer());
@ -180,7 +181,7 @@ public class TestUnbufferedCharStream extends BaseTest {
input.release(m2); // drop to 1 marker
input.consume();
input.release(m1); // shifts remaining char to beginning
assertEquals(CharStream.EOF, input.LA(1));
assertEquals(IntStream.EOF, input.LA(1));
assertEquals("\uFFFF", input.getBuffer());
}