forked from jasder/antlr
TestPositionAdjustingLexer
This commit is contained in:
Sam Harwell
parent
328dedd778
commit
5db5c3393d
|
|
@ -0,0 +1,112 @@
|
|||
lexer grammar PositionAdjustingLexer;
|
||||
|
||||
@members {
|
||||
@Override
|
||||
public Token nextToken() {
|
||||
if (!(_interp instanceof PositionAdjustingLexerATNSimulator)) {
|
||||
_interp = new PositionAdjustingLexerATNSimulator(this, _ATN, _decisionToDFA, _sharedContextCache);
|
||||
}
|
||||
|
||||
return super.nextToken();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Token emit() {
|
||||
switch (_type) {
|
||||
case TOKENS:
|
||||
handleAcceptPositionForKeyword("tokens");
|
||||
break;
|
||||
|
||||
case LABEL:
|
||||
handleAcceptPositionForIdentifier();
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return super.emit();
|
||||
}
|
||||
|
||||
private boolean handleAcceptPositionForIdentifier() {
|
||||
String tokenText = getText();
|
||||
int identifierLength = 0;
|
||||
while (identifierLength < tokenText.length() && isIdentifierChar(tokenText.charAt(identifierLength))) {
|
||||
identifierLength++;
|
||||
}
|
||||
|
||||
if (getInputStream().index() > _tokenStartCharIndex + identifierLength) {
|
||||
int offset = identifierLength - 1;
|
||||
getInterpreter().resetAcceptPosition(getInputStream(), _tokenStartCharIndex + offset, _tokenStartLine, _tokenStartCharPositionInLine + offset);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean handleAcceptPositionForKeyword(String keyword) {
|
||||
if (getInputStream().index() > _tokenStartCharIndex + keyword.length()) {
|
||||
int offset = keyword.length() - 1;
|
||||
getInterpreter().resetAcceptPosition(getInputStream(), _tokenStartCharIndex + offset, _tokenStartLine, _tokenStartCharPositionInLine + offset);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PositionAdjustingLexerATNSimulator getInterpreter() {
|
||||
return (PositionAdjustingLexerATNSimulator)super.getInterpreter();
|
||||
}
|
||||
|
||||
private static boolean isIdentifierChar(char c) {
|
||||
return Character.isLetterOrDigit(c) || c == '_';
|
||||
}
|
||||
|
||||
protected static class PositionAdjustingLexerATNSimulator extends LexerATNSimulator {
|
||||
|
||||
public PositionAdjustingLexerATNSimulator(Lexer recog, ATN atn,
|
||||
DFA[] decisionToDFA,
|
||||
PredictionContextCache sharedContextCache)
|
||||
{
|
||||
super(recog, atn, decisionToDFA, sharedContextCache);
|
||||
}
|
||||
|
||||
protected void resetAcceptPosition(CharStream input, int index, int line, int charPositionInLine) {
|
||||
input.seek(index);
|
||||
this.line = line;
|
||||
this.charPositionInLine = charPositionInLine;
|
||||
consume(input);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
ASSIGN : '=' ;
|
||||
PLUS_ASSIGN : '+=' ;
|
||||
LCURLY: '{';
|
||||
|
||||
// 'tokens' followed by '{'
|
||||
TOKENS : 'tokens' IGNORED '{';
|
||||
|
||||
// IDENTIFIER followed by '+=' or '='
|
||||
LABEL
|
||||
: IDENTIFIER IGNORED '+'? '='
|
||||
;
|
||||
|
||||
IDENTIFIER
|
||||
: [a-zA-Z_] [a-zA-Z0-9_]*
|
||||
;
|
||||
|
||||
fragment
|
||||
IGNORED
|
||||
: [ \t\r\n]*
|
||||
;
|
||||
|
||||
NEWLINE
|
||||
: [\r\n]+ -> skip
|
||||
;
|
||||
|
||||
WS
|
||||
: [ \t]+ -> skip
|
||||
;
|
||||
|
|
@ -30,8 +30,13 @@
|
|||
|
||||
package org.antlr.v4.test;
|
||||
|
||||
import org.antlr.v4.runtime.misc.Nullable;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
|
||||
public class TestLexerExec extends BaseTest {
|
||||
@Test public void testQuoteTranslation() throws Exception {
|
||||
String grammar =
|
||||
|
|
@ -638,4 +643,60 @@ public class TestLexerExec extends BaseTest {
|
|||
assertEquals(expecting, found);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPositionAdjustingLexer() throws Exception {
|
||||
String grammar = load("PositionAdjustingLexer.g4", null);
|
||||
String input =
|
||||
"tokens\n" +
|
||||
"tokens {\n" +
|
||||
"notLabel\n" +
|
||||
"label1 =\n" +
|
||||
"label2 +=\n" +
|
||||
"notLabel\n";
|
||||
String found = execLexer("PositionAdjustingLexer.g4", grammar, "PositionAdjustingLexer", input);
|
||||
|
||||
final int TOKENS = 4;
|
||||
final int LABEL = 5;
|
||||
final int IDENTIFIER = 6;
|
||||
String expecting =
|
||||
"[@0,0:5='tokens',<" + IDENTIFIER + ">,1:0]\n" +
|
||||
"[@1,7:12='tokens',<" + TOKENS + ">,2:0]\n" +
|
||||
"[@2,14:14='{',<3>,2:7]\n" +
|
||||
"[@3,16:23='notLabel',<" + IDENTIFIER + ">,3:0]\n" +
|
||||
"[@4,25:30='label1',<" + LABEL + ">,4:0]\n" +
|
||||
"[@5,32:32='=',<1>,4:7]\n" +
|
||||
"[@6,34:39='label2',<" + LABEL + ">,5:0]\n" +
|
||||
"[@7,41:42='+=',<2>,5:7]\n" +
|
||||
"[@8,44:51='notLabel',<" + IDENTIFIER + ">,6:0]\n" +
|
||||
"[@9,53:52='<EOF>',<-1>,7:0]\n";
|
||||
|
||||
assertEquals(expecting, found);
|
||||
}
|
||||
|
||||
protected String load(String fileName, @Nullable String encoding)
|
||||
throws IOException
|
||||
{
|
||||
if ( fileName==null ) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String fullFileName = getClass().getPackage().getName().replace('.', '/') + '/' + fileName;
|
||||
int size = 65000;
|
||||
InputStreamReader isr;
|
||||
InputStream fis = getClass().getClassLoader().getResourceAsStream(fullFileName);
|
||||
if ( encoding!=null ) {
|
||||
isr = new InputStreamReader(fis, encoding);
|
||||
}
|
||||
else {
|
||||
isr = new InputStreamReader(fis);
|
||||
}
|
||||
try {
|
||||
char[] data = new char[size];
|
||||
int n = isr.read(data);
|
||||
return new String(data, 0, n);
|
||||
}
|
||||
finally {
|
||||
isr.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue