jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

quote literal was not translated properly. updated Lexer to convert antlr literals to target literals properly. added unit test. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9557]
This commit is contained in:
parrt 2011-12-11 10:37:27 -08:00
parent 1923242b37
commit c32b703555
3 changed files with 44 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -745,7 +745,7 @@ public class <lexer.name> extends Lexer {
public static final String[] tokenNames = {
"\<INVALID>", "\<INVALID>", "\<INVALID>",
<lexer.tokenNames:{k | "<k>"}; separator=", ", wrap, anchor>
<lexer.tokenNames:{t | <t>}; separator=", ", wrap, anchor>
};
public static final String[] ruleNames = {
<lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>

20
tool/src/org/antlr/v4/codegen/model/Lexer.java
View File

@ -29,10 +29,16 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import java.util.*;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
public class Lexer extends OutputModelObject {
public String name;
@ -64,6 +70,16 @@ public class Lexer extends OutputModelObject {
}
tokenNames = g.getTokenDisplayNames();
for (int i = 0; i < tokenNames.length; i++) {
if ( tokenNames[i]==null ) continue;
CodeGenerator gen = factory.getGenerator();
if ( tokenNames[i].charAt(0)=='\'' ) {
tokenNames[i] = gen.target.getTargetStringLiteralFromANTLRStringLiteral(gen, tokenNames[i]);
}
else {
tokenNames[i] = gen.target.getTargetStringLiteralFromString(tokenNames[i], true);
}
}
ruleNames = g.rules.keySet();
}

39
tool/test/org/antlr/v4/test/TestLexerExec.java
View File

@ -3,20 +3,31 @@ package org.antlr.v4.test;
import org.junit.Test;
public class TestLexerExec extends BaseTest {
@Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception {
String grammar =
"lexer grammar L;\n"+
"A : '-' I ;\n" +
"I : '0'..'9'+ ;\n"+
"WS : (' '|'\\n') {skip();} ;";
String found = execLexer("L.g", grammar, "L", "34 -21 3");
String expecting =
"[@0,0:1='34',<4>,1:0]\n" +
"[@1,3:5='-21',<3>,1:3]\n" +
"[@2,7:7='3',<4>,1:7]\n" +
"[@3,8:7='<EOF>',<-1>,1:8]\n"; // EOF has no length so range is 8:7 not 8:8
assertEquals(expecting, found);
}
@Test public void testQuoteTranslation() throws Exception {
String grammar =
"lexer grammar L;\n"+
"QUOTE : '\"' ;\n"; // make sure this compiles
String found = execLexer("L.g", grammar, "L", "\"");
String expecting =
"[@0,0:0='\"',<3>,1:0]\n" +
"[@1,1:0='<EOF>',<-1>,1:1]\n";
assertEquals(expecting, found);
}
@Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception {
String grammar =
"lexer grammar L;\n"+
"A : '-' I ;\n" +
"I : '0'..'9'+ ;\n"+
"WS : (' '|'\\n') {skip();} ;";
String found = execLexer("L.g", grammar, "L", "34 -21 3");
String expecting =
"[@0,0:1='34',<4>,1:0]\n" +
"[@1,3:5='-21',<3>,1:3]\n" +
"[@2,7:7='3',<4>,1:7]\n" +
"[@3,8:7='<EOF>',<-1>,1:8]\n"; // EOF has no length so range is 8:7 not 8:8
assertEquals(expecting, found);
}
@Test public void testActionExecutedInDFA() throws Exception {
String grammar =