jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

quote literal was not translated properly. updated Lexer to convert antlr literals to target literals properly. added unit test. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9557]
This commit is contained in:
parrt 2011-12-11 10:37:27 -08:00
parent 1923242b37
commit c32b703555
3 changed files with 44 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -745,7 +745,7 @@ public class <lexer.name> extends Lexer {
public static final String[] tokenNames = { public static final String[] tokenNames = {
"\<INVALID>", "\<INVALID>", "\<INVALID>", "\<INVALID>", "\<INVALID>", "\<INVALID>",
<lexer.tokenNames:{k | "<k>"}; separator=", ", wrap, anchor> <lexer.tokenNames:{t | <t>}; separator=", ", wrap, anchor>
}; };
public static final String[] ruleNames = { public static final String[] ruleNames = {
<lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>

20
tool/src/org/antlr/v4/codegen/model/Lexer.java
View File

@ -29,10 +29,16 @@
package org.antlr.v4.codegen.model; package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.tool.*; import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import java.util.*; import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
public class Lexer extends OutputModelObject { public class Lexer extends OutputModelObject {
public String name; public String name;
@ -64,6 +70,16 @@ public class Lexer extends OutputModelObject {
} }
tokenNames = g.getTokenDisplayNames(); tokenNames = g.getTokenDisplayNames();
for (int i = 0; i < tokenNames.length; i++) {
if ( tokenNames[i]==null ) continue;
CodeGenerator gen = factory.getGenerator();
if ( tokenNames[i].charAt(0)=='\'' ) {
tokenNames[i] = gen.target.getTargetStringLiteralFromANTLRStringLiteral(gen, tokenNames[i]);
}
else {
tokenNames[i] = gen.target.getTargetStringLiteralFromString(tokenNames[i], true);
}
}
ruleNames = g.rules.keySet(); ruleNames = g.rules.keySet();
} }

11
tool/test/org/antlr/v4/test/TestLexerExec.java
View File

@ -3,6 +3,17 @@ package org.antlr.v4.test;
import org.junit.Test; import org.junit.Test;
public class TestLexerExec extends BaseTest { public class TestLexerExec extends BaseTest {
@Test public void testQuoteTranslation() throws Exception {
String grammar =
"lexer grammar L;\n"+
"QUOTE : '\"' ;\n"; // make sure this compiles
String found = execLexer("L.g", grammar, "L", "\"");
String expecting =
"[@0,0:0='\"',<3>,1:0]\n" +
"[@1,1:0='<EOF>',<-1>,1:1]\n";
assertEquals(expecting, found);
}
@Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception { @Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception {
String grammar = String grammar =
"lexer grammar L;\n"+ "lexer grammar L;\n"+