jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Deterministic token names and types for string literals

This commit is contained in:
Sam Harwell 2014-09-28 16:01:06 -05:00
parent 0942edbedd
commit 08ed19078d
5 changed files with 16 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tool/src/org/antlr/v4/tool/Grammar.java
View File

@ -70,6 +70,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -1180,7 +1181,7 @@ public class Grammar implements AttributeResolver {
}
public Set<String> getStringLiterals() {
final Set<String> strings = new HashSet<String>();
final Set<String> strings = new LinkedHashSet<String>();
GrammarTreeVisitor collector = new GrammarTreeVisitor() {
@Override
public void stringRef(TerminalAST ref) {

6
tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java
View File

@ -418,6 +418,7 @@ public class GrammarTransformPipeline {
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
nextLit:
for (String lit : stringLiterals) {
// if lexer already has a rule for literal, continue
@ -439,9 +440,12 @@ public class GrammarTransformPipeline {
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.addChild(new TerminalAST(idToken));
litRule.addChild(blk);
lexerRulesRoot.insertChild(0, litRule); // add first
lexerRulesRoot.insertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
lexerRulesRoot.freshenParentAndChildIndexes(); // reset indexes and set litRule parent
// next literal will be added after the one just added
insertIndex++;
}
// TODO: take out after stable if slow

12
tool/test/org/antlr/v4/test/TestFullContextParsing.java
View File

@ -196,8 +196,8 @@ public class TestFullContextParsing extends BaseTest {
input, true);
expecting =
"Decision 1:\n" +
"s0-'else'->:s1^=>1\n" +
"s0-'}'->:s2=>2\n";
"s0-'}'->:s2=>2\n" +
"s0-'else'->:s1^=>1\n";
assertEquals(expecting, result);
assertEquals("line 1:29 reportAttemptingFullContext d=1 (stat), input='else'\n" +
"line 1:38 reportAmbiguity d=1 (stat): ambigAlts={1, 2}, input='elsefoo}'\n",
@ -228,8 +228,8 @@ public class TestFullContextParsing extends BaseTest {
input, true);
expecting =
"Decision 1:\n" +
"s0-'else'->:s1^=>1\n" +
"s0-'}'->:s2=>2\n";
"s0-'}'->:s2=>2\n" +
"s0-'else'->:s1^=>1\n";
assertEquals(expecting, result);
assertEquals("line 1:19 reportAttemptingFullContext d=1 (stat), input='else'\n" +
"line 1:19 reportContextSensitivity d=1 (stat), input='else'\n" +
@ -244,8 +244,8 @@ public class TestFullContextParsing extends BaseTest {
input, true);
expecting =
"Decision 1:\n" +
"s0-'else'->:s1^=>1\n" +
"s0-'}'->:s2=>2\n";
"s0-'}'->:s2=>2\n" +
"s0-'else'->:s1^=>1\n";
assertEquals(expecting, result);
assertEquals("line 1:19 reportAttemptingFullContext d=1 (stat), input='else'\n" +
"line 1:19 reportContextSensitivity d=1 (stat), input='else'\n" +

2
tool/test/org/antlr/v4/test/TestLexerErrors.java
View File

@ -201,7 +201,7 @@ public class TestLexerErrors extends BaseTest {
String result = execLexer("T.g4", grammar, "TLexer", "x : x", false);
String expecting =
"[@0,0:0='x',<3>,1:0]\n" +
"[@1,2:2=':',<2>,1:2]\n" +
"[@1,2:2=':',<1>,1:2]\n" +
"[@2,4:4='x',<3>,1:4]\n" +
"[@3,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, result);

4
tool/test/org/antlr/v4/test/TestParseErrors.java
View File

@ -82,7 +82,7 @@ public class TestParseErrors extends BaseTest {
"grammar T;\n" +
"a : 'a' x='b' {System.out.println(\"conjured=\"+$x);} 'c' ;";
String result = execParser("T.g4", grammar, "TParser", "TLexer", "a", "ac", false);
String expecting = "conjured=[@-1,-1:-1='<missing 'b'>',<1>,1:1]\n";
String expecting = "conjured=[@-1,-1:-1='<missing 'b'>',<2>,1:1]\n";
assertEquals(expecting, result);
}
@ -101,7 +101,7 @@ public class TestParseErrors extends BaseTest {
"grammar T;\n" +
"a : 'a' x=('b'|'c') {System.out.println(\"conjured=\"+$x);} 'd' ;";
String result = execParser("T.g4", grammar, "TParser", "TLexer", "a", "ad", false);
String expecting = "conjured=[@-1,-1:-1='<missing 'b'>',<1>,1:1]\n";
String expecting = "conjured=[@-1,-1:-1='<missing 'b'>',<2>,1:1]\n";
assertEquals(expecting, result);
}