jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

23 import grammar unit tests pass 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8896]
This commit is contained in:
parrt 2011-07-20 18:55:23 -08:00
parent 36f3590f6c
commit 40f9e867cd
7 changed files with 102 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
runtime/Java/src/org/antlr/v4/runtime/tree/BaseTree.java
View File

@ -163,6 +163,17 @@ public abstract class BaseTree implements Tree {
return killed;
}
public boolean deleteChild(Tree t) {
for (int i=0; i<children.size(); i++) {
Object c = children.get(i);
if ( c == t ) {
deleteChild(t.getChildIndex());
return true;
}
}
return false;
}
/** Delete children from start to stop and replace with t even if t is
* a list (nil-root tree). num of children can increase or decrease.
* For huge child lists, inserting children can force walking rest of

43
tool/src/org/antlr/v4/Tool.java
View File

@ -318,18 +318,10 @@ public class Tool {
/** Try current dir then dir of g then lib dir */
public GrammarRootAST loadImportedGrammar(Grammar g, String fileName) throws IOException {
System.out.println("loadImportedGrammar "+fileName+" from "+g.fileName);
File importedFile = new File(fileName);
if ( !importedFile.exists() ) {
File gfile = new File(g.fileName);
String parentDir = gfile.getParent();
importedFile = new File(parentDir, fileName);
if ( !importedFile.exists() ) { // try in lib dir
importedFile = new File(libDirectory, fileName);
if ( !importedFile.exists() ) {
errMgr.toolError(ErrorType.CANNOT_FIND_IMPORTED_FILE, g.fileName, fileName);
return null;
}
}
File importedFile = getImportedGrammarFile(g, fileName);
if ( importedFile==null ) {
errMgr.toolError(ErrorType.CANNOT_FIND_IMPORTED_FILE, fileName, g.fileName);
return null;
}
ANTLRFileStream in = new ANTLRFileStream(importedFile.getAbsolutePath());
return load(in);
@ -549,7 +541,11 @@ public class Tool {
}
}
}
elements.removeAll(actionsWeMoved);
for (GrammarAST r : actionsWeMoved) {
combinedAST.deleteChild( r );
}
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.getFirstChildWithType(ANTLRParser.RULES);
if ( combinedRulesRoot==null ) return lexerAST;
@ -606,6 +602,7 @@ public class Tool {
lexerRulesRoot.freshenParentAndChildIndexes(); // reset indexes and set litRule parent
}
// TODO: take out after stable if slow
lexerAST.sanityCheckParentAndChildIndexes();
combinedAST.sanityCheckParentAndChildIndexes();
// System.out.println(combinedAST.toTokenString());
@ -655,7 +652,7 @@ public class Tool {
*
* If outputDirectory==null then write a String.
*/
public Writer getOutputFile(Grammar g, String fileName) throws IOException {
public Writer getOutputFileWriter(Grammar g, String fileName) throws IOException {
if (outputDirectory == null) {
return new StringWriter();
}
@ -680,6 +677,22 @@ public class Tool {
return new BufferedWriter(fw);
}
public File getImportedGrammarFile(Grammar g, String fileName) {
File importedFile = new File(fileName);
if ( !importedFile.exists() ) {
File gfile = new File(g.fileName);
String parentDir = gfile.getParent();
importedFile = new File(parentDir, fileName);
if ( !importedFile.exists() ) { // try in lib dir
importedFile = new File(libDirectory, fileName);
if ( !importedFile.exists() ) {
return null;
}
}
}
return importedFile;
}
/**
* Return the location where ANTLR will generate output files for a given
* file. This is a base directory and output files will be relative to
@ -744,7 +757,7 @@ public class Tool {
}
protected void writeDOTFile(Grammar g, String name, String dot) throws IOException {
Writer fw = getOutputFile(g, name + ".dot");
Writer fw = getOutputFileWriter(g, name + ".dot");
fw.write(dot);
fw.close();
}

2
tool/src/org/antlr/v4/codegen/CodeGenerator.java
View File

@ -193,7 +193,7 @@ public class CodeGenerator {
public void write(ST code, String fileName) throws IOException {
long start = System.currentTimeMillis();
Writer w = g.tool.getOutputFile(g, fileName);
Writer w = g.tool.getOutputFileWriter(g, fileName);
STWriter wr = new AutoIndentWriter(w);
wr.setLineWidth(lineWidth);
code.write(wr);

2
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -128,7 +128,7 @@ public enum ErrorType {
//TOKEN_VOCAB_IN_DELEGATE("tokenVocab option ignored in imported grammar <arg>", ErrorSeverity.ERROR),
OPTIONS_IN_DELEGATE("options ignored in imported grammar <arg>", ErrorSeverity.WARNING),
// TOKEN_ALIAS_IN_DELEGATE("can't assign string to token name <arg> to string in imported grammar <arg2>", ErrorSeverity.ERROR),
CANNOT_FIND_IMPORTED_FILE("can't find or load grammar <arg>", ErrorSeverity.ERROR),
CANNOT_FIND_IMPORTED_FILE("can't find or load grammar <arg> from <arg2>", ErrorSeverity.ERROR),
INVALID_IMPORT("<arg.typeString> grammar <arg.name> cannot import <arg2.typeString> grammar <arg2.name>", ErrorSeverity.ERROR),
IMPORTED_TOKENS_RULE_EMPTY("", ErrorSeverity.ERROR),
IMPORT_NAME_CLASH("<arg.typeString> grammar <arg.name> and imported <arg2.typeString> grammar <arg2.name> both generate <arg2.recognizerName>", ErrorSeverity.ERROR),

9
tool/src/org/antlr/v4/tool/Grammar.java
View File

@ -39,7 +39,7 @@ import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.*;
import org.antlr.v4.semantics.SymbolCollector;
import java.io.IOException;
import java.io.*;
import java.util.*;
public class Grammar implements AttributeResolver {
@ -232,15 +232,18 @@ public class Grammar implements AttributeResolver {
grammarAST = tool.loadImportedGrammar(this, importedGrammarName + ".g");
}
catch (IOException ioe) {
tool.errMgr.toolError(ErrorType.CANNOT_FIND_IMPORTED_FILE, ioe, fileName);
tool.errMgr.toolError(ErrorType.CANNOT_FIND_IMPORTED_FILE, ioe, importedGrammarName+".g");
continue;
}
// did it come back as error node or missing?
if ( grammarAST==null || grammarAST instanceof GrammarASTErrorNode ) return;
GrammarRootAST ast = (GrammarRootAST)grammarAST;
Grammar g = tool.createGrammar(ast);
g.fileName = importedGrammarName+".g";
File f = tool.getImportedGrammarFile(this, importedGrammarName+".g");
g.fileName = f.getAbsolutePath();
g.parent = this;
importedGrammars.add(g);
g.loadImportedGrammars(); // recursively pursue any imports in this import
}
}

13
tool/src/org/antlr/v4/tool/GrammarAST.java
View File

@ -89,12 +89,15 @@ public class GrammarAST extends CommonTree {
return null;
}
public void deleteChild(GrammarAST t) {
List<GrammarAST> dup = new ArrayList<GrammarAST>();
dup.addAll(children);
for (Object c : dup) {
if ( c == t ) deleteChild(t.getChildIndex());
public boolean deleteChild(org.antlr.runtime.tree.Tree t) {
for (int i=0; i<children.size(); i++) {
Object c = children.get(i);
if ( c == t ) {
deleteChild(t.getChildIndex());
return true;
}
}
return false;
}
// TODO: move to basetree when i settle on how runtime works

236
tool/test/org/antlr/v4/test/TestCompositeGrammars.java
View File

@ -359,7 +359,6 @@ public class TestCompositeGrammars extends BaseTest {
assertEquals("JavaDecl: floatx=3;\n", found);
}
/*
@Test public void testDelegatorRuleOverridesDelegates() throws Exception {
String slave =
"parser grammar S;\n" +
@ -384,7 +383,6 @@ public class TestCompositeGrammars extends BaseTest {
assertEquals("M.b\n" +
"S.a\n", found);
}
// LEXER INHERITANCE
@Test public void testLexerDelegatorInvokesDelegateRule() throws Exception {
@ -399,8 +397,14 @@ public class TestCompositeGrammars extends BaseTest {
"import S;\n" +
"B : 'b' ;\n" +
"WS : (' '|'\\n') {skip();} ;\n" ;
String expecting =
"S.A\n" +
"[@0,0:0='a',<5>,1:0]\n" +
"[@1,1:1='b',<3>,1:1]\n" +
"[@2,2:2='c',<6>,1:2]\n" +
"[@3,3:3='<EOF>',<-1>,1:3]\n";
String found = execLexer("M.g", master, "M", "abc", debug);
assertEquals("S.A\nabc\n", found);
assertEquals(expecting, found);
}
@Test public void testLexerDelegatorRuleOverridesDelegate() throws Exception {
@ -416,119 +420,26 @@ public class TestCompositeGrammars extends BaseTest {
"A : 'a' B {System.out.println(\"M.A\");} ;\n" +
"WS : (' '|'\\n') {skip();} ;\n" ;
String found = execLexer("M.g", master, "M", "ab", debug);
assertEquals("S.B\n" +
"M.A\n" +
"ab\n", found);
assertEquals("M.A\n" +
"[@0,0:1='ab',<3>,1:0]\n" +
"[@1,2:2='<EOF>',<-1>,1:2]\n", found);
}
@Test public void testLexerDelegatorRuleOverridesDelegateLeavingNoRules() throws Exception {
// M.Tokens has nothing to predict tokens from S. Should
// not include S.Tokens alt in this case?
@Test public void testKeywordVSIDOrder() throws Exception {
// rules in lexer are imported at END so rules in master override
// *and* get priority over imported rules. So importing ID doesn't
// mess up keywords in master grammar
ErrorQueue equeue = new ErrorQueue();
String slave =
"lexer grammar S;\n" +
"A : 'a' {System.out.println(\"S.A\");} ;\n";
mkdir(tmpdir);
writeFile(tmpdir, "S.g", slave);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' {System.out.println(\"M.A\");} ;\n" +
"WS : (' '|'\\n') {skip();} ;\n" ;
writeFile(tmpdir, "/M.g", master);
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
Tool antlr = newTool(new String[] {"-lib", tmpdir});
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite);
composite.setDelegationRoot(g);
g.parseAndBuildAST();
composite.assignTokenTypes();
composite.defineGrammarSymbols();
composite.createNFAs();
g.createLookaheadDFAs(false);
// predict only alts from M not S
String expectingDFA =
".s0-'a'->.s1\n" +
".s0-{'\\n', ' '}->:s3=>2\n" +
".s1-<EOT>->:s2=>1\n";
org.antlr.analysis.DFA dfa = g.getLookaheadDFA(1);
FASerializer serializer = new FASerializer(g);
String result = serializer.serialize(dfa.startState);
assertEquals(expectingDFA, result);
// must not be a "unreachable alt: Tokens" error
assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
}
@Test public void testInvalidImportMechanism() throws Exception {
// M.Tokens has nothing to predict tokens from S. Should
// not include S.Tokens alt in this case?
String slave =
"lexer grammar S;\n" +
"A : 'a' {System.out.println(\"S.A\");} ;\n";
mkdir(tmpdir);
writeFile(tmpdir, "S.g", slave);
String master =
"tree grammar M;\n" +
"import S;\n" +
"a : A ;";
writeFile(tmpdir, "/M.g", master);
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
Tool antlr = newTool(new String[] {"-lib", tmpdir});
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite);
composite.setDelegationRoot(g);
g.parseAndBuildAST();
assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size());
assertEquals("unexpected errors: "+equeue, 0, equeue.warnings.size());
String expectedError =
"error(161): "+tmpdir.toString().replaceFirst("\\-[0-9]+","")+"/M.g:2:8: tree grammar M cannot import lexer grammar S";
assertEquals(expectedError, equeue.errors.get(0).toString().replaceFirst("\\-[0-9]+",""));
}
@Test public void testSyntacticPredicateRulesAreNotInherited() throws Exception {
// if this compiles, it means that synpred1_S is defined in S.java
// but not MParser.java. MParser has its own synpred1_M which must
// be separate to compile.
String slave =
"parser grammar S;\n" +
"a : 'a' {System.out.println(\"S.a1\");}\n" +
" | 'a' {System.out.println(\"S.a2\");}\n" +
" ;\n" +
"b : 'x' | 'y' {;} ;\n"; // preds generated but not need in DFA here
mkdir(tmpdir);
writeFile(tmpdir, "S.g", slave);
String master =
"grammar M;\n" +
"options {backtrack=true;}\n" +
"import S;\n" +
"start : a b ;\n" +
"nonsense : 'q' | 'q' {;} ;" + // forces def of preds here in M
"WS : (' '|'\\n') {skip();} ;\n" ;
String found = execParser("M.g", master, "MParser", "MLexer",
"start", "ax", debug);
assertEquals("S.a1\n", found);
}
@Test public void testKeywordVSIDGivesNoWarning() throws Exception {
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
String slave =
"lexer grammar S;\n" +
"A : 'abc' {System.out.println(\"S.A\");} ;\n" +
"ID : 'a'..'z'+ ;\n";
mkdir(tmpdir);
writeFile(tmpdir, "S.g", slave);
String master =
"grammar M;\n" +
"import S;\n" +
"a : A {System.out.println(\"M.a\");} ;\n" +
"a : A {System.out.println(\"M.a: \"+$A);} ;\n" +
"A : 'abc' {System.out.println(\"M.A\");} ;\n" +
"WS : (' '|'\\n') {skip();} ;\n" ;
String found = execParser("M.g", master, "MParser", "MLexer",
"a", "abc", debug);
@ -536,46 +447,20 @@ public class TestCompositeGrammars extends BaseTest {
assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
assertEquals("unexpected warnings: "+equeue, 0, equeue.warnings.size());
assertEquals("S.A\nM.a\n", found);
}
@Test public void testWarningForUndefinedToken() throws Exception {
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
String slave =
"lexer grammar S;\n" +
"A : 'abc' {System.out.println(\"S.A\");} ;\n";
mkdir(tmpdir);
writeFile(tmpdir, "S.g", slave);
String master =
"grammar M;\n" +
"import S;\n" +
"a : ABC A {System.out.println(\"M.a\");} ;\n" +
"WS : (' '|'\\n') {skip();} ;\n" ;
// A is defined in S but M should still see it and not give warning.
// only problem is ABC.
rawGenerateAndBuildRecognizer("M.g", master, "MParser", "MLexer", debug);
assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
assertEquals("unexpected warnings: "+equeue, 1, equeue.warnings.size());
String expectedError =
"warning(105): "+tmpdir.toString().replaceFirst("\\-[0-9]+","")+ File.separator+"M.g:3:5: no lexer rule corresponding to token: ABC";
assertEquals(expectedError, equeue.warnings.get(0).toString().replaceFirst("\\-[0-9]+",""));
assertEquals("M.A\n" +
"M.a: [@0,0:2='abc',<3>,1:0]\n", found);
}
// Make sure that M can import S that imports T.
@Test public void test3LevelImport() throws Exception {
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
String slave =
"parser grammar T;\n" +
"a : T ;\n" ;
mkdir(tmpdir);
writeFile(tmpdir, "T.g", slave);
String slave2 =
"parser grammar S;\n" + // A, B, C token type order
"parser grammar S;\n" +
"import T;\n" +
"a : S ;\n" ;
mkdir(tmpdir);
@ -586,23 +471,17 @@ public class TestCompositeGrammars extends BaseTest {
"import S;\n" +
"a : M ;\n" ;
writeFile(tmpdir, "M.g", master);
Tool antlr = newTool(new String[] {"-lib", tmpdir});
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite);
composite.setDelegationRoot(g);
g.parseAndBuildAST();
g.composite.assignTokenTypes();
g.composite.defineGrammarSymbols();
Grammar g = new Grammar(tmpdir+"/M.g", master, equeue);
String expectedTokenIDToTypeMap = "[M=4, S=5, T=6]";
String expectedTokenIDToTypeMap = "{EOF=-1, M=3}"; // S and T aren't imported; overridden
String expectedStringLiteralToTypeMap = "{}";
String expectedTypeToTokenList = "[M, S, T]";
String expectedTypeToTokenList = "[M]";
assertEquals(expectedTokenIDToTypeMap,
realElements(g.composite.tokenIDToTypeMap).toString());
assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString());
g.tokenNameToTypeMap.toString());
assertEquals(expectedStringLiteralToTypeMap, g.stringLiteralToTypeMap.toString());
assertEquals(expectedTypeToTokenList,
realElements(g.composite.typeToTokenList).toString());
realElements(g.typeToTokenList).toString());
assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
@ -614,7 +493,6 @@ public class TestCompositeGrammars extends BaseTest {
@Test public void testBigTreeOfImports() throws Exception {
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
String slave =
"parser grammar T;\n" +
"x : T ;\n" ;
@ -649,25 +527,19 @@ public class TestCompositeGrammars extends BaseTest {
"import S,A;\n" +
"a : M ;\n" ;
writeFile(tmpdir, "M.g", master);
Tool antlr = newTool(new String[] {"-lib", tmpdir});
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite);
composite.setDelegationRoot(g);
g.parseAndBuildAST();
g.composite.assignTokenTypes();
g.composite.defineGrammarSymbols();
Grammar g = new Grammar(tmpdir+"/M.g", master, equeue);
String expectedTokenIDToTypeMap = "[A=4, B=5, C=6, M=7, S=8, T=9]";
assertEquals(equeue.errors.toString(), "[]");
assertEquals(equeue.warnings.toString(), "[]");
String expectedTokenIDToTypeMap = "{EOF=-1, M=3, S=4, T=5, A=6, B=7, C=8}";
String expectedStringLiteralToTypeMap = "{}";
String expectedTypeToTokenList = "[A, B, C, M, S, T]";
String expectedTypeToTokenList = "[M, S, T, A, B, C]";
assertEquals(expectedTokenIDToTypeMap,
realElements(g.composite.tokenIDToTypeMap).toString());
assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString());
g.tokenNameToTypeMap.toString());
assertEquals(expectedStringLiteralToTypeMap, g.stringLiteralToTypeMap.toString());
assertEquals(expectedTypeToTokenList,
realElements(g.composite.typeToTokenList).toString());
assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
realElements(g.typeToTokenList).toString());
boolean ok =
rawGenerateAndBuildRecognizer("M.g", master, "MParser", null, false);
@ -677,7 +549,6 @@ public class TestCompositeGrammars extends BaseTest {
@Test public void testRulesVisibleThroughMultilevelImport() throws Exception {
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
String slave =
"parser grammar T;\n" +
"x : T ;\n" ;
@ -695,23 +566,17 @@ public class TestCompositeGrammars extends BaseTest {
"import S;\n" +
"a : M x ;\n" ; // x MUST BE VISIBLE TO M
writeFile(tmpdir, "M.g", master);
Tool antlr = newTool(new String[] {"-lib", tmpdir});
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar(antlr,tmpdir+"/M.g",composite);
composite.setDelegationRoot(g);
g.parseAndBuildAST();
g.composite.assignTokenTypes();
g.composite.defineGrammarSymbols();
Grammar g = new Grammar(tmpdir+"/M.g", master, equeue);
String expectedTokenIDToTypeMap = "[M=4, S=5, T=6]";
String expectedTokenIDToTypeMap = "{EOF=-1, M=3, T=4}";
String expectedStringLiteralToTypeMap = "{}";
String expectedTypeToTokenList = "[M, S, T]";
String expectedTypeToTokenList = "[M, T]";
assertEquals(expectedTokenIDToTypeMap,
realElements(g.composite.tokenIDToTypeMap).toString());
assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString());
g.tokenNameToTypeMap.toString());
assertEquals(expectedStringLiteralToTypeMap, g.stringLiteralToTypeMap.toString());
assertEquals(expectedTypeToTokenList,
realElements(g.composite.typeToTokenList).toString());
realElements(g.typeToTokenList).toString());
assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
}
@ -719,7 +584,6 @@ public class TestCompositeGrammars extends BaseTest {
@Test public void testNestedComposite() throws Exception {
// Wasn't compiling. http://www.antlr.org/jira/browse/ANTLR-438
ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
String gstr =
"lexer grammar L;\n" +
"T1: '1';\n" +
@ -749,23 +613,17 @@ public class TestCompositeGrammars extends BaseTest {
mkdir(tmpdir);
writeFile(tmpdir, "G3.g", G3str);
Tool antlr = newTool(new String[] {"-lib", tmpdir});
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar(antlr,tmpdir+"/G3.g",composite);
composite.setDelegationRoot(g);
g.parseAndBuildAST();
g.composite.assignTokenTypes();
g.composite.defineGrammarSymbols();
Grammar g = new Grammar(tmpdir+"/G3.g", G3str, equeue);
String expectedTokenIDToTypeMap = "[T1=4, T2=5, T3=6, T4=7]";
String expectedTokenIDToTypeMap = "{EOF=-1, T4=3, T3=4}";
String expectedStringLiteralToTypeMap = "{}";
String expectedTypeToTokenList = "[T1, T2, T3, T4]";
String expectedTypeToTokenList = "[T4, T3]";
assertEquals(expectedTokenIDToTypeMap,
realElements(g.composite.tokenIDToTypeMap).toString());
assertEquals(expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.toString());
g.tokenNameToTypeMap.toString());
assertEquals(expectedStringLiteralToTypeMap, g.stringLiteralToTypeMap.toString());
assertEquals(expectedTypeToTokenList,
realElements(g.composite.typeToTokenList).toString());
realElements(g.typeToTokenList).toString());
assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
@ -774,7 +632,7 @@ public class TestCompositeGrammars extends BaseTest {
boolean expecting = true; // should be ok
assertEquals(expecting, ok);
}
*/
@Test public void testHeadersPropogatedCorrectlyToImportedGrammars() throws Exception {
String slave =
"parser grammar S;\n" +