forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

This is one implementation addressing issue 160. 

Change to support import of lexer grammars containing modes into other
lexer grammars. The semantics for this are,
   * sets of channels from all grammars are merged
   * rules of modes found in an imported grammar which are in the root
     grammar are merged into the root grammar mode.
   * modes which are not in the root grammar are added to the root
     grammar, excluding modes which become empty due to a re-definition of
     rules in the root grammar.
This commit is contained in:
Nicolas 2017-11-28 21:57:57 +13:00
parent e158824ca8
commit d02844c813
2 changed files with 205 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java
View File

@ -61,6 +61,131 @@ public class TestCompositeGrammars extends BaseJavaToolTest {
assertEquals(0, equeue.size());
}
@Test public void testImportIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"C : 'c';\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testImportModesIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' -> pushMode(X);\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testImportChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"C : 'c';\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testImportMixedChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"channels {CH_C}\n" +
"C : 'c' -> channel(CH_C);\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testMergeModesIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' -> pushMode(X);\n" +
"mode X;\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testEmptyModesInLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"C : 'e';\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testDelegatesSeeSameTokenType() throws Exception {
String slaveS =
"parser grammar S;\n"+

80
tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java
View File

@ -161,6 +161,7 @@ public class GrammarTransformPipeline {
GrammarAST id = (GrammarAST) root.getChild(0);
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream());
GrammarAST channelsRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.CHANNELS);
GrammarAST tokensRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);
List<GrammarAST> actionRoots = root.getNodesWithType(ANTLRParser.AT);
@ -172,7 +173,39 @@ public class GrammarTransformPipeline {
List<GrammarAST> rootRules = RULES.getNodesWithType(ANTLRParser.RULE);
for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText());
// make list of modes we have in root grammar
List<GrammarAST> rootModes = root.getNodesWithType(ANTLRParser.MODE);
Set<String> rootModeNames = new HashSet<String>();
for (GrammarAST m : rootModes) rootModeNames.add(m.getChild(0).getText());
List<GrammarAST> addedModes = new ArrayList<GrammarAST>();
for (Grammar imp : imports) {
// COPY CHANNELS
GrammarAST imp_channelRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.CHANNELS);
if ( imp_channelRoot != null) {
rootGrammar.tool.log("grammar", "imported channels: "+imp_channelRoot.getChildren());
if (channelsRoot==null) {
channelsRoot = imp_channelRoot.dupTree();
channelsRoot.g = rootGrammar;
root.insertChild(1, channelsRoot); // ^(GRAMMAR ID TOKENS...)
} else {
for (int c = 0; c < imp_channelRoot.getChildCount(); ++c) {
String channel = imp_channelRoot.getChild(c).getText();
boolean channelIsInRootGrammar = false;
for (int rc = 0; rc < channelsRoot.getChildCount(); ++rc) {
String rootChannel = channelsRoot.getChild(rc).getText();
if (rootChannel.equals(channel)) {
channelIsInRootGrammar = true;
break;
}
}
if (!channelIsInRootGrammar) {
channelsRoot.addChild(imp_channelRoot.getChild(c).dupNode());
}
}
}
}
// COPY TOKENS
GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);
if ( imp_tokensRoot!=null ) {
@ -242,7 +275,54 @@ public class GrammarTransformPipeline {
}
}
// COPY MODES
// The strategy is to copy all the mode sections rules across to any
// mode section in the new grammar with the same name or a new
// mode section if no matching mode is resolved. Rules which are
// already in the new grammar are ignored for copy. If the mode
// section being added ends up empty it is not added to the merged
// grammar.
List<GrammarAST> modes = imp.ast.getNodesWithType(ANTLRParser.MODE);
if (modes != null) {
for (GrammarAST m : modes) {
rootGrammar.tool.log("grammar", "imported mode: " + m.toStringTree());
String name = m.getChild(0).getText();
boolean rootAlreadyHasMode = rootModeNames.contains(name);
GrammarAST destinationAST = null;
if (rootAlreadyHasMode) {
for (GrammarAST m2 : rootModes) {
if (m2.getChild(0).getText().equals(name)) {
destinationAST = m2;
break;
}
}
} else {
destinationAST = m.dupNode();
destinationAST.addChild(m.getChild(0).dupNode());
}
int addedRules = 0;
List<GrammarAST> modeRules = m.getAllChildrenWithType(ANTLRParser.RULE);
for (GrammarAST r : modeRules) {
rootGrammar.tool.log("grammar", "imported rule: "+r.toStringTree());
String ruleName = r.getChild(0).getText();
boolean rootAlreadyHasRule = rootRuleNames.contains(ruleName);
if (!rootAlreadyHasRule) {
destinationAST.addChild(r);
addedRules++;
rootRuleNames.add(ruleName);
}
}
if (!rootAlreadyHasMode && addedRules > 0) {
rootGrammar.ast.addChild(destinationAST);
rootModeNames.add(name);
rootModes.add(destinationAST);
}
}
}
// COPY RULES
// Rules copied in the mode copy phase are not copied again.
List<GrammarAST> rules = imp.ast.getNodesWithType(ANTLRParser.RULE);
if ( rules!=null ) {
for (GrammarAST r : rules) {