jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge branch 'master' into lexer-atn-prop

This commit is contained in:
Terence Parr 2017-12-06 12:10:14 -08:00 committed by GitHub
parent e22fbbcce8 b9adef5878
commit dd4a1c8709
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 263 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contributors.txt
View File

@ -174,4 +174,5 @@ YYYY/MM/DD, github id, Full name, email
2017/11/02, jasonmoo, Jason Mooberry, jason.mooberry@gmail.com
2017/11/05, ajaypanyala, Ajay Panyala, ajay.panyala@gmail.com
2017/11/24, zqlu.cn, Zhiqiang Lu, zqlu.cn@gmail.com
2017/11/28, niccroad, Nicolas Croad, nic.croad@gmail.com
2017/12/03, oranoran, Oran Epelbaum, oran / epelbaum me

9
doc/grammars.md
View File

@ -81,7 +81,10 @@ $ grun MyELang stat
<= line 3:0 extraneous input ';' expecting {INT, ID}
```
If there were any `tokens` specifications, the main grammar would merge the token sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars.
If there are modes in the main grammar or any of the imported grammars then the import process will import those modes and merge their rules where they are not overridden. In the event any mode becomes empty as all its
rules have been overridden by rules outside the mode this mode will be discarded.
If there were any `tokens` specifications, the main grammar would merge the token sets. If there were any `channel` specifications, the main grammar would merge the channel sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars.
Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define rule `r`, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`.
@ -91,9 +94,9 @@ Imported grammars can also import other grammars. ANTLR pursues all imported gra
Not every kind of grammar can import every other kind of grammar:
* Lexer grammars can import lexers.
* Lexer grammars can import lexers, including lexers containing modes.
* Parsers can import parsers.
* Combined grammars can import lexers or parsers.
* Combined grammars can import parsers or lexers without modes.
ANTLR adds imported rules to the end of the rule list in a main lexer grammar. That means lexer rules in the main grammar get precedence over imported rules. For example, if a main grammar defines rule `IF : ’if’ ;` and an imported grammar defines rule `ID : [a-z]+ ;` (which also recognizes `if`), the imported `ID` won’t hide the main grammar’s `IF` token definition.

2
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Company>The ANTLR Organization</Company>
<Version>4.7.2</Version>
<Version>4.7.1.1</Version>
<NeutralLanguage>en-US</NeutralLanguage>
<TargetFrameworks>netstandard1.3;net35</TargetFrameworks>
<NoWarn>$(NoWarn);CS1591;CS1574;CS1580</NoWarn>

175
tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java
View File

@ -61,6 +61,181 @@ public class TestCompositeGrammars extends BaseJavaToolTest {
assertEquals(0, equeue.size());
}
@Test public void testImportIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"C : 'c';\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testImportModesIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' -> pushMode(X);\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testImportChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"C : 'c';\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testImportMixedChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"channels {CH_C}\n" +
"C : 'c' -> channel(CH_C);\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testImportClashingChannelsIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"channels {CH_A, CH_B, CH_C}\n" +
"A : 'a' -> channel(CH_A);\n" +
"B : 'b' -> channel(CH_B);\n" +
"C : 'C' -> channel(CH_C);\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"channels {CH_C}\n" +
"C : 'c' -> channel(CH_C);\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testMergeModesIntoLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' -> pushMode(X);\n" +
"mode X;\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testEmptyModesInLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"C : 'e';\n" +
"B : 'b';\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(0, equeue.errors.size());
}
@Test public void testCombinedGrammarImportsModalLexerGrammar() throws Exception {
BaseRuntimeTest.mkdir(tmpdir);
String master =
"grammar M;\n" +
"import S;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"r : A B;\n";
writeFile(tmpdir, "M.g4", master);
String slave =
"lexer grammar S;\n" +
"D : 'd';\n" +
"mode X;\n" +
"C : 'c' -> popMode;\n";
writeFile(tmpdir, "S.g4", slave);
ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir);
assertEquals(1, equeue.errors.size());
ANTLRMessage msg = equeue.errors.get(0);
assertEquals(ErrorType.MODE_NOT_IN_LEXER, msg.getErrorType());
assertEquals("X", msg.getArgs()[0]);
assertEquals(3, msg.line);
assertEquals(5, msg.charPosition);
assertEquals("M.g4", new File(msg.fileName).getName());
}
@Test public void testDelegatesSeeSameTokenType() throws Exception {
String slaveS =
"parser grammar S;\n"+

80
tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java
View File

@ -161,6 +161,7 @@ public class GrammarTransformPipeline {
GrammarAST id = (GrammarAST) root.getChild(0);
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream());
GrammarAST channelsRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.CHANNELS);
GrammarAST tokensRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);
List<GrammarAST> actionRoots = root.getNodesWithType(ANTLRParser.AT);
@ -172,7 +173,39 @@ public class GrammarTransformPipeline {
List<GrammarAST> rootRules = RULES.getNodesWithType(ANTLRParser.RULE);
for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText());
// make list of modes we have in root grammar
List<GrammarAST> rootModes = root.getNodesWithType(ANTLRParser.MODE);
Set<String> rootModeNames = new HashSet<String>();
for (GrammarAST m : rootModes) rootModeNames.add(m.getChild(0).getText());
List<GrammarAST> addedModes = new ArrayList<GrammarAST>();
for (Grammar imp : imports) {
// COPY CHANNELS
GrammarAST imp_channelRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.CHANNELS);
if ( imp_channelRoot != null) {
rootGrammar.tool.log("grammar", "imported channels: "+imp_channelRoot.getChildren());
if (channelsRoot==null) {
channelsRoot = imp_channelRoot.dupTree();
channelsRoot.g = rootGrammar;
root.insertChild(1, channelsRoot); // ^(GRAMMAR ID TOKENS...)
} else {
for (int c = 0; c < imp_channelRoot.getChildCount(); ++c) {
String channel = imp_channelRoot.getChild(c).getText();
boolean channelIsInRootGrammar = false;
for (int rc = 0; rc < channelsRoot.getChildCount(); ++rc) {
String rootChannel = channelsRoot.getChild(rc).getText();
if (rootChannel.equals(channel)) {
channelIsInRootGrammar = true;
break;
}
}
if (!channelIsInRootGrammar) {
channelsRoot.addChild(imp_channelRoot.getChild(c).dupNode());
}
}
}
}
// COPY TOKENS
GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);
if ( imp_tokensRoot!=null ) {
@ -242,7 +275,54 @@ public class GrammarTransformPipeline {
}
}
// COPY MODES
// The strategy is to copy all the mode sections rules across to any
// mode section in the new grammar with the same name or a new
// mode section if no matching mode is resolved. Rules which are
// already in the new grammar are ignored for copy. If the mode
// section being added ends up empty it is not added to the merged
// grammar.
List<GrammarAST> modes = imp.ast.getNodesWithType(ANTLRParser.MODE);
if (modes != null) {
for (GrammarAST m : modes) {
rootGrammar.tool.log("grammar", "imported mode: " + m.toStringTree());
String name = m.getChild(0).getText();
boolean rootAlreadyHasMode = rootModeNames.contains(name);
GrammarAST destinationAST = null;
if (rootAlreadyHasMode) {
for (GrammarAST m2 : rootModes) {
if (m2.getChild(0).getText().equals(name)) {
destinationAST = m2;
break;
}
}
} else {
destinationAST = m.dupNode();
destinationAST.addChild(m.getChild(0).dupNode());
}
int addedRules = 0;
List<GrammarAST> modeRules = m.getAllChildrenWithType(ANTLRParser.RULE);
for (GrammarAST r : modeRules) {
rootGrammar.tool.log("grammar", "imported rule: "+r.toStringTree());
String ruleName = r.getChild(0).getText();
boolean rootAlreadyHasRule = rootRuleNames.contains(ruleName);
if (!rootAlreadyHasRule) {
destinationAST.addChild(r);
addedRules++;
rootRuleNames.add(ruleName);
}
}
if (!rootAlreadyHasMode && addedRules > 0) {
rootGrammar.ast.addChild(destinationAST);
rootModeNames.add(name);
rootModes.add(destinationAST);
}
}
}
// COPY RULES
// Rules copied in the mode copy phase are not copied again.
List<GrammarAST> rules = imp.ast.getNodesWithType(ANTLRParser.RULE);
if ( rules!=null ) {
for (GrammarAST r : rules) {