From d02844c8135aae0b935c8f9b17927fd77b0c2fdc Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 28 Nov 2017 21:57:57 +1300 Subject: [PATCH 1/6] This is one implementation addressing issue 160. Change to support import of lexer grammars containing modes into other lexer grammars. The semantics for this are, * sets of channels from all grammars are merged * rules of modes found in an imported grammar which are in the root grammar are merged into the root grammar mode. * modes which are not in the root grammar are added to the root grammar, excluding modes which become empty due to a re-definition of rules in the root grammar. --- .../v4/test/tool/TestCompositeGrammars.java | 125 ++++++++++++++++++ .../v4/tool/GrammarTransformPipeline.java | 80 +++++++++++ 2 files changed, 205 insertions(+) diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java index 75dc35d51..ecfc04f5d 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java @@ -61,6 +61,131 @@ public class TestCompositeGrammars extends BaseJavaToolTest { assertEquals(0, equeue.size()); } + @Test public void testImportIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a';\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "C : 'c';\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testImportModesIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a' -> pushMode(X);\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testImportChannelsIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "channels {CH_A, CH_B}\n" + + "A : 'a' -> channel(CH_A);\n" + + "B : 'b' -> channel(CH_B);\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "C : 'c';\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testImportMixedChannelsIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "channels {CH_A, CH_B}\n" + + "A : 'a' -> channel(CH_A);\n" + + "B : 'b' -> channel(CH_B);\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "channels {CH_C}\n" + + "C : 'c' -> channel(CH_C);\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testMergeModesIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a' -> pushMode(X);\n" + + "mode X;\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testEmptyModesInLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a';\n" + + "C : 'e';\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + @Test public void testDelegatesSeeSameTokenType() throws Exception { String slaveS = "parser grammar S;\n"+ diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index e03652b6e..f6e4cdfbf 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -161,6 +161,7 @@ public class GrammarTransformPipeline { GrammarAST id = (GrammarAST) root.getChild(0); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream()); + GrammarAST channelsRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.CHANNELS); GrammarAST tokensRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.TOKENS_SPEC); List actionRoots = root.getNodesWithType(ANTLRParser.AT); @@ -172,7 +173,39 @@ public class GrammarTransformPipeline { List rootRules = RULES.getNodesWithType(ANTLRParser.RULE); for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText()); + // make list of modes we have in root grammar + List rootModes = root.getNodesWithType(ANTLRParser.MODE); + Set rootModeNames = new HashSet(); + for (GrammarAST m : rootModes) rootModeNames.add(m.getChild(0).getText()); + List addedModes = new ArrayList(); + for (Grammar imp : imports) { + // COPY CHANNELS + GrammarAST imp_channelRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.CHANNELS); + if ( imp_channelRoot != null) { + rootGrammar.tool.log("grammar", "imported channels: "+imp_channelRoot.getChildren()); + if (channelsRoot==null) { + channelsRoot = imp_channelRoot.dupTree(); + channelsRoot.g = rootGrammar; + root.insertChild(1, channelsRoot); // ^(GRAMMAR ID TOKENS...) + } else { + for (int c = 0; c < imp_channelRoot.getChildCount(); ++c) { + String channel = imp_channelRoot.getChild(c).getText(); + boolean channelIsInRootGrammar = false; + for (int rc = 0; rc < channelsRoot.getChildCount(); ++rc) { + String rootChannel = channelsRoot.getChild(rc).getText(); + if (rootChannel.equals(channel)) { + channelIsInRootGrammar = true; + break; + } + } + if (!channelIsInRootGrammar) { + channelsRoot.addChild(imp_channelRoot.getChild(c).dupNode()); + } + } + } + } + // COPY TOKENS GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS_SPEC); if ( imp_tokensRoot!=null ) { @@ -242,7 +275,54 @@ public class GrammarTransformPipeline { } } + // COPY MODES + // The strategy is to copy all the mode sections rules across to any + // mode section in the new grammar with the same name or a new + // mode section if no matching mode is resolved. Rules which are + // already in the new grammar are ignored for copy. If the mode + // section being added ends up empty it is not added to the merged + // grammar. + List modes = imp.ast.getNodesWithType(ANTLRParser.MODE); + if (modes != null) { + for (GrammarAST m : modes) { + rootGrammar.tool.log("grammar", "imported mode: " + m.toStringTree()); + String name = m.getChild(0).getText(); + boolean rootAlreadyHasMode = rootModeNames.contains(name); + GrammarAST destinationAST = null; + if (rootAlreadyHasMode) { + for (GrammarAST m2 : rootModes) { + if (m2.getChild(0).getText().equals(name)) { + destinationAST = m2; + break; + } + } + } else { + destinationAST = m.dupNode(); + destinationAST.addChild(m.getChild(0).dupNode()); + } + + int addedRules = 0; + List modeRules = m.getAllChildrenWithType(ANTLRParser.RULE); + for (GrammarAST r : modeRules) { + rootGrammar.tool.log("grammar", "imported rule: "+r.toStringTree()); + String ruleName = r.getChild(0).getText(); + boolean rootAlreadyHasRule = rootRuleNames.contains(ruleName); + if (!rootAlreadyHasRule) { + destinationAST.addChild(r); + addedRules++; + rootRuleNames.add(ruleName); + } + } + if (!rootAlreadyHasMode && addedRules > 0) { + rootGrammar.ast.addChild(destinationAST); + rootModeNames.add(name); + rootModes.add(destinationAST); + } + } + } + // COPY RULES + // Rules copied in the mode copy phase are not copied again. List rules = imp.ast.getNodesWithType(ANTLRParser.RULE); if ( rules!=null ) { for (GrammarAST r : rules) { From d397b9a65a27797dd53584a2f781d131e8ba9584 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 28 Nov 2017 23:07:35 +1300 Subject: [PATCH 2/6] Add my name to contributions file. --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index a270f1248..5955629b2 100644 --- a/contributors.txt +++ b/contributors.txt @@ -171,3 +171,4 @@ YYYY/MM/DD, github id, Full name, email 2017/05/29, rlfnb, Ralf Neeb, rlfnb@rlfnb.de 2017/10/29, gendalph, Максим Прохоренко, Maxim\dotProhorenko@gm@il.com 2017/11/02, jasonmoo, Jason Mooberry, jason.mooberry@gmail.com +2017/11/28, niccroad, Nicolas Croad, nic.croad@gmail.com From 0301bb946cc783b9386d2e9998574ef3c75a09bf Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 30 Nov 2017 23:24:11 +1300 Subject: [PATCH 3/6] Add a test case for importing modal lexers into a composite grammar. The test case demonstrates an error is produced (due to importing modes into a composite grammar). --- .../v4/test/tool/TestCompositeGrammars.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java index ecfc04f5d..460d1539d 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java @@ -184,6 +184,34 @@ public class TestCompositeGrammars extends BaseJavaToolTest { ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); assertEquals(0, equeue.errors.size()); + } + + @Test public void testCombinedGrammarImportsModalLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "grammar M;\n" + + "import S;\n" + + "A : 'a';\n" + + "B : 'b';\n" + + "r : A B;\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(1, equeue.errors.size()); + ANTLRMessage msg = equeue.errors.get(0); + assertEquals(ErrorType.MODE_NOT_IN_LEXER, msg.getErrorType()); + assertEquals("X", msg.getArgs()[0]); + assertEquals(3, msg.line); + assertEquals(5, msg.charPosition); + assertEquals("M.g4", new File(msg.fileName).getName()); } @Test public void testDelegatesSeeSameTokenType() throws Exception { From 490cd7068cf9a80fa5cb9d00e530fa479d1d9047 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 4 Dec 2017 17:25:51 +1300 Subject: [PATCH 4/6] Make some changes to the docs for grammars to describe support for lexers containing modes. --- doc/grammars.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/grammars.md b/doc/grammars.md index c40d974b6..5f88bf20a 100644 --- a/doc/grammars.md +++ b/doc/grammars.md @@ -81,7 +81,10 @@ $ grun MyELang stat <= line 3:0 extraneous input ';' expecting {INT, ID} ``` -If there were any `tokens` specifications, the main grammar would merge the token sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars. +If there are modes in the main grammar or any of the imported grammars then the import process will import those modes and merge their rules where they are not overridden. In the event any mode becomes empty as all its +rules have been overridden by rules outside the mode this mode will be discarded. + +If there were any `tokens` specifications, the main grammar would merge the token sets. If there were any `channel` specifications, the main grammar would merge the channel sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars. Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define rule `r`, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`. @@ -91,9 +94,9 @@ Imported grammars can also import other grammars. ANTLR pursues all imported gra Not every kind of grammar can import every other kind of grammar: -* Lexer grammars can import lexers. +* Lexer grammars can import lexers, including lexers containing modes. * Parsers can import parsers. -* Combined grammars can import lexers or parsers. +* Combined grammars can import parsers or lexers without modes. ANTLR adds imported rules to the end of the rule list in a main lexer grammar. That means lexer rules in the main grammar get precedence over imported rules. For example, if a main grammar defines rule `IF : ’if’ ;` and an imported grammar defines rule `ID : [a-z]+ ;` (which also recognizes `if`), the imported `ID` won’t hide the main grammar’s `IF` token definition. From d2685bf37ffdfa23bcacaf494af25473e8a0f4a9 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 4 Dec 2017 17:35:08 +1300 Subject: [PATCH 5/6] Add a test case for merging clashing channel names during import. --- .../v4/test/tool/TestCompositeGrammars.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java index 460d1539d..03368785b 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java @@ -141,6 +141,28 @@ public class TestCompositeGrammars extends BaseJavaToolTest { ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); assertEquals(0, equeue.errors.size()); } + + @Test public void testImportClashingChannelsIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "channels {CH_A, CH_B, CH_C}\n" + + "A : 'a' -> channel(CH_A);\n" + + "B : 'b' -> channel(CH_B);\n" + + "C : 'C' -> channel(CH_C);\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "channels {CH_C}\n" + + "C : 'c' -> channel(CH_C);\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } @Test public void testMergeModesIntoLexerGrammar() throws Exception { BaseRuntimeTest.mkdir(tmpdir); From b9adef5878c5b84eb9b70bad2df56334fb7e1b9e Mon Sep 17 00:00:00 2001 From: parrt Date: Mon, 4 Dec 2017 14:12:20 -0800 Subject: [PATCH 6/6] set version for C# dotnet to 4.7.1.1 for publishing --- .../runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj index 23cbfa49f..bc6e6467e 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj @@ -1,7 +1,7 @@  The ANTLR Organization - 4.7.2 + 4.7.1.1 en-US netstandard1.3;net35 $(NoWarn);CS1591;CS1574;CS1580