From 5cf4cba9c7e00d334c3681d26482772289cbaf76 Mon Sep 17 00:00:00 2001 From: parrt Date: Mon, 31 Oct 2011 16:27:34 -0800 Subject: [PATCH] fix bug in modes [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9231] --- .../Java/src/org/antlr/v4/runtime/Lexer.java | 48 +++++----- .../v4/runtime/LexerRecognitionExeption.java | 45 +++++----- .../v4/runtime/atn/LexerATNSimulator.java | 47 +++++----- tool/playground/L.g | 8 +- tool/playground/TestL.java | 35 +++++++- .../antlr/v4/automata/LexerATNFactory.java | 58 ++++++------ .../org/antlr/v4/codegen/CodeGenerator.java | 59 ++++++------ tool/src/org/antlr/v4/misc/MultiMap.java | 45 ++++++++++ tool/src/org/antlr/v4/tool/LexerGrammar.java | 46 +++++----- .../antlr/v4/test/TestATNSerialization.java | 90 ++++++++++++++++++- .../org/antlr/v4/test/TestLexerErrors.java | 46 +++++----- 11 files changed, 355 insertions(+), 172 deletions(-) create mode 100644 tool/src/org/antlr/v4/misc/MultiMap.java diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index 0eb0a272b..cb5cc2b6d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime; @@ -171,7 +171,7 @@ public abstract class Lexer extends Recognizer } public void pushMode(int m) { -// System.out.println("pushMode "+m); + if ( LexerATNSimulator.debug ) System.out.println("pushMode "+m); if ( modeStack==null ) modeStack = new QStack(); modeStack.push(mode); mode(m); @@ -179,7 +179,7 @@ public abstract class Lexer extends Recognizer public int popMode() { if ( modeStack==null ) throw new EmptyStackException(); -// System.out.println("popMode back to "+modeStack.peek()); + if ( LexerATNSimulator.debug ) System.out.println("popMode back to "+modeStack.peek()); mode( modeStack.pop() ); return mode; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java b/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java index 21a4b4054..fbdb7d234 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java @@ -1,34 +1,35 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime; +// TODO: del or rename LexerNoViableAlt? public class LexerRecognitionExeption extends RuntimeException { /** Who threw the exception? */ public Lexer lexer; diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 76cd262d7..6b72e004c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -1,36 +1,37 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime.atn; import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.dfa.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.dfa.DFAState; import org.antlr.v4.runtime.misc.OrderedHashSet; /** "dup" of ParserInterpreter */ diff --git a/tool/playground/L.g b/tool/playground/L.g index da77adf36..0ad9f25b9 100644 --- a/tool/playground/L.g +++ b/tool/playground/L.g @@ -1,3 +1,7 @@ lexer grammar L; -I : '0'..'9'+ {System.out.println("I");} ; -WS : (' '|'\n') {skip();} ; +STRING_START : '"' {pushMode(STRING_MODE); more();} ; +WS : ' '|' +' {skip();} ; +mode STRING_MODE; +STRING : '"' {popMode();} ; +ANY : . {more();} ; diff --git a/tool/playground/TestL.java b/tool/playground/TestL.java index 713500c96..1e2eccc9b 100644 --- a/tool/playground/TestL.java +++ b/tool/playground/TestL.java @@ -1,9 +1,40 @@ -import org.antlr.v4.runtime.*; +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import org.antlr.v4.runtime.ANTLRFileStream; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonTokenStream; public class TestL { public static void main(String[] args) throws Exception { CharStream input = new ANTLRFileStream(args[0]); - input = new ANTLRStringStream("3 3"); +// input = new ANTLRStringStream("3 3"); L lexer = new L(input); CommonTokenStream tokens = new CommonTokenStream(lexer); tokens.fill(); diff --git a/tool/src/org/antlr/v4/automata/LexerATNFactory.java b/tool/src/org/antlr/v4/automata/LexerATNFactory.java index 780aa990c..14850ed27 100644 --- a/tool/src/org/antlr/v4/automata/LexerATNFactory.java +++ b/tool/src/org/antlr/v4/automata/LexerATNFactory.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.automata; @@ -35,17 +35,23 @@ import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.misc.IntervalSet; -import org.antlr.v4.tool.*; -import org.antlr.v4.tool.ast.*; +import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.Rule; +import org.antlr.v4.tool.ast.ActionAST; +import org.antlr.v4.tool.ast.GrammarAST; +import org.antlr.v4.tool.ast.TerminalAST; import java.util.List; +import java.util.Set; public class LexerATNFactory extends ParserATNFactory { public LexerATNFactory(LexerGrammar g) { super(g); } public ATN createATN() { // BUILD ALL START STATES (ONE PER MODE) - for (String modeName : ((LexerGrammar)g).modes.keySet()) { + System.out.println(((LexerGrammar)g).modes); + Set modes = ((LexerGrammar) g).modes.keySet(); + for (String modeName : modes) { // create s0, start state; implied Tokens rule node TokensStartState startState = (TokensStartState)newState(TokensStartState.class, null); @@ -66,7 +72,7 @@ public class LexerATNFactory extends ParserATNFactory { _createATN(g.rules.values()); // LINK MODE START STATE TO EACH TOKEN RULE - for (String modeName : ((LexerGrammar)g).modes.keySet()) { + for (String modeName : modes) { List rules = ((LexerGrammar)g).modes.get(modeName); TokensStartState startState = atn.modeNameToStartState.get(modeName); for (Rule r : rules) { diff --git a/tool/src/org/antlr/v4/codegen/CodeGenerator.java b/tool/src/org/antlr/v4/codegen/CodeGenerator.java index 03468c695..a26578d5f 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGenerator.java +++ b/tool/src/org/antlr/v4/codegen/CodeGenerator.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.codegen; @@ -32,12 +32,16 @@ package org.antlr.v4.codegen; import org.antlr.v4.Tool; import org.antlr.v4.codegen.model.OutputModelObject; import org.antlr.v4.runtime.Token; -import org.antlr.v4.tool.*; +import org.antlr.v4.tool.ErrorType; +import org.antlr.v4.tool.Grammar; import org.stringtemplate.v4.*; -import java.io.*; -import java.lang.reflect.*; -import java.util.*; +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; /** General controller for code gen. Can instantiate sub generator(s). */ @@ -141,6 +145,9 @@ public class CodeGenerator { //if ( templates.isDefined("headerFile") ) headerFileST.inspect(); } +// String x = ATNSerializer.getDecoded(g, g.atn); +// System.out.println(x); + return st; } diff --git a/tool/src/org/antlr/v4/misc/MultiMap.java b/tool/src/org/antlr/v4/misc/MultiMap.java new file mode 100644 index 000000000..6ec929896 --- /dev/null +++ b/tool/src/org/antlr/v4/misc/MultiMap.java @@ -0,0 +1,45 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.misc; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; + +public class MultiMap extends LinkedHashMap> { + public void map(K key, V value) { + List elementsForKey = get(key); + if ( elementsForKey==null ) { + elementsForKey = new ArrayList(); + super.put(key, elementsForKey); + } + elementsForKey.add(value); + } +} diff --git a/tool/src/org/antlr/v4/tool/LexerGrammar.java b/tool/src/org/antlr/v4/tool/LexerGrammar.java index 45cfcd0d0..b5dfe31c7 100644 --- a/tool/src/org/antlr/v4/tool/LexerGrammar.java +++ b/tool/src/org/antlr/v4/tool/LexerGrammar.java @@ -1,38 +1,38 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.tool; import org.antlr.runtime.RecognitionException; import org.antlr.v4.Tool; +import org.antlr.v4.misc.MultiMap; import org.antlr.v4.tool.ast.GrammarRootAST; -import org.stringtemplate.v4.misc.MultiMap; /** */ public class LexerGrammar extends Grammar { diff --git a/tool/test/org/antlr/v4/test/TestATNSerialization.java b/tool/test/org/antlr/v4/test/TestATNSerialization.java index 8a7b5d142..18731aa8a 100644 --- a/tool/test/org/antlr/v4/test/TestATNSerialization.java +++ b/tool/test/org/antlr/v4/test/TestATNSerialization.java @@ -1,8 +1,39 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + package org.antlr.v4.test; import org.antlr.v4.automata.ATNSerializer; import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.tool.*; +import org.antlr.v4.tool.DOTGenerator; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; import org.junit.Test; public class TestATNSerialization extends BaseTest { @@ -638,6 +669,63 @@ public class TestATNSerialization extends BaseTest { assertEquals(expecting, result); } + @Test public void testModeInLexer() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "A : 'a'\n ;\n" + + "B : 'b';\n" + + "mode A;\n" + + "C : 'c';\n"+ + "D : 'd';\n"); + String expecting = + "max type 6\n" + + "0:TOKEN_START -1\n" + + "1:TOKEN_START -1\n" + + "2:RULE_START 0\n" + + "3:RULE_STOP 0\n" + + "4:RULE_START 1\n" + + "5:RULE_STOP 1\n" + + "6:RULE_START 2\n" + + "7:RULE_STOP 2\n" + + "8:RULE_START 3\n" + + "9:RULE_STOP 3\n" + + "10:BASIC 0\n" + + "11:BASIC 0\n" + + "12:BASIC 1\n" + + "13:BASIC 1\n" + + "14:BASIC 2\n" + + "15:BASIC 2\n" + + "16:BASIC 3\n" + + "17:BASIC 3\n" + + "rule 0:2 3,-1\n" + + "rule 1:4 4,-1\n" + + "rule 2:6 5,-1\n" + + "rule 3:8 6,-1\n" + + "mode 0:0\n" + + "mode 1:1\n" + + "0->2 EPSILON 0,0,0\n" + + "0->4 EPSILON 0,0,0\n" + + "1->6 EPSILON 0,0,0\n" + + "1->8 EPSILON 0,0,0\n" + + "2->10 EPSILON 0,0,0\n" + + "4->12 EPSILON 0,0,0\n" + + "6->14 EPSILON 0,0,0\n" + + "8->16 EPSILON 0,0,0\n" + + "10->11 ATOM 97,0,0\n" + + "11->3 EPSILON 0,0,0\n" + + "12->13 ATOM 98,0,0\n" + + "13->5 EPSILON 0,0,0\n" + + "14->15 ATOM 99,0,0\n" + + "15->7 EPSILON 0,0,0\n" + + "16->17 ATOM 100,0,0\n" + + "17->9 EPSILON 0,0,0\n" + + "0:0 1\n" + + "1:1 1\n"; + ATN atn = createATN(lg); + String result = ATNSerializer.getDecoded(lg, atn); + assertEquals(expecting, result); + } + @Test public void test2ModesInLexer() throws Exception { LexerGrammar lg = new LexerGrammar( "lexer grammar L;\n"+ diff --git a/tool/test/org/antlr/v4/test/TestLexerErrors.java b/tool/test/org/antlr/v4/test/TestLexerErrors.java index 26dba9573..10f0eebcd 100644 --- a/tool/test/org/antlr/v4/test/TestLexerErrors.java +++ b/tool/test/org/antlr/v4/test/TestLexerErrors.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.test; @@ -66,7 +66,7 @@ public class TestLexerErrors extends BaseTest { "A : 'a' 'b' ;\n"; String tokens = execLexer("L.g", grammar, "L", "ax"); String expectingTokens = - "[@0,2:1='',<-1>,1:1]\n"; + "[@0,2:1='',<-1>,1:2]\n"; assertEquals(expectingTokens, tokens); String expectingError = "line 1:0 token recognition error at: 'ax'\n"; String error = stderrDuringParse;