jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

fix bug in modes 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9231]
This commit is contained in:
parrt 2011-10-31 16:27:34 -08:00
parent dd1ad971a9
commit 5cf4cba9c7
11 changed files with 355 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
runtime/Java/src/org/antlr/v4/runtime/Lexer.java
View File

@ -171,7 +171,7 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
}
public void pushMode(int m) {
// System.out.println("pushMode "+m);
if ( LexerATNSimulator.debug ) System.out.println("pushMode "+m);
if ( modeStack==null ) modeStack = new QStack<Integer>();
modeStack.push(mode);
mode(m);
@ -179,7 +179,7 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
public int popMode() {
if ( modeStack==null ) throw new EmptyStackException();
// System.out.println("popMode back to "+modeStack.peek());
if ( LexerATNSimulator.debug ) System.out.println("popMode back to "+modeStack.peek());
mode( modeStack.pop() );
return mode;
}

1
runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java
View File

@ -29,6 +29,7 @@
package org.antlr.v4.runtime;
// TODO: del or rename LexerNoViableAlt?
public class LexerRecognitionExeption extends RuntimeException {
/** Who threw the exception? */
public Lexer lexer;

3
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -30,7 +30,8 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.dfa.*;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.OrderedHashSet;
/** "dup" of ParserInterpreter */

8
tool/playground/L.g
View File

@ -1,3 +1,7 @@
lexer grammar L;
I : '0'..'9'+ {System.out.println("I");} ;
WS : (' '|'\n') {skip();} ;
STRING_START : '"' {pushMode(STRING_MODE); more();} ;
WS : ' '|'
' {skip();} ;
mode STRING_MODE;
STRING : '"' {popMode();} ;
ANY : . {more();} ;

35
tool/playground/TestL.java
View File

@ -1,9 +1,40 @@
import org.antlr.v4.runtime.*;
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
import org.antlr.v4.runtime.ANTLRFileStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonTokenStream;
public class TestL {
public static void main(String[] args) throws Exception {
CharStream input = new ANTLRFileStream(args[0]);
input = new ANTLRStringStream("3 3");
// input = new ANTLRStringStream("3 3");
L lexer = new L(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();

14
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -35,17 +35,23 @@ import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.TerminalAST;
import java.util.List;
import java.util.Set;
public class LexerATNFactory extends ParserATNFactory {
public LexerATNFactory(LexerGrammar g) { super(g); }
public ATN createATN() {
// BUILD ALL START STATES (ONE PER MODE)
for (String modeName : ((LexerGrammar)g).modes.keySet()) {
System.out.println(((LexerGrammar)g).modes);
Set<String> modes = ((LexerGrammar) g).modes.keySet();
for (String modeName : modes) {
// create s0, start state; implied Tokens rule node
TokensStartState startState =
(TokensStartState)newState(TokensStartState.class, null);
@ -66,7 +72,7 @@ public class LexerATNFactory extends ParserATNFactory {
_createATN(g.rules.values());
// LINK MODE START STATE TO EACH TOKEN RULE
for (String modeName : ((LexerGrammar)g).modes.keySet()) {
for (String modeName : modes) {
List<Rule> rules = ((LexerGrammar)g).modes.get(modeName);
TokensStartState startState = atn.modeNameToStartState.get(modeName);
for (Rule r : rules) {

15
tool/src/org/antlr/v4/codegen/CodeGenerator.java
View File

@ -32,12 +32,16 @@ package org.antlr.v4.codegen;
import org.antlr.v4.Tool;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.stringtemplate.v4.*;
import java.io.*;
import java.lang.reflect.*;
import java.util.*;
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
/** General controller for code gen. Can instantiate sub generator(s).
*/
@ -141,6 +145,9 @@ public class CodeGenerator {
//if ( templates.isDefined("headerFile") ) headerFileST.inspect();
}
// String x = ATNSerializer.getDecoded(g, g.atn);
// System.out.println(x);
return st;
}

45
tool/src/org/antlr/v4/misc/MultiMap.java Normal file
View File

@ -0,0 +1,45 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.misc;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
public class MultiMap<K, V> extends LinkedHashMap<K, List<V>> {
public void map(K key, V value) {
List<V> elementsForKey = get(key);
if ( elementsForKey==null ) {
elementsForKey = new ArrayList<V>();
super.put(key, elementsForKey);
}
elementsForKey.add(value);
}
}

2
tool/src/org/antlr/v4/tool/LexerGrammar.java
View File

@ -31,8 +31,8 @@ package org.antlr.v4.tool;
import org.antlr.runtime.RecognitionException;
import org.antlr.v4.Tool;
import org.antlr.v4.misc.MultiMap;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.stringtemplate.v4.misc.MultiMap;
/** */
public class LexerGrammar extends Grammar {

90
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -1,8 +1,39 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.junit.Test;
public class TestATNSerialization extends BaseTest {
@ -638,6 +669,63 @@ public class TestATNSerialization extends BaseTest {
assertEquals(expecting, result);
}
@Test public void testModeInLexer() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"A : 'a'\n ;\n" +
"B : 'b';\n" +
"mode A;\n" +
"C : 'c';\n"+
"D : 'd';\n");
String expecting =
"max type 6\n" +
"0:TOKEN_START -1\n" +
"1:TOKEN_START -1\n" +
"2:RULE_START 0\n" +
"3:RULE_STOP 0\n" +
"4:RULE_START 1\n" +
"5:RULE_STOP 1\n" +
"6:RULE_START 2\n" +
"7:RULE_STOP 2\n" +
"8:RULE_START 3\n" +
"9:RULE_STOP 3\n" +
"10:BASIC 0\n" +
"11:BASIC 0\n" +
"12:BASIC 1\n" +
"13:BASIC 1\n" +
"14:BASIC 2\n" +
"15:BASIC 2\n" +
"16:BASIC 3\n" +
"17:BASIC 3\n" +
"rule 0:2 3,-1\n" +
"rule 1:4 4,-1\n" +
"rule 2:6 5,-1\n" +
"rule 3:8 6,-1\n" +
"mode 0:0\n" +
"mode 1:1\n" +
"0->2 EPSILON 0,0,0\n" +
"0->4 EPSILON 0,0,0\n" +
"1->6 EPSILON 0,0,0\n" +
"1->8 EPSILON 0,0,0\n" +
"2->10 EPSILON 0,0,0\n" +
"4->12 EPSILON 0,0,0\n" +
"6->14 EPSILON 0,0,0\n" +
"8->16 EPSILON 0,0,0\n" +
"10->11 ATOM 97,0,0\n" +
"11->3 EPSILON 0,0,0\n" +
"12->13 ATOM 98,0,0\n" +
"13->5 EPSILON 0,0,0\n" +
"14->15 ATOM 99,0,0\n" +
"15->7 EPSILON 0,0,0\n" +
"16->17 ATOM 100,0,0\n" +
"17->9 EPSILON 0,0,0\n" +
"0:0 1\n" +
"1:1 1\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@Test public void test2ModesInLexer() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+

2
tool/test/org/antlr/v4/test/TestLexerErrors.java
View File

@ -66,7 +66,7 @@ public class TestLexerErrors extends BaseTest {
"A : 'a' 'b' ;\n";
String tokens = execLexer("L.g", grammar, "L", "ax");
String expectingTokens =
"[@0,2:1='<EOF>',<-1>,1:1]\n";
"[@0,2:1='<EOF>',<-1>,1:2]\n";
assertEquals(expectingTokens, tokens);
String expectingError = "line 1:0 token recognition error at: 'ax'\n";
String error = stderrDuringParse;