jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

java.g working again. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8910]
This commit is contained in:
parrt 2011-07-26 10:56:24 -08:00
parent fd3f8b17c9
commit 078e2ab47c
12 changed files with 214 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runtime/Java/src/org/antlr/v4/runtime/Lexer.java
View File

@ -128,7 +128,7 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
eof.setCharPositionInLine(getCharPositionInLine());
return eof;
}
// System.out.println("nextToken at "+((char)input.LA(1))+
// System.out.println("nextToken line "+tokenStartLine+" at "+((char)input.LA(1))+
// " in mode "+mode+
// " at index "+input.index());
int ttype = _interp.match(input, mode);

13
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -35,8 +35,8 @@ import org.antlr.v4.runtime.misc.OrderedHashSet;
/** "dup" of ParserInterpreter */
public class LexerATNSimulator extends ATNSimulator {
public static boolean debug = true;
public static boolean dfa_debug = true;
public static boolean debug = false;
public static boolean dfa_debug = false;
public static final int NUM_EDGES = 255;
protected Lexer recog;
@ -266,7 +266,7 @@ public class LexerATNSimulator extends ATNSimulator {
// if ( st.set.toString().equals("0") ) {
// System.out.println("eh?");
// }
if ( debug ) System.out.println("match set "+st.set.toString());
if ( debug ) System.out.println("match set "+st.set.toString(true));
return st.target;
}
}
@ -311,7 +311,12 @@ public class LexerATNSimulator extends ATNSimulator {
// TODO? if ( closure.contains(t) ) return;
if ( config.state instanceof RuleStopState ) {
if ( debug ) System.out.println("closure at rule stop "+config);
if ( debug ) {
if ( recog!=null ) System.out.println("closure at "+
recog.getRuleNames()[config.state.ruleIndex]+
" rule stop "+config);
else System.out.println("closure at rule stop "+config);
}
if ( config.context == null || config.context.isEmpty() ) {
configs.add(config);
return;

6
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -37,7 +37,7 @@ import org.stringtemplate.v4.misc.MultiMap;
import java.util.*;
public class ParserATNSimulator extends ATNSimulator {
public static boolean debug = true;
public static boolean debug = false;
public static boolean dfa_debug = false;
public static int ATN_failover = 0;
@ -462,10 +462,10 @@ public class ParserATNSimulator extends ATNSimulator {
ATNState p = config.state;
RuleContext newContext;
if ( parser != null ) {
System.out.println("rule trans to rule "+parser.getRuleNames()[t.target.ruleIndex]);
// System.out.println("rule trans to rule "+parser.getRuleNames()[t.target.ruleIndex]);
newContext = parser.newContext(config.context, t.target.stateNumber, t.target.ruleIndex, -999);
newContext.invokingState = p.stateNumber;
System.out.println("new ctx type is "+newContext.getClass().getSimpleName());
// System.out.println("new ctx type is "+newContext.getClass().getSimpleName());
}
else {
newContext = new RuleContext(config.context, p.stateNumber, t.target.stateNumber);

14
tool/playground/L.g
View File

@ -1,8 +1,8 @@
lexer grammar L;
STRING_START : '"' {pushMode(STRING_MODE); more();} ;
WS : ' '|'\n' {skip();} ;
mode STRING_MODE;
STRING : '"' {popMode();} ;
ANY : . {more();} ;
HexLiteral : '0' ('x'|'X') HexDigit+ ;
DecimalLiteral : ('0' | '1'..'9' '0'..'9'*) ;
FloatingPointLiteral : ('0x' | '0X') HexDigit* ('.' HexDigit*)? ;
DOT : '.' ;
ID : 'a'..'z'+ ;
fragment HexDigit : ('0'..'9'|'a'..'f'|'A'..'F') ;
WS : (' '|'\n')+ ;

29
tool/playground/testjava.sh Normal file
View File

@ -0,0 +1,29 @@
java -Xmx500M -cp "/usr/local/lib/antlr4-complete.jar:$CLASSPATH" org.antlr.v4.Tool JavaLexer.g
java -Xmx500M -cp "/usr/local/lib/antlr4-complete.jar:$CLASSPATH" org.antlr.v4.Tool JavaParser.g
javac -cp .:/usr/local/lib/antlr4-complete.jar *.java
time java -Xmx400M -cp .:/usr/local/lib/antlr4-complete.jar TestJava .
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./input
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./JavaLexer.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./JavaParser.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./L.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./MLexer.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./MParser.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./T.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./TestJava.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./TestL.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./TestT.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./TestYang.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./TLexer.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./TParser.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./U.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./UParser.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./YangJavaLexer.java
# parsing /Users/parrt/antlr/code/antlr4/main/tool/playground/./YangJavaParser.java
# Lexer total time 182ms.
# Total time 603ms.
# finished parsing OK
# 726 lexer failovers
# 104048 lexer match calls
# 220 parser failovers
# 107785 parser predict calls

6
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -78,7 +78,7 @@ case <f.ruleIndex> : return <f.name>_argEval((<f.ctxType>)_localctx, s, actionIn
public void action(RuleContext _localctx, int ruleIndex, int actionIndex) {
switch ( ruleIndex ) {
<recog.actionFuncs.values:{f|
case <f.ruleIndex> : <f.name>_action((<f.ctxType>)_localctx, actionIndex);}; separator="\n">
case <f.ruleIndex> : <f.name>_action((<f.ctxType>)_localctx, actionIndex); break;}; separator="\n">
}
}
<actionFuncs.values; separator="\n">
@ -141,7 +141,7 @@ public QStack\<<currentRule.ctxType>\> <currentRule.name>_stk = new QStack\<<cur
_ctx = _localctx;
<currentRule.name>_stk.push(_localctx);
_localctx.start = input.LT(1);
System.out.println("enter "+ruleNames[<currentRule.index>]);
//System.out.println("enter "+ruleNames[<currentRule.index>]);
<namedActions.init>
<locals; separator="\n">
try {
@ -158,7 +158,7 @@ public QStack\<<currentRule.ctxType>\> <currentRule.name>_stk = new QStack\<<cur
<currentRule.name>_stk.pop();
_ctx = (ParserRuleContext)_ctx.parent;
<finallyAction>
System.out.println("exit "+ruleNames[<currentRule.index>]);
//System.out.println("exit "+ruleNames[<currentRule.index>]);
}
return _localctx;
}

2
tool/src/org/antlr/v4/misc/CharSupport.java
View File

@ -92,6 +92,7 @@ public class CharSupport {
/** Given a literal like (the 3 char sequence with single quotes) 'a',
* return the int value of 'a'. Convert escape sequences here also.
* Return -1 if not single char.
*/
public static int getCharValueFromGrammarCharLiteral(String literal) {
switch ( literal.length() ) {
@ -99,6 +100,7 @@ public class CharSupport {
// 'x'
return literal.charAt(1); // no escape char
case 4 :
if ( literal.charAt(1)!='\\' ) return -1;
// '\x' (antlr lexer will catch invalid char)
if ( Character.isDigit(literal.charAt(2)) ) {
// ErrorManager.error(ErrorManager.MSG_SYNTAX_ERROR,

7
tool/src/org/antlr/v4/parse/BlockSetTransformer.g
View File

@ -10,6 +10,7 @@ options {
@header {
package org.antlr.v4.parse;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.misc.*;
import org.antlr.v4.tool.*;
import java.util.List;
import java.util.Set;
@ -71,8 +72,10 @@ boolean inLexer = Character.isUpperCase(currentRuleName.charAt(0));
setElement[boolean inLexer]
: {!rewriteElems.contains($start.getText())}?
( STRING_LITERAL
( a=STRING_LITERAL {!inLexer || CharSupport.getCharValueFromGrammarCharLiteral($a.getText())!=-1}?
| {!inLexer}?=> TOKEN_REF
| {inLexer}?=> ^(RANGE STRING_LITERAL STRING_LITERAL)
| {inLexer}?=> ^(RANGE a=STRING_LITERAL b=STRING_LITERAL)
{CharSupport.getCharValueFromGrammarCharLiteral($a.getText())!=-1 &&
CharSupport.getCharValueFromGrammarCharLiteral($b.getText())!=-1}?
)
;

92
tool/test/org/antlr/v4/test/TestATNConstruction.java
View File

@ -71,16 +71,36 @@ public class TestATNConstruction extends BaseTest {
checkRule(g, "a", expecting);
}
@Test public void testLexerIsntSetMultiCharString() throws Exception {
LexerGrammar g = new LexerGrammar(
"lexer grammar P;\n"+
"A : ('0x' | '0X') ;");
String expecting =
"BlockStart_0->RuleStart_A_1\n" +
"RuleStart_A_1->BlockStart_9\n" +
"BlockStart_9->s3\n" +
"BlockStart_9->s6\n" +
"s3-'0'->s4\n" +
"s6-'0'->s7\n" +
"s4-'x'->s5\n" +
"s7-'X'->s8\n" +
"s5->BlockEnd_10\n" +
"s8->BlockEnd_10\n" +
"BlockEnd_10->RuleStop_A_2\n";
checkTokensRule(g, null, expecting);
}
@Test public void testRange() throws Exception {
LexerGrammar g = new LexerGrammar(
"lexer grammar P;\n"+
"A : 'a'..'c' ;"
);
String expecting =
"BlockStart_0->RuleStart_A_1\n" +
"RuleStart_A_1->s3\n" +
"s3-'a'..'c'->s4\n" +
"s4->RuleStop_A_2\n";
checkTokensRule(g, "A", expecting);
checkTokensRule(g, null, expecting);
}
@Test public void testRangeOrRange() throws Exception {
@ -89,6 +109,7 @@ public class TestATNConstruction extends BaseTest {
"A : ('a'..'c' 'h' | 'q' 'j'..'l') ;"
);
String expecting =
"BlockStart_0->RuleStart_A_1\n" +
"RuleStart_A_1->BlockStart_11\n" +
"BlockStart_11->s3\n" +
"BlockStart_11->s7\n" +
@ -101,7 +122,7 @@ public class TestATNConstruction extends BaseTest {
"s6->BlockEnd_12\n" +
"s10->BlockEnd_12\n" +
"BlockEnd_12->RuleStop_A_2\n";
checkTokensRule(g, "A", expecting);
checkTokensRule(g, null, expecting);
}
@Test public void testStringLiteralInParser() throws Exception {
@ -236,13 +257,13 @@ public class TestATNConstruction extends BaseTest {
"parser grammar P;\n"+
"a : (A|B)?;");
String expecting =
"RuleStart_a_0->BlockStart_8\n" +
"BlockStart_8->s6\n" +
"BlockStart_8->BlockEnd_9\n" +
"s6-{A..B}->s7\n" +
"BlockEnd_9->RuleStop_a_1\n" +
"s7->BlockEnd_9\n" +
"RuleStop_a_1-EOF->s10\n";
"RuleStart_a_0->BlockStart_4\n" +
"BlockStart_4->s2\n" +
"BlockStart_4->BlockEnd_5\n" +
"s2-{A..B}->s3\n" +
"BlockEnd_5->RuleStop_a_1\n" +
"s3->BlockEnd_5\n" +
"RuleStop_a_1-EOF->s6\n";
checkRule(g, "a", expecting);
}
@ -251,12 +272,12 @@ public class TestATNConstruction extends BaseTest {
"parser grammar P;\n"+
"a : (A | B) C;");
String expecting =
"RuleStart_a_0->s6\n" +
"s6-{A..B}->s7\n" +
"s7->s8\n" +
"s8-C->s9\n" +
"s9->RuleStop_a_1\n" +
"RuleStop_a_1-EOF->s10\n";
"RuleStart_a_0->s2\n" +
"s2-{A..B}->s3\n" +
"s3->s4\n" +
"s4-C->s5\n" +
"s5->RuleStop_a_1\n" +
"RuleStop_a_1-EOF->s6\n";
checkRule(g, "a", expecting);
}
@ -282,15 +303,15 @@ public class TestATNConstruction extends BaseTest {
"parser grammar P;\n"+
"a : (A|B)+;");
String expecting =
"RuleStart_a_0->PlusBlockStart_8\n" +
"PlusBlockStart_8->s6\n" +
"s6-{A..B}->s7\n" +
"s7->BlockEnd_9\n" +
"BlockEnd_9->PlusLoopBack_10\n" +
"PlusLoopBack_10->s6\n" +
"PlusLoopBack_10->s11\n" +
"s11->RuleStop_a_1\n" +
"RuleStop_a_1-EOF->s12\n";
"RuleStart_a_0->PlusBlockStart_4\n" +
"PlusBlockStart_4->s2\n" +
"s2-{A..B}->s3\n" +
"s3->BlockEnd_5\n" +
"BlockEnd_5->PlusLoopBack_6\n" +
"PlusLoopBack_6->s2\n" +
"PlusLoopBack_6->s7\n" +
"s7->RuleStop_a_1\n" +
"RuleStop_a_1-EOF->s8\n";
checkRule(g, "a", expecting);
}
@ -924,18 +945,19 @@ public class TestATNConstruction extends BaseTest {
}
void checkTokensRule(LexerGrammar g, String modeName, String expecting) {
if ( g.ast!=null && !g.ast.hasErrors ) {
System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp);
}
}
}
// if ( g.ast!=null && !g.ast.hasErrors ) {
// System.out.println(g.ast.toStringTree());
// Tool antlr = new Tool();
// SemanticPipeline sem = new SemanticPipeline(g);
// sem.process();
// if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
// for (Grammar imp : g.getImportedGrammars()) {
// antlr.processNonCombinedGrammar(imp);
// }
// }
// }
if ( modeName==null ) modeName = "DEFAULT_MODE";
if ( g.modes.get(modeName)==null ) {
System.err.println("no such mode "+modeName);
return;

65
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -439,6 +439,71 @@ public class TestATNSerialization extends BaseTest {
assertEquals(expecting, result);
}
@Test public void testLexerWildcardWithMode() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : 'a'..'z'+ ;\n"+
"mode CMT;" +
"COMMENT : '*/' {skip(); popMode();} ;\n" +
"JUNK : . {more();} ;\n");
String expecting =
"max type 5\n" +
"0:TOKEN_START -1\n" +
"1:TOKEN_START -1\n" +
"2:RULE_START 0\n" +
"3:RULE_STOP 0\n" +
"4:RULE_START 1\n" +
"5:RULE_STOP 1\n" +
"6:RULE_START 2\n" +
"7:RULE_STOP 2\n" +
"8:BASIC 0\n" +
"9:BASIC 0\n" +
"10:PLUS_BLOCK_START 0\n" +
"11:BLOCK_END 0\n" +
"12:PLUS_LOOP_BACK 0\n" +
"13:BASIC 0\n" +
"14:BASIC 1\n" +
"15:BASIC 1\n" +
"16:BASIC 1\n" +
"17:BASIC 1\n" +
"18:BASIC 2\n" +
"19:BASIC 2\n" +
"20:BASIC 2\n" +
"rule 0:2 3,-1\n" +
"rule 1:4 4,0\n" +
"rule 2:6 5,1\n" +
"mode 0:0\n" +
"mode 1:1\n" +
"0->2 EPSILON 0,0\n" +
"1->4 EPSILON 0,0\n" +
"1->6 EPSILON 0,0\n" +
"2->10 EPSILON 0,0\n" +
"4->14 EPSILON 0,0\n" +
"6->18 EPSILON 0,0\n" +
"8->9 RANGE 97,122\n" +
"9->11 EPSILON 0,0\n" +
"10->8 EPSILON 0,0\n" +
"11->12 EPSILON 0,0\n" +
"12->8 EPSILON 0,0\n" +
"12->13 EPSILON 0,0\n" +
"13->3 EPSILON 0,0\n" +
"14->15 ATOM 42,0\n" +
"15->16 ATOM 47,0\n" +
"16->17 EPSILON 0,0\n" +
"17->5 EPSILON 0,0\n" +
"18->19 WILDCARD 0,0\n" +
"19->20 EPSILON 0,0\n" +
"20->7 EPSILON 0,0\n" +
"0:0\n" +
"1:1\n" +
"2:10\n" +
"3:10\n" +
"4:12\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@Test public void testLexerNotSetWithRange2() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+

6
tool/test/org/antlr/v4/test/TestActionTranslation.java
View File

@ -123,7 +123,7 @@ public class TestActionTranslation extends BaseTest {
@Test public void testReturnValues() throws Exception {
String action = "$lab.e; $b.e;";
String expected = "lab.e; _rb.e;";
String expected = "_localctx.lab.e; _localctx._rb.e;";
testActions(attributeTemplate, "inline", action, expected);
}
@ -238,8 +238,8 @@ public class TestActionTranslation extends BaseTest {
@Test public void testRefToTextAttributeForCurrentRule() throws Exception {
String action = "$a.text; $text";
String expected =
"((TokenStream)input).toString(_localctx.start, _localctx.stop); " +
"((TokenStream)input).toString(_localctx.start, _localctx.stop)";
"((TokenStream)input).toString(_localctx.start, input.LT(-1)); " +
"((TokenStream)input).toString(_localctx.start, input.LT(-1))";
testActions(attributeTemplate, "init", action, expected);
testActions(attributeTemplate, "inline", action, expected);
testActions(attributeTemplate, "finally", action, expected);

30
tool/test/org/antlr/v4/test/TestLexerExec.java
View File

@ -67,4 +67,34 @@ public class TestLexerExec extends BaseTest {
"[@7,17:17='<EOF>',<-1>,1:17]\n";
assertEquals(expecting, found);
}
@Test public void testHexVsID() throws Exception {
String grammar =
"lexer grammar L;\n"+
"HexLiteral : '0' ('x'|'X') HexDigit+ ;\n"+
"DecimalLiteral : ('0' | '1'..'9' '0'..'9'*) ;\n" +
"FloatingPointLiteral : ('0x' | '0X') HexDigit* ('.' HexDigit*)? ;\n" +
"DOT : '.' ;\n" +
"ID : 'a'..'z'+ ;\n" +
"fragment HexDigit : ('0'..'9'|'a'..'f'|'A'..'F') ;\n" +
"WS : (' '|'\n')+ ;";
String found = execLexer("L.g", grammar, "L", "x 0 1 a.b a.l");
String expecting =
"[@0,0:0='x',<7>,1:0]\n" +
"[@1,1:1=' ',<8>,1:1]\n" +
"[@2,2:2='0',<4>,1:2]\n" +
"[@3,3:3=' ',<8>,1:3]\n" +
"[@4,4:4='1',<4>,1:4]\n" +
"[@5,5:5=' ',<8>,1:5]\n" +
"[@6,6:6='a',<7>,1:6]\n" +
"[@7,7:7='.',<6>,1:7]\n" +
"[@8,8:8='b',<7>,1:8]\n" +
"[@9,9:9=' ',<8>,1:9]\n" +
"[@10,10:10='a',<7>,1:10]\n" +
"[@11,11:11='.',<6>,1:11]\n" +
"[@12,12:12='l',<7>,1:12]\n" +
"[@13,13:13='<EOF>',<-1>,1:13]\n";
assertEquals(expecting, found);
}
}