Merge pull request #13 from sharwell/parrt-master

Several fixes on the latest updates
This commit is contained in:
Terence Parr 2012-02-15 10:57:34 -08:00
commit b44b0a3c09
10 changed files with 348 additions and 156 deletions

View File

@ -30,6 +30,7 @@
package org.antlr.v4.runtime; package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATNConfigSet; import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.misc.Utils;
public class LexerNoViableAltException extends RecognitionException { public class LexerNoViableAltException extends RecognitionException {
/** Matching attempted at what input index? */ /** Matching attempted at what input index? */
@ -47,7 +48,19 @@ public class LexerNoViableAltException extends RecognitionException {
this.deadEndConfigs = deadEndConfigs; this.deadEndConfigs = deadEndConfigs;
} }
@Override
public CharStream getInputStream() {
return (CharStream)super.getInputStream();
}
@Override
public String toString() { public String toString() {
return "NoViableAltException('')"; String symbol = "";
if (startIndex >= 0 && startIndex < input.size()) {
symbol = getInputStream().substring(startIndex, startIndex);
symbol = Utils.escapeWhitespace(symbol, false);
}
return "NoViableAltException('" + symbol + "')";
} }
} }

View File

@ -1,65 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
// TODO: del or rename LexerNoViableAlt?
public class LexerRecognitionExeption extends RuntimeException {
/** Who threw the exception? */
public Lexer lexer;
/** What is index of token/char were we looking at when the error occurred? */
public int index;
/** The current char when an error occurred. For lexers. */
public int c;
/** Track the line at which the error occurred in case this is
* generated from a lexer. We need to track this since the
* unexpected char doesn't carry the line info.
*/
public int line;
public int charPositionInLine;
/** Used for remote debugger deserialization */
public LexerRecognitionExeption() {
}
public LexerRecognitionExeption(Lexer lexer, CharStream input) {
this.lexer = lexer;
this.index = input.index();
this.c = input.LA(1);
if ( lexer!=null ) {
this.line = lexer.getLine();
this.charPositionInLine = lexer.getCharPositionInLine();
}
}
}

View File

@ -63,7 +63,7 @@ import java.util.List;
* satisfy the superclass interface. * satisfy the superclass interface.
*/ */
public class ParserRuleContext<Symbol> extends RuleContext { public class ParserRuleContext<Symbol> extends RuleContext {
public static final ParserRuleContext EMPTY = new ParserRuleContext(); public static final ParserRuleContext<?> EMPTY = new ParserRuleContext<Object>();
/** If we are debugging or building a parse tree for a visitor, /** If we are debugging or building a parse tree for a visitor,
* we need to track all of the tokens and rule invocations associated * we need to track all of the tokens and rule invocations associated
@ -176,61 +176,100 @@ public class ParserRuleContext<Symbol> extends RuleContext {
@Override @Override
public ParseTree getChild(int i) { public ParseTree getChild(int i) {
return children!=null ? children.get(i) : null; return children!=null && i>=0 && i<children.size() ? children.get(i) : null;
} }
public Object getChild(Class ctxType, int i) { public <T extends ParseTree> T getChild(Class<? extends T> ctxType, int i) {
if ( children==null ) throw new UnsupportedOperationException("there are no children"); if ( children==null || i < 0 || i >= children.size() ) {
int j = -1; // what element have we found with ctxType? return null;
for (Object o : children) {
if ( ctxType.isAssignableFrom(o.getClass()) ) {
j = j+1;
if ( j == i ) return o;
}
} }
return null;
}
public Token getToken(int ttype, int i) { int j = -1; // what element have we found with ctxType?
if ( children==null ) throw new UnsupportedOperationException("there are no children"); for (ParseTree o : children) {
int j = -1; // what token with ttype have we found? if ( ctxType.isInstance(o) ) {
for (Object o : children) { j++;
if ( o instanceof TerminalNode ) { if ( j == i ) {
TerminalNode<Token> tnode = (TerminalNode<Token>)o; return ctxType.cast(o);
if ( tnode.getSymbol().getType()==ttype ) {
j++;
if ( j == i ) return tnode.getSymbol();
} }
} }
} }
return null; return null;
} }
public List<Token> getTokens(int ttype) { public Token getToken(int ttype, int i) {
if ( children==null ) throw new UnsupportedOperationException("there are no children"); if ( children==null || i < 0 || i >= children.size() ) {
List<Token> tokens = null; return null;
for (Object o : children) { }
if ( o instanceof Token ) {
if ( tokens==null ) tokens = new ArrayList<Token>(); int j = -1; // what token with ttype have we found?
tokens.add((Token)o); for (ParseTree o : children) {
if ( o instanceof TerminalNode<?> ) {
TerminalNode<?> tnode = (TerminalNode<?>)o;
if ( tnode.getSymbol() instanceof Token ) {
Token symbol = (Token)tnode.getSymbol();
if ( symbol.getType()==ttype ) {
j++;
if ( j == i ) {
return symbol;
}
}
}
} }
} }
return null;
}
public List<? extends Token> getTokens(int ttype) {
if ( children==null ) {
return Collections.emptyList();
}
List<Token> tokens = null;
for (ParseTree o : children) {
if ( o instanceof TerminalNode<?> ) {
TerminalNode<?> tnode = (TerminalNode<?>)o;
if ( tnode.getSymbol() instanceof Token ) {
Token symbol = (Token)tnode.getSymbol();
if ( tokens==null ) {
tokens = new ArrayList<Token>();
}
tokens.add(symbol);
}
}
}
if ( tokens==null ) {
return Collections.emptyList();
}
return tokens; return tokens;
} }
public ParserRuleContext getRuleContext(Class ctxType, int i) { public <T extends ParserRuleContext<?>> T getRuleContext(Class<? extends T> ctxType, int i) {
return (ParserRuleContext)getChild(ctxType, i); return getChild(ctxType, i);
} }
public List<? extends ParserRuleContext> getRuleContexts(Class ctxType) { public <T extends ParserRuleContext<?>> List<? extends T> getRuleContexts(Class<? extends T> ctxType) {
if ( children==null ) throw new UnsupportedOperationException("there are no children"); if ( children==null ) {
List<ParserRuleContext> contexts = null; return Collections.emptyList();
for (Object o : children) { }
if ( o.getClass().isInstance(ctxType) ) {
if ( contexts==null ) contexts = new ArrayList<ParserRuleContext>(); List<T> contexts = null;
contexts.add((ParserRuleContext)o); for (ParseTree o : children) {
if ( ctxType.isInstance(o) ) {
if ( contexts==null ) {
contexts = new ArrayList<T>();
}
contexts.add(ctxType.cast(o));
} }
} }
if ( contexts==null ) {
return Collections.emptyList();
}
return contexts; return contexts;
} }
@ -247,7 +286,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
public String toString(@NotNull Recognizer<?,?> recog, RuleContext stop) { public String toString(@NotNull Recognizer<?,?> recog, RuleContext stop) {
if ( recog==null ) return super.toString(recog, stop); if ( recog==null ) return super.toString(recog, stop);
StringBuilder buf = new StringBuilder(); StringBuilder buf = new StringBuilder();
ParserRuleContext p = this; ParserRuleContext<?> p = this;
buf.append("["); buf.append("[");
while ( p != null && p != stop ) { while ( p != null && p != stop ) {
ATN atn = recog.getATN(); ATN atn = recog.getATN();
@ -258,7 +297,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
// ATNState invoker = atn.states.get(ctx.invokingState); // ATNState invoker = atn.states.get(ctx.invokingState);
// RuleTransition rt = (RuleTransition)invoker.transition(0); // RuleTransition rt = (RuleTransition)invoker.transition(0);
// buf.append(recog.getRuleNames()[rt.target.ruleIndex]); // buf.append(recog.getRuleNames()[rt.target.ruleIndex]);
p = (ParserRuleContext)p.parent; p = (ParserRuleContext<?>)p.parent;
} }
buf.append("]"); buf.append("]");
return buf.toString(); return buf.toString();

View File

@ -81,6 +81,11 @@
<target>1.6</target> <target>1.6</target>
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.12</version>
</plugin>
</plugins> </plugins>
</build> </build>

View File

@ -496,7 +496,7 @@ RuleContextListDecl(rdecl) ::= "public List\<<rdecl.ctxName>> <rdecl.name> = new
ContextTokenGetterDecl(t) ::= ContextTokenGetterDecl(t) ::=
"public Token <t.name>() { return getToken(<parser.name>.<t.name>, 0); }" "public Token <t.name>() { return getToken(<parser.name>.<t.name>, 0); }"
ContextTokenListGetterDecl(t) ::= ContextTokenListGetterDecl(t) ::=
"public List\<Token> <t.name>() { return getTokens(<parser.name>.<t.name>); }" "public List\<? extends Token> <t.name>() { return getTokens(<parser.name>.<t.name>); }"
ContextTokenListIndexedGetterDecl(t) ::= << ContextTokenListIndexedGetterDecl(t) ::= <<
public Token <t.name>(int i) { public Token <t.name>(int i) {
return getToken(<parser.name>.<t.name>, i); return getToken(<parser.name>.<t.name>, i);
@ -508,7 +508,7 @@ public <r.ctxName> <r.name>() {
} }
>> >>
ContextRuleListGetterDecl(r) ::= << ContextRuleListGetterDecl(r) ::= <<
public List\<<r.ctxName>\> <r.name>() { public List\<? extends <r.ctxName>\> <r.name>() {
return (List\<<r.ctxName>\>)getRuleContexts(<r.ctxName>.class); return (List\<<r.ctxName>\>)getRuleContexts(<r.ctxName>.class);
} }
>> >>

View File

@ -71,7 +71,7 @@ public abstract class BaseTest {
public static final String newline = System.getProperty("line.separator"); public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator"); public static final String pathSep = System.getProperty("path.separator");
public static final boolean TEST_IN_SAME_PROCESS = false; public static final boolean TEST_IN_SAME_PROCESS = Boolean.parseBoolean(System.getProperty("antlr.testinprocess"));
/** /**
* Build up the full classpath we need, including the surefire path (if present) * Build up the full classpath we need, including the surefire path (if present)
@ -510,7 +510,7 @@ public abstract class BaseTest {
} }
if ( parserName!=null ) { if ( parserName!=null ) {
files.add(parserName+".java"); files.add(parserName+".java");
files.add("Blank"+grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"Listener.java"); files.add(grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"BaseListener.java");
} }
ok = compile(files.toArray(new String[files.size()])); ok = compile(files.toArray(new String[files.size()]));
if ( !ok ) { allIsWell = false; } if ( !ok ) { allIsWell = false; }

View File

@ -169,7 +169,6 @@
* Character.isJavaIdentifierPart(int) returns true." * Character.isJavaIdentifierPart(int) returns true."
*/ */
grammar Java; grammar Java;
options {backtrack=true; memoize=true;}
@lexer::members { @lexer::members {
protected boolean enumIsKeyword = true; protected boolean enumIsKeyword = true;
@ -184,7 +183,9 @@ compilationUnit
( packageDeclaration importDeclaration* typeDeclaration* ( packageDeclaration importDeclaration* typeDeclaration*
| classOrInterfaceDeclaration typeDeclaration* | classOrInterfaceDeclaration typeDeclaration*
) )
EOF
| packageDeclaration? importDeclaration* typeDeclaration* | packageDeclaration? importDeclaration* typeDeclaration*
EOF
; ;
packageDeclaration packageDeclaration
@ -499,7 +500,7 @@ constructorBody
explicitConstructorInvocation explicitConstructorInvocation
: nonWildcardTypeArguments? ('this' | 'super') arguments ';' : nonWildcardTypeArguments? ('this' | 'super') arguments ';'
| expression '.' nonWildcardTypeArguments? 'super' arguments ';' | primary '.' nonWildcardTypeArguments? 'super' arguments ';'
; ;
@ -711,17 +712,12 @@ constantExpression
; ;
expression expression
: parExpression : primary
| 'this'
| 'super'
| literal
| Identifier
| expression '.' Identifier | expression '.' Identifier
| expression '.' 'class' // should be type.class but causes backtracking
| expression '.' 'this' | expression '.' 'this'
| expression '.' 'super' '(' expressionList? ')' | expression '.' 'super' '(' expressionList? ')'
| expression '.' 'super' '.' Identifier arguments?
| expression '.' 'new' Identifier '(' expressionList? ')' | expression '.' 'new' Identifier '(' expressionList? ')'
| expression '.' 'super' '.' Identifier arguments?
| expression '.' explicitGenericInvocation | expression '.' explicitGenericInvocation
| 'new' creator | 'new' creator
| expression '[' expression ']' | expression '[' expression ']'
@ -754,8 +750,20 @@ expression
| '>' '>' '='<assoc=right> | '>' '>' '='<assoc=right>
| '>' '>' '>' '='<assoc=right> | '>' '>' '>' '='<assoc=right>
| '<' '<' '='<assoc=right> | '<' '<' '='<assoc=right>
| '%='<assoc=right>) expression | '%='<assoc=right>
; )
expression
;
primary
: '(' expression ')'
| 'this'
| 'super'
| literal
| Identifier
| type '.' 'class'
| 'void' '.' 'class'
;
creator creator
: nonWildcardTypeArguments createdName classCreatorRest : nonWildcardTypeArguments createdName classCreatorRest
@ -827,15 +835,16 @@ FloatingPointLiteral
| ('0'..'9')+ Exponent FloatTypeSuffix? | ('0'..'9')+ Exponent FloatTypeSuffix?
| ('0'..'9')+ FloatTypeSuffix | ('0'..'9')+ FloatTypeSuffix
| '0' ('x'|'X') | '0' ('x'|'X')
( HexDigit+ '.' HexDigit* Exponent? FloatTypeSuffix? ( HexDigit+ ('.' HexDigit*)? HexExponent FloatTypeSuffix?
| '.' HexDigit+ Exponent? FloatTypeSuffix? | '.' HexDigit+ HexExponent FloatTypeSuffix?
| HexDigit+ Exponent FloatTypeSuffix?
| HexDigit+ FloatTypeSuffix
) )
; ;
fragment fragment
Exponent : ('e'|'E'|'p'|'P') ('+'|'-')? ('0'..'9')+ ; Exponent : ('e'|'E') ('+'|'-')? ('0'..'9')+ ;
fragment
HexExponent : ('p'|'P') ('+'|'-')? ('0'..'9')+ ;
fragment fragment
FloatTypeSuffix : ('f'|'F'|'d'|'D') ; FloatTypeSuffix : ('f'|'F'|'d'|'D') ;
@ -917,13 +926,13 @@ JavaIDDigit
'\u1040'..'\u1049' '\u1040'..'\u1049'
; ;
WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ -> channel(HIDDEN)
; ;
COMMENT COMMENT
: '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} : '/*' .* '*/' -> channel(HIDDEN)
; ;
LINE_COMMENT LINE_COMMENT
: '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;} : '//' ~('\n'|'\r')* '\r'? '\n' -> channel(HIDDEN)
; ;

View File

@ -1021,7 +1021,7 @@ WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;}
; ;
COMMENT COMMENT
: '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} : '/*' .* '*/' {$channel=HIDDEN;}
; ;
LINE_COMMENT LINE_COMMENT

View File

@ -2,7 +2,7 @@ package org.antlr.v4.test;
import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.LexerRecognitionExeption; import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.Utils; import org.antlr.v4.runtime.misc.Utils;
@ -72,7 +72,7 @@ public class TestATNLexerInterpreter extends BaseTest {
" | 'xy' .\n" + // should not pursue '.' since xy already hit stop " | 'xy' .\n" + // should not pursue '.' since xy already hit stop
" ;\n"); " ;\n");
checkLexerMatches(lg, "xy", "A, EOF"); checkLexerMatches(lg, "xy", "A, EOF");
LexerRecognitionExeption e = checkLexerMatches(lg, "xyz", "A, EOF"); RecognitionException e = checkLexerMatches(lg, "xyz", "A, EOF");
assertEquals("NoViableAltException('z')", e.toString()); assertEquals("NoViableAltException('z')", e.toString());
} }
@ -83,7 +83,7 @@ public class TestATNLexerInterpreter extends BaseTest {
" | 'xy' . 'z'\n" + // will not pursue '.' since xy already hit stop (prior alt) " | 'xy' . 'z'\n" + // will not pursue '.' since xy already hit stop (prior alt)
" ;\n"); " ;\n");
// checkLexerMatches(lg, "xy", "A, EOF"); // checkLexerMatches(lg, "xy", "A, EOF");
LexerRecognitionExeption e = checkLexerMatches(lg, "xyqz", "A, EOF"); RecognitionException e = checkLexerMatches(lg, "xyqz", "A, EOF");
assertEquals("NoViableAltException('q')", e.toString()); assertEquals("NoViableAltException('q')", e.toString());
} }
@ -244,7 +244,7 @@ public class TestATNLexerInterpreter extends BaseTest {
checkLexerMatches(lg, "a", expecting); checkLexerMatches(lg, "a", expecting);
} }
protected LexerRecognitionExeption checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
ATN atn = createATN(lg); ATN atn = createATN(lg);
CharStream input = new ANTLRInputStream(inputString); CharStream input = new ANTLRInputStream(inputString);
ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE"); ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE");
@ -252,11 +252,11 @@ public class TestATNLexerInterpreter extends BaseTest {
System.out.println(dot.getDOT(startState, true)); System.out.println(dot.getDOT(startState, true));
List<String> tokenTypes = null; List<String> tokenTypes = null;
LexerRecognitionExeption retException = null; RecognitionException retException = null;
try { try {
tokenTypes = getTokenTypes(lg, atn, input, false); tokenTypes = getTokenTypes(lg, atn, input, false);
} }
catch (LexerRecognitionExeption lre) { retException = lre; } catch (RecognitionException lre) { retException = lre; }
if ( retException!=null ) return retException; if ( retException!=null ) return retException;
String result = Utils.join(tokenTypes.iterator(), ", "); String result = Utils.join(tokenTypes.iterator(), ", ");

View File

@ -1,3 +1,31 @@
/*
* [The "BSD license"]
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test; package org.antlr.v4.test;
import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.*;
@ -14,11 +42,19 @@ import java.lang.reflect.Method;
import java.net.URL; import java.net.URL;
import java.net.URLClassLoader; import java.net.URLClassLoader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
public class TestPerformance extends BaseTest { public class TestPerformance extends BaseTest {
/** Parse all java files under this package within the JDK_SOURCE_ROOT. */ /** Parse all java files under this package within the JDK_SOURCE_ROOT. */
private static final String TOP_PACKAGE = "java"; private static final String TOP_PACKAGE = "java.lang";
/** True to load java files from sub-packages of {@link #TOP_PACKAGE}. */ /** True to load java files from sub-packages of {@link #TOP_PACKAGE}. */
private static final boolean RECURSIVE = true; private static final boolean RECURSIVE = true;
@ -27,26 +63,41 @@ public class TestPerformance extends BaseTest {
* the standard grammar (Java.g). In either case, the grammar is renamed in the temporary directory to Java.g * the standard grammar (Java.g). In either case, the grammar is renamed in the temporary directory to Java.g
* before compiling. * before compiling.
*/ */
private static final boolean USE_LR_GRAMMAR = false; private static final boolean USE_LR_GRAMMAR = true;
/** /**
* True to specify the -Xforceatn option when generating the grammar, forcing all decisions in JavaParser to * True to specify the -Xforce-atn option when generating the grammar, forcing all decisions in JavaParser to
* be handled by {@link v2ParserATNSimulator#adaptivePredict}. * be handled by {@link ParserATNSimulator#adaptivePredict}.
*/ */
private static final boolean FORCE_ATN = false; private static final boolean FORCE_ATN = false;
/**
* True to specify the -atn option when generating the grammar. This will cause ANTLR
* to export the ATN for each decision as a DOT (GraphViz) file.
*/
private static final boolean EXPORT_ATN_GRAPHS = true;
/**
* True to delete temporary (generated and compiled) files when the test completes.
*/
private static final boolean DELETE_TEMP_FILES = true;
private static final boolean PAUSE_FOR_HEAP_DUMP = false;
/** Parse each file with JavaParser.compilationUnit */ /** Parse each file with JavaParser.compilationUnit */
private static final boolean RUN_PARSER = true; private static final boolean RUN_PARSER = true;
/** True to use {@link BailErrorStrategy}, False to use {@link DefaultErrorStrategy} */ /** True to use {@link BailErrorStrategy}, False to use {@link DefaultErrorStrategy} */
private static final boolean BAIL_ON_ERROR = false; private static final boolean BAIL_ON_ERROR = true;
/** This value is passed to {@link org.antlr.v4.runtime.Parser#setBuildParseTree}. */ /** This value is passed to {@link Parser#setBuildParseTree}. */
private static final boolean BUILD_PARSE_TREES = false; private static final boolean BUILD_PARSE_TREES = false;
/** /**
* Use ParseTreeWalker.DEFAULT.walk with the BlankJavaParserListener to show parse tree walking overhead. * Use ParseTreeWalker.DEFAULT.walk with the BlankJavaParserListener to show parse tree walking overhead.
* If {@link #BUILD_PARSE_TREES} is false, the listener will instead be called during the parsing process via * If {@link #BUILD_PARSE_TREES} is false, the listener will instead be called during the parsing process via
* {@link org.antlr.v4.runtime.Parser#addParseListener}. * {@link Parser#addParseListener}.
*/ */
private static final boolean BLANK_LISTENER = false; private static final boolean BLANK_LISTENER = false;
private static final boolean SHOW_DFA_STATE_STATS = true;
private static final boolean SHOW_CONFIG_STATS = false;
/** /**
* If true, a single JavaLexer will be used, and {@link Lexer#setInputStream} will be called to initialize it * If true, a single JavaLexer will be used, and {@link Lexer#setInputStream} will be called to initialize it
* for each source file. In this mode, the cached DFA will be persisted throughout the lexing process. * for each source file. In this mode, the cached DFA will be persisted throughout the lexing process.
@ -67,25 +118,30 @@ public class TestPerformance extends BaseTest {
/** Total number of passes to make over the source */ /** Total number of passes to make over the source */
private static final int PASSES = 4; private static final int PASSES = 4;
private Lexer sharedLexer; private static Lexer sharedLexer;
private Parser sharedParser; private static Parser sharedParser;
@SuppressWarnings({"FieldCanBeLocal"}) @SuppressWarnings({"FieldCanBeLocal"})
private ParseTreeListener<Token> sharedListener; private static ParseTreeListener<Token> sharedListener;
private int tokenCount; private int tokenCount;
private int currentPass;
@Test @Test
// @Ignore //@Ignore
public void compileJdk() throws IOException { public void compileJdk() throws IOException {
compileParser(USE_LR_GRAMMAR);
JavaParserFactory factory = getParserFactory();
String jdkSourceRoot = System.getenv("JDK_SOURCE_ROOT"); String jdkSourceRoot = System.getenv("JDK_SOURCE_ROOT");
if (jdkSourceRoot == null) {
jdkSourceRoot = System.getProperty("JDK_SOURCE_ROOT");
}
if (jdkSourceRoot == null) { if (jdkSourceRoot == null) {
System.err.println("The JDK_SOURCE_ROOT environment variable must be set for performance testing."); System.err.println("The JDK_SOURCE_ROOT environment variable must be set for performance testing.");
return; return;
} }
if (!TOP_PACKAGE.isEmpty()) { compileParser(USE_LR_GRAMMAR);
JavaParserFactory factory = getParserFactory();
if (!TOP_PACKAGE.isEmpty()) {
jdkSourceRoot = jdkSourceRoot + '/' + TOP_PACKAGE.replace('.', '/'); jdkSourceRoot = jdkSourceRoot + '/' + TOP_PACKAGE.replace('.', '/');
} }
@ -93,10 +149,13 @@ public class TestPerformance extends BaseTest {
assertTrue(directory.isDirectory()); assertTrue(directory.isDirectory());
Collection<CharStream> sources = loadSources(directory, RECURSIVE); Collection<CharStream> sources = loadSources(directory, RECURSIVE);
System.out.format("Lex=true, Parse=%s, ForceAtn=%s, Bail=%s, BuildParseTree=%s, BlankListener=%s\n",
RUN_PARSER, FORCE_ATN, BAIL_ON_ERROR, BUILD_PARSE_TREES, BLANK_LISTENER); System.out.print(getOptionsDescription());
currentPass = 0;
parse1(factory, sources); parse1(factory, sources);
for (int i = 0; i < PASSES - 1; i++) { for (int i = 0; i < PASSES - 1; i++) {
currentPass = i + 1;
if (CLEAR_DFA) { if (CLEAR_DFA) {
sharedLexer = null; sharedLexer = null;
sharedParser = null; sharedParser = null;
@ -104,6 +163,55 @@ public class TestPerformance extends BaseTest {
parse2(factory, sources); parse2(factory, sources);
} }
sources.clear();
if (PAUSE_FOR_HEAP_DUMP) {
System.gc();
System.out.println("Pausing before application exit.");
try {
Thread.sleep(4000);
} catch (InterruptedException ex) {
Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
@Override
protected void eraseTempDir() {
if (DELETE_TEMP_FILES) {
super.eraseTempDir();
}
}
public static String getOptionsDescription() {
StringBuilder builder = new StringBuilder();
builder.append("Input=");
if (TestPerformance.TOP_PACKAGE.isEmpty()) {
builder.append("*");
}
else {
builder.append(TOP_PACKAGE).append(".*");
}
builder.append(", Grammar=").append(USE_LR_GRAMMAR ? "LR" : "Standard");
builder.append(", ForceAtn=").append(FORCE_ATN);
builder.append('\n');
builder.append("Op=Lex").append(RUN_PARSER ? "+Parse" : " only");
builder.append(", Strategy=").append(BAIL_ON_ERROR ? BailErrorStrategy.class.getSimpleName() : DefaultErrorStrategy.class.getSimpleName());
builder.append(", BuildParseTree=").append(BUILD_PARSE_TREES);
builder.append(", WalkBlankListener=").append(BLANK_LISTENER);
builder.append('\n');
builder.append("Lexer=").append(REUSE_LEXER ? "setInputStream" : "newInstance");
builder.append(", Parser=").append(REUSE_PARSER ? "setInputStream" : "newInstance");
builder.append(", AfterPass=").append(CLEAR_DFA ? "newInstance" : "setInputStream");
builder.append('\n');
return builder.toString();
} }
/** /**
@ -158,6 +266,8 @@ public class TestPerformance extends BaseTest {
} }
} }
int configOutputSize = 0;
protected void parseSources(JavaParserFactory factory, Collection<CharStream> sources) { protected void parseSources(JavaParserFactory factory, Collection<CharStream> sources) {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
tokenCount = 0; tokenCount = 0;
@ -167,7 +277,7 @@ public class TestPerformance extends BaseTest {
input.seek(0); input.seek(0);
inputSize += input.size(); inputSize += input.size();
// this incurred a great deal of overhead and was causing significant variations in performance results. // this incurred a great deal of overhead and was causing significant variations in performance results.
//System.out.format("Parsing file %s\n", file.getAbsolutePath()); //System.out.format("Parsing file %s\n", input.getSourceName());
try { try {
factory.parseFile(input); factory.parseFile(input);
} catch (IllegalStateException ex) { } catch (IllegalStateException ex) {
@ -180,6 +290,80 @@ public class TestPerformance extends BaseTest {
inputSize / 1024, inputSize / 1024,
tokenCount, tokenCount,
System.currentTimeMillis() - startTime); System.currentTimeMillis() - startTime);
if (RUN_PARSER) {
// make sure the individual DFAState objects actually have unique ATNConfig arrays
final ParserATNSimulator<?> interpreter = sharedParser.getInterpreter();
final DFA[] decisionToDFA = interpreter.decisionToDFA;
if (SHOW_DFA_STATE_STATS) {
int states = 0;
for (int i = 0; i < decisionToDFA.length; i++) {
DFA dfa = decisionToDFA[i];
if (dfa == null || dfa.states == null) {
continue;
}
states += dfa.states.size();
}
System.out.format("There are %d DFAState instances.\n", states);
}
int localDfaCount = 0;
int globalDfaCount = 0;
int localConfigCount = 0;
int globalConfigCount = 0;
int[] contextsInDFAState = new int[0];
for (int i = 0; i < decisionToDFA.length; i++) {
DFA dfa = decisionToDFA[i];
if (dfa == null || dfa.states == null) {
continue;
}
if (SHOW_CONFIG_STATS) {
for (DFAState state : dfa.states.keySet()) {
if (state.configset.size() >= contextsInDFAState.length) {
contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configset.size() + 1);
}
if (state.isAcceptState) {
boolean hasGlobal = false;
for (ATNConfig config : state.configset) {
if (config.reachesIntoOuterContext > 0) {
globalConfigCount++;
hasGlobal = true;
} else {
localConfigCount++;
}
}
if (hasGlobal) {
globalDfaCount++;
} else {
localDfaCount++;
}
}
contextsInDFAState[state.configset.size()]++;
}
}
}
if (SHOW_CONFIG_STATS && currentPass == 0) {
System.out.format(" DFA accept states: %d total, %d with only local context, %d with a global context\n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
System.out.format(" Config stats: %d total, %d local, %d global\n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
if (SHOW_DFA_STATE_STATS) {
for (int i = 0; i < contextsInDFAState.length; i++) {
if (contextsInDFAState[i] != 0) {
System.out.format(" %d configs = %d\n", i, contextsInDFAState[i]);
}
}
}
}
}
} }
protected void compileParser(boolean leftRecursive) throws IOException { protected void compileParser(boolean leftRecursive) throws IOException {
@ -187,8 +371,15 @@ public class TestPerformance extends BaseTest {
String sourceName = leftRecursive ? "Java-LR.g" : "Java.g"; String sourceName = leftRecursive ? "Java-LR.g" : "Java.g";
String body = load(sourceName, null); String body = load(sourceName, null);
@SuppressWarnings({"ConstantConditions"}) @SuppressWarnings({"ConstantConditions"})
String[] extraOptions = FORCE_ATN ? new String[] {"-Xforceatn"} : new String[0]; List<String> extraOptions = new ArrayList<String>();
boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptions); if (FORCE_ATN) {
extraOptions.add("-Xforce-atn");
}
if (EXPORT_ATN_GRAPHS) {
extraOptions.add("-atn");
}
String[] extraOptionsArray = extraOptions.toArray(new String[extraOptions.size()]);
boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptionsArray);
assertTrue(success); assertTrue(success);
} }
@ -228,7 +419,7 @@ public class TestPerformance extends BaseTest {
final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("JavaParser"); final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("JavaParser");
@SuppressWarnings({"unchecked"}) @SuppressWarnings({"unchecked"})
final Class<? extends ParseTreeListener<Token>> listenerClass = (Class<? extends ParseTreeListener<Token>>)loader.loadClass("BlankJavaListener"); final Class<? extends ParseTreeListener<Token>> listenerClass = (Class<? extends ParseTreeListener<Token>>)loader.loadClass("BlankJavaListener");
this.sharedListener = listenerClass.newInstance(); TestPerformance.sharedListener = listenerClass.newInstance();
final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class); final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class); final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
@ -271,7 +462,7 @@ public class TestPerformance extends BaseTest {
Method parseMethod = parserClass.getMethod("compilationUnit"); Method parseMethod = parserClass.getMethod("compilationUnit");
Object parseResult = parseMethod.invoke(sharedParser); Object parseResult = parseMethod.invoke(sharedParser);
assert parseResult instanceof ParseTree; Assert.assertTrue(parseResult instanceof ParseTree);
if (BUILD_PARSE_TREES && BLANK_LISTENER) { if (BUILD_PARSE_TREES && BLANK_LISTENER) {
ParseTreeWalker.DEFAULT.walk(sharedListener, (ParseTree)parseResult); ParseTreeWalker.DEFAULT.walk(sharedListener, (ParseTree)parseResult);