forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #13 from sharwell/parrt-master 

Several fixes on the latest updates
This commit is contained in:
Terence Parr 2012-02-15 10:57:34 -08:00
parent 3d56d40975 685cf7b7d5
commit b44b0a3c09
10 changed files with 348 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java
View File

@ -30,6 +30,7 @@
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.misc.Utils;
public class LexerNoViableAltException extends RecognitionException {
/** Matching attempted at what input index? */
@ -47,7 +48,19 @@ public class LexerNoViableAltException extends RecognitionException {
this.deadEndConfigs = deadEndConfigs;
}
@Override
public CharStream getInputStream() {
return (CharStream)super.getInputStream();
}
@Override
public String toString() {
return "NoViableAltException('')";
String symbol = "";
if (startIndex >= 0 && startIndex < input.size()) {
symbol = getInputStream().substring(startIndex, startIndex);
symbol = Utils.escapeWhitespace(symbol, false);
}
return "NoViableAltException('" + symbol + "')";
}
}

65
runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java
View File

@ -1,65 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
// TODO: del or rename LexerNoViableAlt?
public class LexerRecognitionExeption extends RuntimeException {
/** Who threw the exception? */
public Lexer lexer;
/** What is index of token/char were we looking at when the error occurred? */
public int index;
/** The current char when an error occurred. For lexers. */
public int c;
/** Track the line at which the error occurred in case this is
* generated from a lexer. We need to track this since the
* unexpected char doesn't carry the line info.
*/
public int line;
public int charPositionInLine;
/** Used for remote debugger deserialization */
public LexerRecognitionExeption() {
}
public LexerRecognitionExeption(Lexer lexer, CharStream input) {
this.lexer = lexer;
this.index = input.index();
this.c = input.LA(1);
if ( lexer!=null ) {
this.line = lexer.getLine();
this.charPositionInLine = lexer.getCharPositionInLine();
}
}
}

117
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
View File

@ -63,7 +63,7 @@ import java.util.List;
* satisfy the superclass interface.
*/
public class ParserRuleContext<Symbol> extends RuleContext {
public static final ParserRuleContext EMPTY = new ParserRuleContext();
public static final ParserRuleContext<?> EMPTY = new ParserRuleContext<Object>();
/** If we are debugging or building a parse tree for a visitor,
* we need to track all of the tokens and rule invocations associated
@ -176,61 +176,100 @@ public class ParserRuleContext<Symbol> extends RuleContext {
@Override
public ParseTree getChild(int i) {
return children!=null ? children.get(i) : null;
return children!=null && i>=0 && i<children.size() ? children.get(i) : null;
}
public Object getChild(Class ctxType, int i) {
if ( children==null ) throw new UnsupportedOperationException("there are no children");
int j = -1; // what element have we found with ctxType?
for (Object o : children) {
if ( ctxType.isAssignableFrom(o.getClass()) ) {
j = j+1;
if ( j == i ) return o;
}
public <T extends ParseTree> T getChild(Class<? extends T> ctxType, int i) {
if ( children==null || i < 0 || i >= children.size() ) {
return null;
}
return null;
}
public Token getToken(int ttype, int i) {
if ( children==null ) throw new UnsupportedOperationException("there are no children");
int j = -1; // what token with ttype have we found?
for (Object o : children) {
if ( o instanceof TerminalNode ) {
TerminalNode<Token> tnode = (TerminalNode<Token>)o;
if ( tnode.getSymbol().getType()==ttype ) {
j++;
if ( j == i ) return tnode.getSymbol();
int j = -1; // what element have we found with ctxType?
for (ParseTree o : children) {
if ( ctxType.isInstance(o) ) {
j++;
if ( j == i ) {
return ctxType.cast(o);
}
}
}
return null;
}
public List<Token> getTokens(int ttype) {
if ( children==null ) throw new UnsupportedOperationException("there are no children");
List<Token> tokens = null;
for (Object o : children) {
if ( o instanceof Token ) {
if ( tokens==null ) tokens = new ArrayList<Token>();
tokens.add((Token)o);
public Token getToken(int ttype, int i) {
if ( children==null || i < 0 || i >= children.size() ) {
return null;
}
int j = -1; // what token with ttype have we found?
for (ParseTree o : children) {
if ( o instanceof TerminalNode<?> ) {
TerminalNode<?> tnode = (TerminalNode<?>)o;
if ( tnode.getSymbol() instanceof Token ) {
Token symbol = (Token)tnode.getSymbol();
if ( symbol.getType()==ttype ) {
j++;
if ( j == i ) {
return symbol;
}
}
}
}
}
return null;
}
public List<? extends Token> getTokens(int ttype) {
if ( children==null ) {
return Collections.emptyList();
}
List<Token> tokens = null;
for (ParseTree o : children) {
if ( o instanceof TerminalNode<?> ) {
TerminalNode<?> tnode = (TerminalNode<?>)o;
if ( tnode.getSymbol() instanceof Token ) {
Token symbol = (Token)tnode.getSymbol();
if ( tokens==null ) {
tokens = new ArrayList<Token>();
}
tokens.add(symbol);
}
}
}
if ( tokens==null ) {
return Collections.emptyList();
}
return tokens;
}
public ParserRuleContext getRuleContext(Class ctxType, int i) {
return (ParserRuleContext)getChild(ctxType, i);
public <T extends ParserRuleContext<?>> T getRuleContext(Class<? extends T> ctxType, int i) {
return getChild(ctxType, i);
}
public List<? extends ParserRuleContext> getRuleContexts(Class ctxType) {
if ( children==null ) throw new UnsupportedOperationException("there are no children");
List<ParserRuleContext> contexts = null;
for (Object o : children) {
if ( o.getClass().isInstance(ctxType) ) {
if ( contexts==null ) contexts = new ArrayList<ParserRuleContext>();
contexts.add((ParserRuleContext)o);
public <T extends ParserRuleContext<?>> List<? extends T> getRuleContexts(Class<? extends T> ctxType) {
if ( children==null ) {
return Collections.emptyList();
}
List<T> contexts = null;
for (ParseTree o : children) {
if ( ctxType.isInstance(o) ) {
if ( contexts==null ) {
contexts = new ArrayList<T>();
}
contexts.add(ctxType.cast(o));
}
}
if ( contexts==null ) {
return Collections.emptyList();
}
return contexts;
}
@ -247,7 +286,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
public String toString(@NotNull Recognizer<?,?> recog, RuleContext stop) {
if ( recog==null ) return super.toString(recog, stop);
StringBuilder buf = new StringBuilder();
ParserRuleContext p = this;
ParserRuleContext<?> p = this;
buf.append("[");
while ( p != null && p != stop ) {
ATN atn = recog.getATN();
@ -258,7 +297,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
// ATNState invoker = atn.states.get(ctx.invokingState);
// RuleTransition rt = (RuleTransition)invoker.transition(0);
// buf.append(recog.getRuleNames()[rt.target.ruleIndex]);
p = (ParserRuleContext)p.parent;
p = (ParserRuleContext<?>)p.parent;
}
buf.append("]");
return buf.toString();

5
tool/pom.xml
View File

@ -81,6 +81,11 @@
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.12</version>
</plugin>
</plugins>
</build>

4
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -496,7 +496,7 @@ RuleContextListDecl(rdecl) ::= "public List\<<rdecl.ctxName>> <rdecl.name> = new
ContextTokenGetterDecl(t) ::=
"public Token <t.name>() { return getToken(<parser.name>.<t.name>, 0); }"
ContextTokenListGetterDecl(t) ::=
"public List\<Token> <t.name>() { return getTokens(<parser.name>.<t.name>); }"
"public List\<? extends Token> <t.name>() { return getTokens(<parser.name>.<t.name>); }"
ContextTokenListIndexedGetterDecl(t) ::= <<
public Token <t.name>(int i) {
return getToken(<parser.name>.<t.name>, i);
@ -508,7 +508,7 @@ public <r.ctxName> <r.name>() {
}
>>
ContextRuleListGetterDecl(r) ::= <<
public List\<<r.ctxName>\> <r.name>() {
public List\<? extends <r.ctxName>\> <r.name>() {
return (List\<<r.ctxName>\>)getRuleContexts(<r.ctxName>.class);
}
>>

4
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -71,7 +71,7 @@ public abstract class BaseTest {
public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator");
public static final boolean TEST_IN_SAME_PROCESS = false;
public static final boolean TEST_IN_SAME_PROCESS = Boolean.parseBoolean(System.getProperty("antlr.testinprocess"));
/**
* Build up the full classpath we need, including the surefire path (if present)
@ -510,7 +510,7 @@ public abstract class BaseTest {
}
if ( parserName!=null ) {
files.add(parserName+".java");
files.add("Blank"+grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"Listener.java");
files.add(grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"BaseListener.java");
}
ok = compile(files.toArray(new String[files.size()]));
if ( !ok ) { allIsWell = false; }

47
tool/test/org/antlr/v4/test/Java-LR.g
View File

@ -169,7 +169,6 @@
* Character.isJavaIdentifierPart(int) returns true."
*/
grammar Java;
options {backtrack=true; memoize=true;}
@lexer::members {
protected boolean enumIsKeyword = true;
@ -184,7 +183,9 @@ compilationUnit
( packageDeclaration importDeclaration* typeDeclaration*
| classOrInterfaceDeclaration typeDeclaration*
)
EOF
| packageDeclaration? importDeclaration* typeDeclaration*
EOF
;
packageDeclaration
@ -499,7 +500,7 @@ constructorBody
explicitConstructorInvocation
: nonWildcardTypeArguments? ('this' | 'super') arguments ';'
| expression '.' nonWildcardTypeArguments? 'super' arguments ';'
| primary '.' nonWildcardTypeArguments? 'super' arguments ';'
;
@ -711,17 +712,12 @@ constantExpression
;
expression
: parExpression
| 'this'
| 'super'
| literal
| Identifier
: primary
| expression '.' Identifier
| expression '.' 'class' // should be type.class but causes backtracking
| expression '.' 'this'
| expression '.' 'super' '(' expressionList? ')'
| expression '.' 'super' '.' Identifier arguments?
| expression '.' 'new' Identifier '(' expressionList? ')'
| expression '.' 'super' '.' Identifier arguments?
| expression '.' explicitGenericInvocation
| 'new' creator
| expression '[' expression ']'
@ -754,8 +750,20 @@ expression
| '>' '>' '='<assoc=right>
| '>' '>' '>' '='<assoc=right>
| '<' '<' '='<assoc=right>
| '%='<assoc=right>) expression
;
| '%='<assoc=right>
)
expression
;
primary
: '(' expression ')'
| 'this'
| 'super'
| literal
| Identifier
| type '.' 'class'
| 'void' '.' 'class'
;
creator
: nonWildcardTypeArguments createdName classCreatorRest
@ -827,15 +835,16 @@ FloatingPointLiteral
| ('0'..'9')+ Exponent FloatTypeSuffix?
| ('0'..'9')+ FloatTypeSuffix
| '0' ('x'|'X')
( HexDigit+ '.' HexDigit* Exponent? FloatTypeSuffix?
| '.' HexDigit+ Exponent? FloatTypeSuffix?
| HexDigit+ Exponent FloatTypeSuffix?
| HexDigit+ FloatTypeSuffix
( HexDigit+ ('.' HexDigit*)? HexExponent FloatTypeSuffix?
| '.' HexDigit+ HexExponent FloatTypeSuffix?
)
;
fragment
Exponent : ('e'|'E'|'p'|'P') ('+'|'-')? ('0'..'9')+ ;
Exponent : ('e'|'E') ('+'|'-')? ('0'..'9')+ ;
fragment
HexExponent : ('p'|'P') ('+'|'-')? ('0'..'9')+ ;
fragment
FloatTypeSuffix : ('f'|'F'|'d'|'D') ;
@ -917,13 +926,13 @@ JavaIDDigit
'\u1040'..'\u1049'
;
WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;}
WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ -> channel(HIDDEN)
;
COMMENT
: '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;}
: '/*' .* '*/' -> channel(HIDDEN)
;
LINE_COMMENT
: '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;}
: '//' ~('\n'|'\r')* '\r'? '\n' -> channel(HIDDEN)
;

2
tool/test/org/antlr/v4/test/Java.g
View File

@ -1021,7 +1021,7 @@ WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;}
;
COMMENT
: '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;}
: '/*' .* '*/' {$channel=HIDDEN;}
;
LINE_COMMENT

12
tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java
View File

@ -2,7 +2,7 @@ package org.antlr.v4.test;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.LexerRecognitionExeption;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.Utils;
@ -72,7 +72,7 @@ public class TestATNLexerInterpreter extends BaseTest {
" | 'xy' .\n" + // should not pursue '.' since xy already hit stop
" ;\n");
checkLexerMatches(lg, "xy", "A, EOF");
LexerRecognitionExeption e = checkLexerMatches(lg, "xyz", "A, EOF");
RecognitionException e = checkLexerMatches(lg, "xyz", "A, EOF");
assertEquals("NoViableAltException('z')", e.toString());
}
@ -83,7 +83,7 @@ public class TestATNLexerInterpreter extends BaseTest {
" | 'xy' . 'z'\n" + // will not pursue '.' since xy already hit stop (prior alt)
" ;\n");
// checkLexerMatches(lg, "xy", "A, EOF");
LexerRecognitionExeption e = checkLexerMatches(lg, "xyqz", "A, EOF");
RecognitionException e = checkLexerMatches(lg, "xyqz", "A, EOF");
assertEquals("NoViableAltException('q')", e.toString());
}
@ -244,7 +244,7 @@ public class TestATNLexerInterpreter extends BaseTest {
checkLexerMatches(lg, "a", expecting);
}
protected LexerRecognitionExeption checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
ATN atn = createATN(lg);
CharStream input = new ANTLRInputStream(inputString);
ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE");
@ -252,11 +252,11 @@ public class TestATNLexerInterpreter extends BaseTest {
System.out.println(dot.getDOT(startState, true));
List<String> tokenTypes = null;
LexerRecognitionExeption retException = null;
RecognitionException retException = null;
try {
tokenTypes = getTokenTypes(lg, atn, input, false);
}
catch (LexerRecognitionExeption lre) { retException = lre; }
catch (RecognitionException lre) { retException = lre; }
if ( retException!=null ) return retException;
String result = Utils.join(tokenTypes.iterator(), ", ");

233
tool/test/org/antlr/v4/test/TestPerformance.java
View File

@ -1,3 +1,31 @@
/*
* [The "BSD license"]
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test;
import org.antlr.v4.runtime.*;
@ -14,11 +42,19 @@ import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
public class TestPerformance extends BaseTest {
/** Parse all java files under this package within the JDK_SOURCE_ROOT. */
private static final String TOP_PACKAGE = "java";
private static final String TOP_PACKAGE = "java.lang";
/** True to load java files from sub-packages of {@link #TOP_PACKAGE}. */
private static final boolean RECURSIVE = true;
@ -27,26 +63,41 @@ public class TestPerformance extends BaseTest {
* the standard grammar (Java.g). In either case, the grammar is renamed in the temporary directory to Java.g
* before compiling.
*/
private static final boolean USE_LR_GRAMMAR = false;
private static final boolean USE_LR_GRAMMAR = true;
/**
* True to specify the -Xforceatn option when generating the grammar, forcing all decisions in JavaParser to
* be handled by {@link v2ParserATNSimulator#adaptivePredict}.
* True to specify the -Xforce-atn option when generating the grammar, forcing all decisions in JavaParser to
* be handled by {@link ParserATNSimulator#adaptivePredict}.
*/
private static final boolean FORCE_ATN = false;
/**
* True to specify the -atn option when generating the grammar. This will cause ANTLR
* to export the ATN for each decision as a DOT (GraphViz) file.
*/
private static final boolean EXPORT_ATN_GRAPHS = true;
/**
* True to delete temporary (generated and compiled) files when the test completes.
*/
private static final boolean DELETE_TEMP_FILES = true;
private static final boolean PAUSE_FOR_HEAP_DUMP = false;
/** Parse each file with JavaParser.compilationUnit */
private static final boolean RUN_PARSER = true;
/** True to use {@link BailErrorStrategy}, False to use {@link DefaultErrorStrategy} */
private static final boolean BAIL_ON_ERROR = false;
/** This value is passed to {@link org.antlr.v4.runtime.Parser#setBuildParseTree}. */
private static final boolean BAIL_ON_ERROR = true;
/** This value is passed to {@link Parser#setBuildParseTree}. */
private static final boolean BUILD_PARSE_TREES = false;
/**
* Use ParseTreeWalker.DEFAULT.walk with the BlankJavaParserListener to show parse tree walking overhead.
* If {@link #BUILD_PARSE_TREES} is false, the listener will instead be called during the parsing process via
* {@link org.antlr.v4.runtime.Parser#addParseListener}.
* {@link Parser#addParseListener}.
*/
private static final boolean BLANK_LISTENER = false;
private static final boolean SHOW_DFA_STATE_STATS = true;
private static final boolean SHOW_CONFIG_STATS = false;
/**
* If true, a single JavaLexer will be used, and {@link Lexer#setInputStream} will be called to initialize it
* for each source file. In this mode, the cached DFA will be persisted throughout the lexing process.
@ -67,25 +118,30 @@ public class TestPerformance extends BaseTest {
/** Total number of passes to make over the source */
private static final int PASSES = 4;
private Lexer sharedLexer;
private Parser sharedParser;
private static Lexer sharedLexer;
private static Parser sharedParser;
@SuppressWarnings({"FieldCanBeLocal"})
private ParseTreeListener<Token> sharedListener;
private static ParseTreeListener<Token> sharedListener;
private int tokenCount;
private int currentPass;
@Test
// @Ignore
//@Ignore
public void compileJdk() throws IOException {
compileParser(USE_LR_GRAMMAR);
JavaParserFactory factory = getParserFactory();
String jdkSourceRoot = System.getenv("JDK_SOURCE_ROOT");
if (jdkSourceRoot == null) {
jdkSourceRoot = System.getProperty("JDK_SOURCE_ROOT");
}
if (jdkSourceRoot == null) {
System.err.println("The JDK_SOURCE_ROOT environment variable must be set for performance testing.");
return;
}
if (!TOP_PACKAGE.isEmpty()) {
compileParser(USE_LR_GRAMMAR);
JavaParserFactory factory = getParserFactory();
if (!TOP_PACKAGE.isEmpty()) {
jdkSourceRoot = jdkSourceRoot + '/' + TOP_PACKAGE.replace('.', '/');
}
@ -93,10 +149,13 @@ public class TestPerformance extends BaseTest {
assertTrue(directory.isDirectory());
Collection<CharStream> sources = loadSources(directory, RECURSIVE);
System.out.format("Lex=true, Parse=%s, ForceAtn=%s, Bail=%s, BuildParseTree=%s, BlankListener=%s\n",
RUN_PARSER, FORCE_ATN, BAIL_ON_ERROR, BUILD_PARSE_TREES, BLANK_LISTENER);
System.out.print(getOptionsDescription());
currentPass = 0;
parse1(factory, sources);
for (int i = 0; i < PASSES - 1; i++) {
currentPass = i + 1;
if (CLEAR_DFA) {
sharedLexer = null;
sharedParser = null;
@ -104,6 +163,55 @@ public class TestPerformance extends BaseTest {
parse2(factory, sources);
}
sources.clear();
if (PAUSE_FOR_HEAP_DUMP) {
System.gc();
System.out.println("Pausing before application exit.");
try {
Thread.sleep(4000);
} catch (InterruptedException ex) {
Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
@Override
protected void eraseTempDir() {
if (DELETE_TEMP_FILES) {
super.eraseTempDir();
}
}
public static String getOptionsDescription() {
StringBuilder builder = new StringBuilder();
builder.append("Input=");
if (TestPerformance.TOP_PACKAGE.isEmpty()) {
builder.append("*");
}
else {
builder.append(TOP_PACKAGE).append(".*");
}
builder.append(", Grammar=").append(USE_LR_GRAMMAR ? "LR" : "Standard");
builder.append(", ForceAtn=").append(FORCE_ATN);
builder.append('\n');
builder.append("Op=Lex").append(RUN_PARSER ? "+Parse" : " only");
builder.append(", Strategy=").append(BAIL_ON_ERROR ? BailErrorStrategy.class.getSimpleName() : DefaultErrorStrategy.class.getSimpleName());
builder.append(", BuildParseTree=").append(BUILD_PARSE_TREES);
builder.append(", WalkBlankListener=").append(BLANK_LISTENER);
builder.append('\n');
builder.append("Lexer=").append(REUSE_LEXER ? "setInputStream" : "newInstance");
builder.append(", Parser=").append(REUSE_PARSER ? "setInputStream" : "newInstance");
builder.append(", AfterPass=").append(CLEAR_DFA ? "newInstance" : "setInputStream");
builder.append('\n');
return builder.toString();
}
/**
@ -158,6 +266,8 @@ public class TestPerformance extends BaseTest {
}
}
int configOutputSize = 0;
protected void parseSources(JavaParserFactory factory, Collection<CharStream> sources) {
long startTime = System.currentTimeMillis();
tokenCount = 0;
@ -167,7 +277,7 @@ public class TestPerformance extends BaseTest {
input.seek(0);
inputSize += input.size();
// this incurred a great deal of overhead and was causing significant variations in performance results.
//System.out.format("Parsing file %s\n", file.getAbsolutePath());
//System.out.format("Parsing file %s\n", input.getSourceName());
try {
factory.parseFile(input);
} catch (IllegalStateException ex) {
@ -180,6 +290,80 @@ public class TestPerformance extends BaseTest {
inputSize / 1024,
tokenCount,
System.currentTimeMillis() - startTime);
if (RUN_PARSER) {
// make sure the individual DFAState objects actually have unique ATNConfig arrays
final ParserATNSimulator<?> interpreter = sharedParser.getInterpreter();
final DFA[] decisionToDFA = interpreter.decisionToDFA;
if (SHOW_DFA_STATE_STATS) {
int states = 0;
for (int i = 0; i < decisionToDFA.length; i++) {
DFA dfa = decisionToDFA[i];
if (dfa == null || dfa.states == null) {
continue;
}
states += dfa.states.size();
}
System.out.format("There are %d DFAState instances.\n", states);
}
int localDfaCount = 0;
int globalDfaCount = 0;
int localConfigCount = 0;
int globalConfigCount = 0;
int[] contextsInDFAState = new int[0];
for (int i = 0; i < decisionToDFA.length; i++) {
DFA dfa = decisionToDFA[i];
if (dfa == null || dfa.states == null) {
continue;
}
if (SHOW_CONFIG_STATS) {
for (DFAState state : dfa.states.keySet()) {
if (state.configset.size() >= contextsInDFAState.length) {
contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configset.size() + 1);
}
if (state.isAcceptState) {
boolean hasGlobal = false;
for (ATNConfig config : state.configset) {
if (config.reachesIntoOuterContext > 0) {
globalConfigCount++;
hasGlobal = true;
} else {
localConfigCount++;
}
}
if (hasGlobal) {
globalDfaCount++;
} else {
localDfaCount++;
}
}
contextsInDFAState[state.configset.size()]++;
}
}
}
if (SHOW_CONFIG_STATS && currentPass == 0) {
System.out.format(" DFA accept states: %d total, %d with only local context, %d with a global context\n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
System.out.format(" Config stats: %d total, %d local, %d global\n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
if (SHOW_DFA_STATE_STATS) {
for (int i = 0; i < contextsInDFAState.length; i++) {
if (contextsInDFAState[i] != 0) {
System.out.format(" %d configs = %d\n", i, contextsInDFAState[i]);
}
}
}
}
}
}
protected void compileParser(boolean leftRecursive) throws IOException {
@ -187,8 +371,15 @@ public class TestPerformance extends BaseTest {
String sourceName = leftRecursive ? "Java-LR.g" : "Java.g";
String body = load(sourceName, null);
@SuppressWarnings({"ConstantConditions"})
String[] extraOptions = FORCE_ATN ? new String[] {"-Xforceatn"} : new String[0];
boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptions);
List<String> extraOptions = new ArrayList<String>();
if (FORCE_ATN) {
extraOptions.add("-Xforce-atn");
}
if (EXPORT_ATN_GRAPHS) {
extraOptions.add("-atn");
}
String[] extraOptionsArray = extraOptions.toArray(new String[extraOptions.size()]);
boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptionsArray);
assertTrue(success);
}
@ -228,7 +419,7 @@ public class TestPerformance extends BaseTest {
final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("JavaParser");
@SuppressWarnings({"unchecked"})
final Class<? extends ParseTreeListener<Token>> listenerClass = (Class<? extends ParseTreeListener<Token>>)loader.loadClass("BlankJavaListener");
this.sharedListener = listenerClass.newInstance();
TestPerformance.sharedListener = listenerClass.newInstance();
final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
@ -271,7 +462,7 @@ public class TestPerformance extends BaseTest {
Method parseMethod = parserClass.getMethod("compilationUnit");
Object parseResult = parseMethod.invoke(sharedParser);
assert parseResult instanceof ParseTree;
Assert.assertTrue(parseResult instanceof ParseTree);
if (BUILD_PARSE_TREES && BLANK_LISTENER) {
ParseTreeWalker.DEFAULT.walk(sharedListener, (ParseTree)parseResult);