diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java index 9644389f7..fd43776e3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java @@ -30,6 +30,7 @@ package org.antlr.v4.runtime; import org.antlr.v4.runtime.atn.ATNConfigSet; +import org.antlr.v4.runtime.misc.Utils; public class LexerNoViableAltException extends RecognitionException { /** Matching attempted at what input index? */ @@ -47,7 +48,19 @@ public class LexerNoViableAltException extends RecognitionException { this.deadEndConfigs = deadEndConfigs; } + @Override + public CharStream getInputStream() { + return (CharStream)super.getInputStream(); + } + + @Override public String toString() { - return "NoViableAltException('')"; + String symbol = ""; + if (startIndex >= 0 && startIndex < input.size()) { + symbol = getInputStream().substring(startIndex, startIndex); + symbol = Utils.escapeWhitespace(symbol, false); + } + + return "NoViableAltException('" + symbol + "')"; } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java b/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java deleted file mode 100644 index fbdb7d234..000000000 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.antlr.v4.runtime; - -// TODO: del or rename LexerNoViableAlt? -public class LexerRecognitionExeption extends RuntimeException { - /** Who threw the exception? */ - public Lexer lexer; - - /** What is index of token/char were we looking at when the error occurred? */ - public int index; - - /** The current char when an error occurred. For lexers. */ - public int c; - - /** Track the line at which the error occurred in case this is - * generated from a lexer. We need to track this since the - * unexpected char doesn't carry the line info. - */ - public int line; - - public int charPositionInLine; - - /** Used for remote debugger deserialization */ - public LexerRecognitionExeption() { - } - - public LexerRecognitionExeption(Lexer lexer, CharStream input) { - this.lexer = lexer; - this.index = input.index(); - this.c = input.LA(1); - if ( lexer!=null ) { - this.line = lexer.getLine(); - this.charPositionInLine = lexer.getCharPositionInLine(); - } - } - -} diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java index cb438bd2f..70dac3a32 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java @@ -63,7 +63,7 @@ import java.util.List; * satisfy the superclass interface. */ public class ParserRuleContext extends RuleContext { - public static final ParserRuleContext EMPTY = new ParserRuleContext(); + public static final ParserRuleContext EMPTY = new ParserRuleContext(); /** If we are debugging or building a parse tree for a visitor, * we need to track all of the tokens and rule invocations associated @@ -176,61 +176,100 @@ public class ParserRuleContext extends RuleContext { @Override public ParseTree getChild(int i) { - return children!=null ? children.get(i) : null; + return children!=null && i>=0 && i T getChild(Class ctxType, int i) { + if ( children==null || i < 0 || i >= children.size() ) { + return null; } - return null; - } - public Token getToken(int ttype, int i) { - if ( children==null ) throw new UnsupportedOperationException("there are no children"); - int j = -1; // what token with ttype have we found? - for (Object o : children) { - if ( o instanceof TerminalNode ) { - TerminalNode tnode = (TerminalNode)o; - if ( tnode.getSymbol().getType()==ttype ) { - j++; - if ( j == i ) return tnode.getSymbol(); + int j = -1; // what element have we found with ctxType? + for (ParseTree o : children) { + if ( ctxType.isInstance(o) ) { + j++; + if ( j == i ) { + return ctxType.cast(o); } } } return null; } - public List getTokens(int ttype) { - if ( children==null ) throw new UnsupportedOperationException("there are no children"); - List tokens = null; - for (Object o : children) { - if ( o instanceof Token ) { - if ( tokens==null ) tokens = new ArrayList(); - tokens.add((Token)o); + public Token getToken(int ttype, int i) { + if ( children==null || i < 0 || i >= children.size() ) { + return null; + } + + int j = -1; // what token with ttype have we found? + for (ParseTree o : children) { + if ( o instanceof TerminalNode ) { + TerminalNode tnode = (TerminalNode)o; + if ( tnode.getSymbol() instanceof Token ) { + Token symbol = (Token)tnode.getSymbol(); + if ( symbol.getType()==ttype ) { + j++; + if ( j == i ) { + return symbol; + } + } + } } } + + return null; + } + + public List getTokens(int ttype) { + if ( children==null ) { + return Collections.emptyList(); + } + + List tokens = null; + for (ParseTree o : children) { + if ( o instanceof TerminalNode ) { + TerminalNode tnode = (TerminalNode)o; + if ( tnode.getSymbol() instanceof Token ) { + Token symbol = (Token)tnode.getSymbol(); + if ( tokens==null ) { + tokens = new ArrayList(); + } + tokens.add(symbol); + } + } + } + + if ( tokens==null ) { + return Collections.emptyList(); + } + return tokens; } - public ParserRuleContext getRuleContext(Class ctxType, int i) { - return (ParserRuleContext)getChild(ctxType, i); + public > T getRuleContext(Class ctxType, int i) { + return getChild(ctxType, i); } - public List getRuleContexts(Class ctxType) { - if ( children==null ) throw new UnsupportedOperationException("there are no children"); - List contexts = null; - for (Object o : children) { - if ( o.getClass().isInstance(ctxType) ) { - if ( contexts==null ) contexts = new ArrayList(); - contexts.add((ParserRuleContext)o); + public > List getRuleContexts(Class ctxType) { + if ( children==null ) { + return Collections.emptyList(); + } + + List contexts = null; + for (ParseTree o : children) { + if ( ctxType.isInstance(o) ) { + if ( contexts==null ) { + contexts = new ArrayList(); + } + + contexts.add(ctxType.cast(o)); } } + + if ( contexts==null ) { + return Collections.emptyList(); + } + return contexts; } @@ -247,7 +286,7 @@ public class ParserRuleContext extends RuleContext { public String toString(@NotNull Recognizer recog, RuleContext stop) { if ( recog==null ) return super.toString(recog, stop); StringBuilder buf = new StringBuilder(); - ParserRuleContext p = this; + ParserRuleContext p = this; buf.append("["); while ( p != null && p != stop ) { ATN atn = recog.getATN(); @@ -258,7 +297,7 @@ public class ParserRuleContext extends RuleContext { // ATNState invoker = atn.states.get(ctx.invokingState); // RuleTransition rt = (RuleTransition)invoker.transition(0); // buf.append(recog.getRuleNames()[rt.target.ruleIndex]); - p = (ParserRuleContext)p.parent; + p = (ParserRuleContext)p.parent; } buf.append("]"); return buf.toString(); diff --git a/tool/pom.xml b/tool/pom.xml index 2fb24e25e..c36db1bdb 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -81,6 +81,11 @@ 1.6 + + + maven-surefire-plugin + 2.12 + diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 898b35e28..5c0d27bba 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -496,7 +496,7 @@ RuleContextListDecl(rdecl) ::= "public List\<> = new ContextTokenGetterDecl(t) ::= "public Token () { return getToken(., 0); }" ContextTokenListGetterDecl(t) ::= - "public List\ () { return getTokens(.); }" + "public List\ () { return getTokens(.); }" ContextTokenListIndexedGetterDecl(t) ::= << public Token (int i) { return getToken(., i); @@ -508,7 +508,7 @@ public () { } >> ContextRuleListGetterDecl(r) ::= << -public List\<\> () { +public List\\> () { return (List\<\>)getRuleContexts(.class); } >> diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index 208bcb7ee..eec6343a1 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -71,7 +71,7 @@ public abstract class BaseTest { public static final String newline = System.getProperty("line.separator"); public static final String pathSep = System.getProperty("path.separator"); - public static final boolean TEST_IN_SAME_PROCESS = false; + public static final boolean TEST_IN_SAME_PROCESS = Boolean.parseBoolean(System.getProperty("antlr.testinprocess")); /** * Build up the full classpath we need, including the surefire path (if present) @@ -510,7 +510,7 @@ public abstract class BaseTest { } if ( parserName!=null ) { files.add(parserName+".java"); - files.add("Blank"+grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"Listener.java"); + files.add(grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"BaseListener.java"); } ok = compile(files.toArray(new String[files.size()])); if ( !ok ) { allIsWell = false; } diff --git a/tool/test/org/antlr/v4/test/Java-LR.g b/tool/test/org/antlr/v4/test/Java-LR.g index 97e5facbf..06aa32128 100644 --- a/tool/test/org/antlr/v4/test/Java-LR.g +++ b/tool/test/org/antlr/v4/test/Java-LR.g @@ -169,7 +169,6 @@ * Character.isJavaIdentifierPart(int) returns true." */ grammar Java; -options {backtrack=true; memoize=true;} @lexer::members { protected boolean enumIsKeyword = true; @@ -184,7 +183,9 @@ compilationUnit ( packageDeclaration importDeclaration* typeDeclaration* | classOrInterfaceDeclaration typeDeclaration* ) + EOF | packageDeclaration? importDeclaration* typeDeclaration* + EOF ; packageDeclaration @@ -499,7 +500,7 @@ constructorBody explicitConstructorInvocation : nonWildcardTypeArguments? ('this' | 'super') arguments ';' - | expression '.' nonWildcardTypeArguments? 'super' arguments ';' + | primary '.' nonWildcardTypeArguments? 'super' arguments ';' ; @@ -711,17 +712,12 @@ constantExpression ; expression - : parExpression - | 'this' - | 'super' - | literal - | Identifier + : primary | expression '.' Identifier - | expression '.' 'class' // should be type.class but causes backtracking | expression '.' 'this' | expression '.' 'super' '(' expressionList? ')' - | expression '.' 'super' '.' Identifier arguments? | expression '.' 'new' Identifier '(' expressionList? ')' + | expression '.' 'super' '.' Identifier arguments? | expression '.' explicitGenericInvocation | 'new' creator | expression '[' expression ']' @@ -754,8 +750,20 @@ expression | '>' '>' '=' | '>' '>' '>' '=' | '<' '<' '=' - | '%=') expression - ; + | '%=' + ) + expression + ; + +primary + : '(' expression ')' + | 'this' + | 'super' + | literal + | Identifier + | type '.' 'class' + | 'void' '.' 'class' + ; creator : nonWildcardTypeArguments createdName classCreatorRest @@ -827,15 +835,16 @@ FloatingPointLiteral | ('0'..'9')+ Exponent FloatTypeSuffix? | ('0'..'9')+ FloatTypeSuffix | '0' ('x'|'X') - ( HexDigit+ '.' HexDigit* Exponent? FloatTypeSuffix? - | '.' HexDigit+ Exponent? FloatTypeSuffix? - | HexDigit+ Exponent FloatTypeSuffix? - | HexDigit+ FloatTypeSuffix + ( HexDigit+ ('.' HexDigit*)? HexExponent FloatTypeSuffix? + | '.' HexDigit+ HexExponent FloatTypeSuffix? ) ; fragment -Exponent : ('e'|'E'|'p'|'P') ('+'|'-')? ('0'..'9')+ ; +Exponent : ('e'|'E') ('+'|'-')? ('0'..'9')+ ; + +fragment +HexExponent : ('p'|'P') ('+'|'-')? ('0'..'9')+ ; fragment FloatTypeSuffix : ('f'|'F'|'d'|'D') ; @@ -917,13 +926,13 @@ JavaIDDigit '\u1040'..'\u1049' ; -WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} +WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ -> channel(HIDDEN) ; COMMENT - : '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} + : '/*' .* '*/' -> channel(HIDDEN) ; LINE_COMMENT - : '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;} + : '//' ~('\n'|'\r')* '\r'? '\n' -> channel(HIDDEN) ; diff --git a/tool/test/org/antlr/v4/test/Java.g b/tool/test/org/antlr/v4/test/Java.g index e5b174fda..f5ac3657d 100644 --- a/tool/test/org/antlr/v4/test/Java.g +++ b/tool/test/org/antlr/v4/test/Java.g @@ -1021,7 +1021,7 @@ WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;} ; COMMENT - : '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} + : '/*' .* '*/' {$channel=HIDDEN;} ; LINE_COMMENT diff --git a/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java index 3e021d8f0..947d6a6c1 100644 --- a/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java @@ -2,7 +2,7 @@ package org.antlr.v4.test; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.LexerRecognitionExeption; +import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.Utils; @@ -72,7 +72,7 @@ public class TestATNLexerInterpreter extends BaseTest { " | 'xy' .\n" + // should not pursue '.' since xy already hit stop " ;\n"); checkLexerMatches(lg, "xy", "A, EOF"); - LexerRecognitionExeption e = checkLexerMatches(lg, "xyz", "A, EOF"); + RecognitionException e = checkLexerMatches(lg, "xyz", "A, EOF"); assertEquals("NoViableAltException('z')", e.toString()); } @@ -83,7 +83,7 @@ public class TestATNLexerInterpreter extends BaseTest { " | 'xy' . 'z'\n" + // will not pursue '.' since xy already hit stop (prior alt) " ;\n"); // checkLexerMatches(lg, "xy", "A, EOF"); - LexerRecognitionExeption e = checkLexerMatches(lg, "xyqz", "A, EOF"); + RecognitionException e = checkLexerMatches(lg, "xyqz", "A, EOF"); assertEquals("NoViableAltException('q')", e.toString()); } @@ -244,7 +244,7 @@ public class TestATNLexerInterpreter extends BaseTest { checkLexerMatches(lg, "a", expecting); } - protected LexerRecognitionExeption checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { + protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { ATN atn = createATN(lg); CharStream input = new ANTLRInputStream(inputString); ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE"); @@ -252,11 +252,11 @@ public class TestATNLexerInterpreter extends BaseTest { System.out.println(dot.getDOT(startState, true)); List tokenTypes = null; - LexerRecognitionExeption retException = null; + RecognitionException retException = null; try { tokenTypes = getTokenTypes(lg, atn, input, false); } - catch (LexerRecognitionExeption lre) { retException = lre; } + catch (RecognitionException lre) { retException = lre; } if ( retException!=null ) return retException; String result = Utils.join(tokenTypes.iterator(), ", "); diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java index 98622ed83..2740bc1b8 100644 --- a/tool/test/org/antlr/v4/test/TestPerformance.java +++ b/tool/test/org/antlr/v4/test/TestPerformance.java @@ -1,3 +1,31 @@ +/* + * [The "BSD license"] + * Copyright (c) 2012 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + package org.antlr.v4.test; import org.antlr.v4.runtime.*; @@ -14,11 +42,19 @@ import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.antlr.v4.runtime.atn.ATNConfig; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.dfa.DFAState; public class TestPerformance extends BaseTest { /** Parse all java files under this package within the JDK_SOURCE_ROOT. */ - private static final String TOP_PACKAGE = "java"; + private static final String TOP_PACKAGE = "java.lang"; /** True to load java files from sub-packages of {@link #TOP_PACKAGE}. */ private static final boolean RECURSIVE = true; @@ -27,26 +63,41 @@ public class TestPerformance extends BaseTest { * the standard grammar (Java.g). In either case, the grammar is renamed in the temporary directory to Java.g * before compiling. */ - private static final boolean USE_LR_GRAMMAR = false; + private static final boolean USE_LR_GRAMMAR = true; /** - * True to specify the -Xforceatn option when generating the grammar, forcing all decisions in JavaParser to - * be handled by {@link v2ParserATNSimulator#adaptivePredict}. + * True to specify the -Xforce-atn option when generating the grammar, forcing all decisions in JavaParser to + * be handled by {@link ParserATNSimulator#adaptivePredict}. */ private static final boolean FORCE_ATN = false; + /** + * True to specify the -atn option when generating the grammar. This will cause ANTLR + * to export the ATN for each decision as a DOT (GraphViz) file. + */ + private static final boolean EXPORT_ATN_GRAPHS = true; + /** + * True to delete temporary (generated and compiled) files when the test completes. + */ + private static final boolean DELETE_TEMP_FILES = true; + + private static final boolean PAUSE_FOR_HEAP_DUMP = false; /** Parse each file with JavaParser.compilationUnit */ private static final boolean RUN_PARSER = true; /** True to use {@link BailErrorStrategy}, False to use {@link DefaultErrorStrategy} */ - private static final boolean BAIL_ON_ERROR = false; - /** This value is passed to {@link org.antlr.v4.runtime.Parser#setBuildParseTree}. */ + private static final boolean BAIL_ON_ERROR = true; + /** This value is passed to {@link Parser#setBuildParseTree}. */ private static final boolean BUILD_PARSE_TREES = false; /** * Use ParseTreeWalker.DEFAULT.walk with the BlankJavaParserListener to show parse tree walking overhead. * If {@link #BUILD_PARSE_TREES} is false, the listener will instead be called during the parsing process via - * {@link org.antlr.v4.runtime.Parser#addParseListener}. + * {@link Parser#addParseListener}. */ private static final boolean BLANK_LISTENER = false; + private static final boolean SHOW_DFA_STATE_STATS = true; + + private static final boolean SHOW_CONFIG_STATS = false; + /** * If true, a single JavaLexer will be used, and {@link Lexer#setInputStream} will be called to initialize it * for each source file. In this mode, the cached DFA will be persisted throughout the lexing process. @@ -67,25 +118,30 @@ public class TestPerformance extends BaseTest { /** Total number of passes to make over the source */ private static final int PASSES = 4; - private Lexer sharedLexer; - private Parser sharedParser; + private static Lexer sharedLexer; + private static Parser sharedParser; @SuppressWarnings({"FieldCanBeLocal"}) - private ParseTreeListener sharedListener; + private static ParseTreeListener sharedListener; private int tokenCount; + private int currentPass; @Test -// @Ignore + //@Ignore public void compileJdk() throws IOException { - compileParser(USE_LR_GRAMMAR); - JavaParserFactory factory = getParserFactory(); String jdkSourceRoot = System.getenv("JDK_SOURCE_ROOT"); + if (jdkSourceRoot == null) { + jdkSourceRoot = System.getProperty("JDK_SOURCE_ROOT"); + } if (jdkSourceRoot == null) { System.err.println("The JDK_SOURCE_ROOT environment variable must be set for performance testing."); return; } - if (!TOP_PACKAGE.isEmpty()) { + compileParser(USE_LR_GRAMMAR); + JavaParserFactory factory = getParserFactory(); + + if (!TOP_PACKAGE.isEmpty()) { jdkSourceRoot = jdkSourceRoot + '/' + TOP_PACKAGE.replace('.', '/'); } @@ -93,10 +149,13 @@ public class TestPerformance extends BaseTest { assertTrue(directory.isDirectory()); Collection sources = loadSources(directory, RECURSIVE); - System.out.format("Lex=true, Parse=%s, ForceAtn=%s, Bail=%s, BuildParseTree=%s, BlankListener=%s\n", - RUN_PARSER, FORCE_ATN, BAIL_ON_ERROR, BUILD_PARSE_TREES, BLANK_LISTENER); + + System.out.print(getOptionsDescription()); + + currentPass = 0; parse1(factory, sources); for (int i = 0; i < PASSES - 1; i++) { + currentPass = i + 1; if (CLEAR_DFA) { sharedLexer = null; sharedParser = null; @@ -104,6 +163,55 @@ public class TestPerformance extends BaseTest { parse2(factory, sources); } + + sources.clear(); + if (PAUSE_FOR_HEAP_DUMP) { + System.gc(); + System.out.println("Pausing before application exit."); + try { + Thread.sleep(4000); + } catch (InterruptedException ex) { + Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex); + } + } + } + + @Override + protected void eraseTempDir() { + if (DELETE_TEMP_FILES) { + super.eraseTempDir(); + } + } + + public static String getOptionsDescription() { + StringBuilder builder = new StringBuilder(); + builder.append("Input="); + if (TestPerformance.TOP_PACKAGE.isEmpty()) { + builder.append("*"); + } + else { + builder.append(TOP_PACKAGE).append(".*"); + } + + builder.append(", Grammar=").append(USE_LR_GRAMMAR ? "LR" : "Standard"); + builder.append(", ForceAtn=").append(FORCE_ATN); + + builder.append('\n'); + + builder.append("Op=Lex").append(RUN_PARSER ? "+Parse" : " only"); + builder.append(", Strategy=").append(BAIL_ON_ERROR ? BailErrorStrategy.class.getSimpleName() : DefaultErrorStrategy.class.getSimpleName()); + builder.append(", BuildParseTree=").append(BUILD_PARSE_TREES); + builder.append(", WalkBlankListener=").append(BLANK_LISTENER); + + builder.append('\n'); + + builder.append("Lexer=").append(REUSE_LEXER ? "setInputStream" : "newInstance"); + builder.append(", Parser=").append(REUSE_PARSER ? "setInputStream" : "newInstance"); + builder.append(", AfterPass=").append(CLEAR_DFA ? "newInstance" : "setInputStream"); + + builder.append('\n'); + + return builder.toString(); } /** @@ -158,6 +266,8 @@ public class TestPerformance extends BaseTest { } } + int configOutputSize = 0; + protected void parseSources(JavaParserFactory factory, Collection sources) { long startTime = System.currentTimeMillis(); tokenCount = 0; @@ -167,7 +277,7 @@ public class TestPerformance extends BaseTest { input.seek(0); inputSize += input.size(); // this incurred a great deal of overhead and was causing significant variations in performance results. - //System.out.format("Parsing file %s\n", file.getAbsolutePath()); + //System.out.format("Parsing file %s\n", input.getSourceName()); try { factory.parseFile(input); } catch (IllegalStateException ex) { @@ -180,6 +290,80 @@ public class TestPerformance extends BaseTest { inputSize / 1024, tokenCount, System.currentTimeMillis() - startTime); + + if (RUN_PARSER) { + // make sure the individual DFAState objects actually have unique ATNConfig arrays + final ParserATNSimulator interpreter = sharedParser.getInterpreter(); + final DFA[] decisionToDFA = interpreter.decisionToDFA; + + if (SHOW_DFA_STATE_STATS) { + int states = 0; + + for (int i = 0; i < decisionToDFA.length; i++) { + DFA dfa = decisionToDFA[i]; + if (dfa == null || dfa.states == null) { + continue; + } + + states += dfa.states.size(); + } + + System.out.format("There are %d DFAState instances.\n", states); + } + + int localDfaCount = 0; + int globalDfaCount = 0; + int localConfigCount = 0; + int globalConfigCount = 0; + int[] contextsInDFAState = new int[0]; + + for (int i = 0; i < decisionToDFA.length; i++) { + DFA dfa = decisionToDFA[i]; + if (dfa == null || dfa.states == null) { + continue; + } + + if (SHOW_CONFIG_STATS) { + for (DFAState state : dfa.states.keySet()) { + if (state.configset.size() >= contextsInDFAState.length) { + contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configset.size() + 1); + } + + if (state.isAcceptState) { + boolean hasGlobal = false; + for (ATNConfig config : state.configset) { + if (config.reachesIntoOuterContext > 0) { + globalConfigCount++; + hasGlobal = true; + } else { + localConfigCount++; + } + } + + if (hasGlobal) { + globalDfaCount++; + } else { + localDfaCount++; + } + } + + contextsInDFAState[state.configset.size()]++; + } + } + } + + if (SHOW_CONFIG_STATS && currentPass == 0) { + System.out.format(" DFA accept states: %d total, %d with only local context, %d with a global context\n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount); + System.out.format(" Config stats: %d total, %d local, %d global\n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount); + if (SHOW_DFA_STATE_STATS) { + for (int i = 0; i < contextsInDFAState.length; i++) { + if (contextsInDFAState[i] != 0) { + System.out.format(" %d configs = %d\n", i, contextsInDFAState[i]); + } + } + } + } + } } protected void compileParser(boolean leftRecursive) throws IOException { @@ -187,8 +371,15 @@ public class TestPerformance extends BaseTest { String sourceName = leftRecursive ? "Java-LR.g" : "Java.g"; String body = load(sourceName, null); @SuppressWarnings({"ConstantConditions"}) - String[] extraOptions = FORCE_ATN ? new String[] {"-Xforceatn"} : new String[0]; - boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptions); + List extraOptions = new ArrayList(); + if (FORCE_ATN) { + extraOptions.add("-Xforce-atn"); + } + if (EXPORT_ATN_GRAPHS) { + extraOptions.add("-atn"); + } + String[] extraOptionsArray = extraOptions.toArray(new String[extraOptions.size()]); + boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptionsArray); assertTrue(success); } @@ -228,7 +419,7 @@ public class TestPerformance extends BaseTest { final Class parserClass = (Class)loader.loadClass("JavaParser"); @SuppressWarnings({"unchecked"}) final Class> listenerClass = (Class>)loader.loadClass("BlankJavaListener"); - this.sharedListener = listenerClass.newInstance(); + TestPerformance.sharedListener = listenerClass.newInstance(); final Constructor lexerCtor = lexerClass.getConstructor(CharStream.class); final Constructor parserCtor = parserClass.getConstructor(TokenStream.class); @@ -271,7 +462,7 @@ public class TestPerformance extends BaseTest { Method parseMethod = parserClass.getMethod("compilationUnit"); Object parseResult = parseMethod.invoke(sharedParser); - assert parseResult instanceof ParseTree; + Assert.assertTrue(parseResult instanceof ParseTree); if (BUILD_PARSE_TREES && BLANK_LISTENER) { ParseTreeWalker.DEFAULT.walk(sharedListener, (ParseTree)parseResult);