Merge pull request #13 from sharwell/parrt-master
Several fixes on the latest updates
This commit is contained in:
Terence Parr
commit
b44b0a3c09
|
|
@ -30,6 +30,7 @@
|
|||
package org.antlr.v4.runtime;
|
||||
|
||||
import org.antlr.v4.runtime.atn.ATNConfigSet;
|
||||
import org.antlr.v4.runtime.misc.Utils;
|
||||
|
||||
public class LexerNoViableAltException extends RecognitionException {
|
||||
/** Matching attempted at what input index? */
|
||||
|
|
@ -47,7 +48,19 @@ public class LexerNoViableAltException extends RecognitionException {
|
|||
this.deadEndConfigs = deadEndConfigs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharStream getInputStream() {
|
||||
return (CharStream)super.getInputStream();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "NoViableAltException('')";
|
||||
String symbol = "";
|
||||
if (startIndex >= 0 && startIndex < input.size()) {
|
||||
symbol = getInputStream().substring(startIndex, startIndex);
|
||||
symbol = Utils.escapeWhitespace(symbol, false);
|
||||
}
|
||||
|
||||
return "NoViableAltException('" + symbol + "')";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
/*
|
||||
[The "BSD license"]
|
||||
Copyright (c) 2011 Terence Parr
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
3. The name of the author may not be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
||||
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
package org.antlr.v4.runtime;
|
||||
|
||||
// TODO: del or rename LexerNoViableAlt?
|
||||
public class LexerRecognitionExeption extends RuntimeException {
|
||||
/** Who threw the exception? */
|
||||
public Lexer lexer;
|
||||
|
||||
/** What is index of token/char were we looking at when the error occurred? */
|
||||
public int index;
|
||||
|
||||
/** The current char when an error occurred. For lexers. */
|
||||
public int c;
|
||||
|
||||
/** Track the line at which the error occurred in case this is
|
||||
* generated from a lexer. We need to track this since the
|
||||
* unexpected char doesn't carry the line info.
|
||||
*/
|
||||
public int line;
|
||||
|
||||
public int charPositionInLine;
|
||||
|
||||
/** Used for remote debugger deserialization */
|
||||
public LexerRecognitionExeption() {
|
||||
}
|
||||
|
||||
public LexerRecognitionExeption(Lexer lexer, CharStream input) {
|
||||
this.lexer = lexer;
|
||||
this.index = input.index();
|
||||
this.c = input.LA(1);
|
||||
if ( lexer!=null ) {
|
||||
this.line = lexer.getLine();
|
||||
this.charPositionInLine = lexer.getCharPositionInLine();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -63,7 +63,7 @@ import java.util.List;
|
|||
* satisfy the superclass interface.
|
||||
*/
|
||||
public class ParserRuleContext<Symbol> extends RuleContext {
|
||||
public static final ParserRuleContext EMPTY = new ParserRuleContext();
|
||||
public static final ParserRuleContext<?> EMPTY = new ParserRuleContext<Object>();
|
||||
|
||||
/** If we are debugging or building a parse tree for a visitor,
|
||||
* we need to track all of the tokens and rule invocations associated
|
||||
|
|
@ -176,61 +176,100 @@ public class ParserRuleContext<Symbol> extends RuleContext {
|
|||
|
||||
@Override
|
||||
public ParseTree getChild(int i) {
|
||||
return children!=null ? children.get(i) : null;
|
||||
return children!=null && i>=0 && i<children.size() ? children.get(i) : null;
|
||||
}
|
||||
|
||||
public <T extends ParseTree> T getChild(Class<? extends T> ctxType, int i) {
|
||||
if ( children==null || i < 0 || i >= children.size() ) {
|
||||
return null;
|
||||
}
|
||||
|
||||
public Object getChild(Class ctxType, int i) {
|
||||
if ( children==null ) throw new UnsupportedOperationException("there are no children");
|
||||
int j = -1; // what element have we found with ctxType?
|
||||
for (Object o : children) {
|
||||
if ( ctxType.isAssignableFrom(o.getClass()) ) {
|
||||
j = j+1;
|
||||
if ( j == i ) return o;
|
||||
for (ParseTree o : children) {
|
||||
if ( ctxType.isInstance(o) ) {
|
||||
j++;
|
||||
if ( j == i ) {
|
||||
return ctxType.cast(o);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public Token getToken(int ttype, int i) {
|
||||
if ( children==null ) throw new UnsupportedOperationException("there are no children");
|
||||
int j = -1; // what token with ttype have we found?
|
||||
for (Object o : children) {
|
||||
if ( o instanceof TerminalNode ) {
|
||||
TerminalNode<Token> tnode = (TerminalNode<Token>)o;
|
||||
if ( tnode.getSymbol().getType()==ttype ) {
|
||||
j++;
|
||||
if ( j == i ) return tnode.getSymbol();
|
||||
}
|
||||
}
|
||||
}
|
||||
if ( children==null || i < 0 || i >= children.size() ) {
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<Token> getTokens(int ttype) {
|
||||
if ( children==null ) throw new UnsupportedOperationException("there are no children");
|
||||
int j = -1; // what token with ttype have we found?
|
||||
for (ParseTree o : children) {
|
||||
if ( o instanceof TerminalNode<?> ) {
|
||||
TerminalNode<?> tnode = (TerminalNode<?>)o;
|
||||
if ( tnode.getSymbol() instanceof Token ) {
|
||||
Token symbol = (Token)tnode.getSymbol();
|
||||
if ( symbol.getType()==ttype ) {
|
||||
j++;
|
||||
if ( j == i ) {
|
||||
return symbol;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<? extends Token> getTokens(int ttype) {
|
||||
if ( children==null ) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
List<Token> tokens = null;
|
||||
for (Object o : children) {
|
||||
if ( o instanceof Token ) {
|
||||
if ( tokens==null ) tokens = new ArrayList<Token>();
|
||||
tokens.add((Token)o);
|
||||
for (ParseTree o : children) {
|
||||
if ( o instanceof TerminalNode<?> ) {
|
||||
TerminalNode<?> tnode = (TerminalNode<?>)o;
|
||||
if ( tnode.getSymbol() instanceof Token ) {
|
||||
Token symbol = (Token)tnode.getSymbol();
|
||||
if ( tokens==null ) {
|
||||
tokens = new ArrayList<Token>();
|
||||
}
|
||||
tokens.add(symbol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ( tokens==null ) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public ParserRuleContext getRuleContext(Class ctxType, int i) {
|
||||
return (ParserRuleContext)getChild(ctxType, i);
|
||||
public <T extends ParserRuleContext<?>> T getRuleContext(Class<? extends T> ctxType, int i) {
|
||||
return getChild(ctxType, i);
|
||||
}
|
||||
|
||||
public List<? extends ParserRuleContext> getRuleContexts(Class ctxType) {
|
||||
if ( children==null ) throw new UnsupportedOperationException("there are no children");
|
||||
List<ParserRuleContext> contexts = null;
|
||||
for (Object o : children) {
|
||||
if ( o.getClass().isInstance(ctxType) ) {
|
||||
if ( contexts==null ) contexts = new ArrayList<ParserRuleContext>();
|
||||
contexts.add((ParserRuleContext)o);
|
||||
public <T extends ParserRuleContext<?>> List<? extends T> getRuleContexts(Class<? extends T> ctxType) {
|
||||
if ( children==null ) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
List<T> contexts = null;
|
||||
for (ParseTree o : children) {
|
||||
if ( ctxType.isInstance(o) ) {
|
||||
if ( contexts==null ) {
|
||||
contexts = new ArrayList<T>();
|
||||
}
|
||||
|
||||
contexts.add(ctxType.cast(o));
|
||||
}
|
||||
}
|
||||
|
||||
if ( contexts==null ) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
return contexts;
|
||||
}
|
||||
|
||||
|
|
@ -247,7 +286,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
|
|||
public String toString(@NotNull Recognizer<?,?> recog, RuleContext stop) {
|
||||
if ( recog==null ) return super.toString(recog, stop);
|
||||
StringBuilder buf = new StringBuilder();
|
||||
ParserRuleContext p = this;
|
||||
ParserRuleContext<?> p = this;
|
||||
buf.append("[");
|
||||
while ( p != null && p != stop ) {
|
||||
ATN atn = recog.getATN();
|
||||
|
|
@ -258,7 +297,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
|
|||
// ATNState invoker = atn.states.get(ctx.invokingState);
|
||||
// RuleTransition rt = (RuleTransition)invoker.transition(0);
|
||||
// buf.append(recog.getRuleNames()[rt.target.ruleIndex]);
|
||||
p = (ParserRuleContext)p.parent;
|
||||
p = (ParserRuleContext<?>)p.parent;
|
||||
}
|
||||
buf.append("]");
|
||||
return buf.toString();
|
||||
|
|
|
|||
|
|
@ -81,6 +81,11 @@
|
|||
<target>1.6</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.12</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
|
|
|||
|
|
@ -496,7 +496,7 @@ RuleContextListDecl(rdecl) ::= "public List\<<rdecl.ctxName>> <rdecl.name> = new
|
|||
ContextTokenGetterDecl(t) ::=
|
||||
"public Token <t.name>() { return getToken(<parser.name>.<t.name>, 0); }"
|
||||
ContextTokenListGetterDecl(t) ::=
|
||||
"public List\<Token> <t.name>() { return getTokens(<parser.name>.<t.name>); }"
|
||||
"public List\<? extends Token> <t.name>() { return getTokens(<parser.name>.<t.name>); }"
|
||||
ContextTokenListIndexedGetterDecl(t) ::= <<
|
||||
public Token <t.name>(int i) {
|
||||
return getToken(<parser.name>.<t.name>, i);
|
||||
|
|
@ -508,7 +508,7 @@ public <r.ctxName> <r.name>() {
|
|||
}
|
||||
>>
|
||||
ContextRuleListGetterDecl(r) ::= <<
|
||||
public List\<<r.ctxName>\> <r.name>() {
|
||||
public List\<? extends <r.ctxName>\> <r.name>() {
|
||||
return (List\<<r.ctxName>\>)getRuleContexts(<r.ctxName>.class);
|
||||
}
|
||||
>>
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ public abstract class BaseTest {
|
|||
public static final String newline = System.getProperty("line.separator");
|
||||
public static final String pathSep = System.getProperty("path.separator");
|
||||
|
||||
public static final boolean TEST_IN_SAME_PROCESS = false;
|
||||
public static final boolean TEST_IN_SAME_PROCESS = Boolean.parseBoolean(System.getProperty("antlr.testinprocess"));
|
||||
|
||||
/**
|
||||
* Build up the full classpath we need, including the surefire path (if present)
|
||||
|
|
@ -510,7 +510,7 @@ public abstract class BaseTest {
|
|||
}
|
||||
if ( parserName!=null ) {
|
||||
files.add(parserName+".java");
|
||||
files.add("Blank"+grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"Listener.java");
|
||||
files.add(grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"BaseListener.java");
|
||||
}
|
||||
ok = compile(files.toArray(new String[files.size()]));
|
||||
if ( !ok ) { allIsWell = false; }
|
||||
|
|
|
|||
|
|
@ -169,7 +169,6 @@
|
|||
* Character.isJavaIdentifierPart(int) returns true."
|
||||
*/
|
||||
grammar Java;
|
||||
options {backtrack=true; memoize=true;}
|
||||
|
||||
@lexer::members {
|
||||
protected boolean enumIsKeyword = true;
|
||||
|
|
@ -184,7 +183,9 @@ compilationUnit
|
|||
( packageDeclaration importDeclaration* typeDeclaration*
|
||||
| classOrInterfaceDeclaration typeDeclaration*
|
||||
)
|
||||
EOF
|
||||
| packageDeclaration? importDeclaration* typeDeclaration*
|
||||
EOF
|
||||
;
|
||||
|
||||
packageDeclaration
|
||||
|
|
@ -499,7 +500,7 @@ constructorBody
|
|||
|
||||
explicitConstructorInvocation
|
||||
: nonWildcardTypeArguments? ('this' | 'super') arguments ';'
|
||||
| expression '.' nonWildcardTypeArguments? 'super' arguments ';'
|
||||
| primary '.' nonWildcardTypeArguments? 'super' arguments ';'
|
||||
;
|
||||
|
||||
|
||||
|
|
@ -711,17 +712,12 @@ constantExpression
|
|||
;
|
||||
|
||||
expression
|
||||
: parExpression
|
||||
| 'this'
|
||||
| 'super'
|
||||
| literal
|
||||
| Identifier
|
||||
: primary
|
||||
| expression '.' Identifier
|
||||
| expression '.' 'class' // should be type.class but causes backtracking
|
||||
| expression '.' 'this'
|
||||
| expression '.' 'super' '(' expressionList? ')'
|
||||
| expression '.' 'super' '.' Identifier arguments?
|
||||
| expression '.' 'new' Identifier '(' expressionList? ')'
|
||||
| expression '.' 'super' '.' Identifier arguments?
|
||||
| expression '.' explicitGenericInvocation
|
||||
| 'new' creator
|
||||
| expression '[' expression ']'
|
||||
|
|
@ -754,7 +750,19 @@ expression
|
|||
| '>' '>' '='<assoc=right>
|
||||
| '>' '>' '>' '='<assoc=right>
|
||||
| '<' '<' '='<assoc=right>
|
||||
| '%='<assoc=right>) expression
|
||||
| '%='<assoc=right>
|
||||
)
|
||||
expression
|
||||
;
|
||||
|
||||
primary
|
||||
: '(' expression ')'
|
||||
| 'this'
|
||||
| 'super'
|
||||
| literal
|
||||
| Identifier
|
||||
| type '.' 'class'
|
||||
| 'void' '.' 'class'
|
||||
;
|
||||
|
||||
creator
|
||||
|
|
@ -827,15 +835,16 @@ FloatingPointLiteral
|
|||
| ('0'..'9')+ Exponent FloatTypeSuffix?
|
||||
| ('0'..'9')+ FloatTypeSuffix
|
||||
| '0' ('x'|'X')
|
||||
( HexDigit+ '.' HexDigit* Exponent? FloatTypeSuffix?
|
||||
| '.' HexDigit+ Exponent? FloatTypeSuffix?
|
||||
| HexDigit+ Exponent FloatTypeSuffix?
|
||||
| HexDigit+ FloatTypeSuffix
|
||||
( HexDigit+ ('.' HexDigit*)? HexExponent FloatTypeSuffix?
|
||||
| '.' HexDigit+ HexExponent FloatTypeSuffix?
|
||||
)
|
||||
;
|
||||
|
||||
fragment
|
||||
Exponent : ('e'|'E'|'p'|'P') ('+'|'-')? ('0'..'9')+ ;
|
||||
Exponent : ('e'|'E') ('+'|'-')? ('0'..'9')+ ;
|
||||
|
||||
fragment
|
||||
HexExponent : ('p'|'P') ('+'|'-')? ('0'..'9')+ ;
|
||||
|
||||
fragment
|
||||
FloatTypeSuffix : ('f'|'F'|'d'|'D') ;
|
||||
|
|
@ -917,13 +926,13 @@ JavaIDDigit
|
|||
'\u1040'..'\u1049'
|
||||
;
|
||||
|
||||
WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;}
|
||||
WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ -> channel(HIDDEN)
|
||||
;
|
||||
|
||||
COMMENT
|
||||
: '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;}
|
||||
: '/*' .* '*/' -> channel(HIDDEN)
|
||||
;
|
||||
|
||||
LINE_COMMENT
|
||||
: '//' ~('\n'|'\r')* '\r'? '\n' {$channel=HIDDEN;}
|
||||
: '//' ~('\n'|'\r')* '\r'? '\n' -> channel(HIDDEN)
|
||||
;
|
||||
|
|
|
|||
|
|
@ -1021,7 +1021,7 @@ WS : (' '|'\r'|'\t'|'\u000C'|'\n')+ {$channel=HIDDEN;}
|
|||
;
|
||||
|
||||
COMMENT
|
||||
: '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;}
|
||||
: '/*' .* '*/' {$channel=HIDDEN;}
|
||||
;
|
||||
|
||||
LINE_COMMENT
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package org.antlr.v4.test;
|
|||
|
||||
import org.antlr.v4.runtime.ANTLRInputStream;
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.v4.runtime.LexerRecognitionExeption;
|
||||
import org.antlr.v4.runtime.RecognitionException;
|
||||
import org.antlr.v4.runtime.atn.ATN;
|
||||
import org.antlr.v4.runtime.atn.ATNState;
|
||||
import org.antlr.v4.runtime.misc.Utils;
|
||||
|
|
@ -72,7 +72,7 @@ public class TestATNLexerInterpreter extends BaseTest {
|
|||
" | 'xy' .\n" + // should not pursue '.' since xy already hit stop
|
||||
" ;\n");
|
||||
checkLexerMatches(lg, "xy", "A, EOF");
|
||||
LexerRecognitionExeption e = checkLexerMatches(lg, "xyz", "A, EOF");
|
||||
RecognitionException e = checkLexerMatches(lg, "xyz", "A, EOF");
|
||||
assertEquals("NoViableAltException('z')", e.toString());
|
||||
}
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ public class TestATNLexerInterpreter extends BaseTest {
|
|||
" | 'xy' . 'z'\n" + // will not pursue '.' since xy already hit stop (prior alt)
|
||||
" ;\n");
|
||||
// checkLexerMatches(lg, "xy", "A, EOF");
|
||||
LexerRecognitionExeption e = checkLexerMatches(lg, "xyqz", "A, EOF");
|
||||
RecognitionException e = checkLexerMatches(lg, "xyqz", "A, EOF");
|
||||
assertEquals("NoViableAltException('q')", e.toString());
|
||||
}
|
||||
|
||||
|
|
@ -244,7 +244,7 @@ public class TestATNLexerInterpreter extends BaseTest {
|
|||
checkLexerMatches(lg, "a", expecting);
|
||||
}
|
||||
|
||||
protected LexerRecognitionExeption checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
|
||||
protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
|
||||
ATN atn = createATN(lg);
|
||||
CharStream input = new ANTLRInputStream(inputString);
|
||||
ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE");
|
||||
|
|
@ -252,11 +252,11 @@ public class TestATNLexerInterpreter extends BaseTest {
|
|||
System.out.println(dot.getDOT(startState, true));
|
||||
|
||||
List<String> tokenTypes = null;
|
||||
LexerRecognitionExeption retException = null;
|
||||
RecognitionException retException = null;
|
||||
try {
|
||||
tokenTypes = getTokenTypes(lg, atn, input, false);
|
||||
}
|
||||
catch (LexerRecognitionExeption lre) { retException = lre; }
|
||||
catch (RecognitionException lre) { retException = lre; }
|
||||
if ( retException!=null ) return retException;
|
||||
|
||||
String result = Utils.join(tokenTypes.iterator(), ", ");
|
||||
|
|
|
|||
|
|
@ -1,3 +1,31 @@
|
|||
/*
|
||||
* [The "BSD license"]
|
||||
* Copyright (c) 2012 Sam Harwell
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* 3. The name of the author may not be used to endorse or promote products
|
||||
* derived from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
package org.antlr.v4.test;
|
||||
|
||||
import org.antlr.v4.runtime.*;
|
||||
|
|
@ -14,11 +42,19 @@ import java.lang.reflect.Method;
|
|||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.antlr.v4.runtime.atn.ATNConfig;
|
||||
import org.antlr.v4.runtime.atn.ParserATNSimulator;
|
||||
import org.antlr.v4.runtime.dfa.DFA;
|
||||
import org.antlr.v4.runtime.dfa.DFAState;
|
||||
|
||||
public class TestPerformance extends BaseTest {
|
||||
/** Parse all java files under this package within the JDK_SOURCE_ROOT. */
|
||||
private static final String TOP_PACKAGE = "java";
|
||||
private static final String TOP_PACKAGE = "java.lang";
|
||||
/** True to load java files from sub-packages of {@link #TOP_PACKAGE}. */
|
||||
private static final boolean RECURSIVE = true;
|
||||
|
||||
|
|
@ -27,26 +63,41 @@ public class TestPerformance extends BaseTest {
|
|||
* the standard grammar (Java.g). In either case, the grammar is renamed in the temporary directory to Java.g
|
||||
* before compiling.
|
||||
*/
|
||||
private static final boolean USE_LR_GRAMMAR = false;
|
||||
private static final boolean USE_LR_GRAMMAR = true;
|
||||
/**
|
||||
* True to specify the -Xforceatn option when generating the grammar, forcing all decisions in JavaParser to
|
||||
* be handled by {@link v2ParserATNSimulator#adaptivePredict}.
|
||||
* True to specify the -Xforce-atn option when generating the grammar, forcing all decisions in JavaParser to
|
||||
* be handled by {@link ParserATNSimulator#adaptivePredict}.
|
||||
*/
|
||||
private static final boolean FORCE_ATN = false;
|
||||
/**
|
||||
* True to specify the -atn option when generating the grammar. This will cause ANTLR
|
||||
* to export the ATN for each decision as a DOT (GraphViz) file.
|
||||
*/
|
||||
private static final boolean EXPORT_ATN_GRAPHS = true;
|
||||
/**
|
||||
* True to delete temporary (generated and compiled) files when the test completes.
|
||||
*/
|
||||
private static final boolean DELETE_TEMP_FILES = true;
|
||||
|
||||
private static final boolean PAUSE_FOR_HEAP_DUMP = false;
|
||||
|
||||
/** Parse each file with JavaParser.compilationUnit */
|
||||
private static final boolean RUN_PARSER = true;
|
||||
/** True to use {@link BailErrorStrategy}, False to use {@link DefaultErrorStrategy} */
|
||||
private static final boolean BAIL_ON_ERROR = false;
|
||||
/** This value is passed to {@link org.antlr.v4.runtime.Parser#setBuildParseTree}. */
|
||||
private static final boolean BAIL_ON_ERROR = true;
|
||||
/** This value is passed to {@link Parser#setBuildParseTree}. */
|
||||
private static final boolean BUILD_PARSE_TREES = false;
|
||||
/**
|
||||
* Use ParseTreeWalker.DEFAULT.walk with the BlankJavaParserListener to show parse tree walking overhead.
|
||||
* If {@link #BUILD_PARSE_TREES} is false, the listener will instead be called during the parsing process via
|
||||
* {@link org.antlr.v4.runtime.Parser#addParseListener}.
|
||||
* {@link Parser#addParseListener}.
|
||||
*/
|
||||
private static final boolean BLANK_LISTENER = false;
|
||||
|
||||
private static final boolean SHOW_DFA_STATE_STATS = true;
|
||||
|
||||
private static final boolean SHOW_CONFIG_STATS = false;
|
||||
|
||||
/**
|
||||
* If true, a single JavaLexer will be used, and {@link Lexer#setInputStream} will be called to initialize it
|
||||
* for each source file. In this mode, the cached DFA will be persisted throughout the lexing process.
|
||||
|
|
@ -67,24 +118,29 @@ public class TestPerformance extends BaseTest {
|
|||
/** Total number of passes to make over the source */
|
||||
private static final int PASSES = 4;
|
||||
|
||||
private Lexer sharedLexer;
|
||||
private Parser sharedParser;
|
||||
private static Lexer sharedLexer;
|
||||
private static Parser sharedParser;
|
||||
@SuppressWarnings({"FieldCanBeLocal"})
|
||||
private ParseTreeListener<Token> sharedListener;
|
||||
private static ParseTreeListener<Token> sharedListener;
|
||||
|
||||
private int tokenCount;
|
||||
private int currentPass;
|
||||
|
||||
@Test
|
||||
//@Ignore
|
||||
public void compileJdk() throws IOException {
|
||||
compileParser(USE_LR_GRAMMAR);
|
||||
JavaParserFactory factory = getParserFactory();
|
||||
String jdkSourceRoot = System.getenv("JDK_SOURCE_ROOT");
|
||||
if (jdkSourceRoot == null) {
|
||||
jdkSourceRoot = System.getProperty("JDK_SOURCE_ROOT");
|
||||
}
|
||||
if (jdkSourceRoot == null) {
|
||||
System.err.println("The JDK_SOURCE_ROOT environment variable must be set for performance testing.");
|
||||
return;
|
||||
}
|
||||
|
||||
compileParser(USE_LR_GRAMMAR);
|
||||
JavaParserFactory factory = getParserFactory();
|
||||
|
||||
if (!TOP_PACKAGE.isEmpty()) {
|
||||
jdkSourceRoot = jdkSourceRoot + '/' + TOP_PACKAGE.replace('.', '/');
|
||||
}
|
||||
|
|
@ -93,10 +149,13 @@ public class TestPerformance extends BaseTest {
|
|||
assertTrue(directory.isDirectory());
|
||||
|
||||
Collection<CharStream> sources = loadSources(directory, RECURSIVE);
|
||||
System.out.format("Lex=true, Parse=%s, ForceAtn=%s, Bail=%s, BuildParseTree=%s, BlankListener=%s\n",
|
||||
RUN_PARSER, FORCE_ATN, BAIL_ON_ERROR, BUILD_PARSE_TREES, BLANK_LISTENER);
|
||||
|
||||
System.out.print(getOptionsDescription());
|
||||
|
||||
currentPass = 0;
|
||||
parse1(factory, sources);
|
||||
for (int i = 0; i < PASSES - 1; i++) {
|
||||
currentPass = i + 1;
|
||||
if (CLEAR_DFA) {
|
||||
sharedLexer = null;
|
||||
sharedParser = null;
|
||||
|
|
@ -104,6 +163,55 @@ public class TestPerformance extends BaseTest {
|
|||
|
||||
parse2(factory, sources);
|
||||
}
|
||||
|
||||
sources.clear();
|
||||
if (PAUSE_FOR_HEAP_DUMP) {
|
||||
System.gc();
|
||||
System.out.println("Pausing before application exit.");
|
||||
try {
|
||||
Thread.sleep(4000);
|
||||
} catch (InterruptedException ex) {
|
||||
Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void eraseTempDir() {
|
||||
if (DELETE_TEMP_FILES) {
|
||||
super.eraseTempDir();
|
||||
}
|
||||
}
|
||||
|
||||
public static String getOptionsDescription() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("Input=");
|
||||
if (TestPerformance.TOP_PACKAGE.isEmpty()) {
|
||||
builder.append("*");
|
||||
}
|
||||
else {
|
||||
builder.append(TOP_PACKAGE).append(".*");
|
||||
}
|
||||
|
||||
builder.append(", Grammar=").append(USE_LR_GRAMMAR ? "LR" : "Standard");
|
||||
builder.append(", ForceAtn=").append(FORCE_ATN);
|
||||
|
||||
builder.append('\n');
|
||||
|
||||
builder.append("Op=Lex").append(RUN_PARSER ? "+Parse" : " only");
|
||||
builder.append(", Strategy=").append(BAIL_ON_ERROR ? BailErrorStrategy.class.getSimpleName() : DefaultErrorStrategy.class.getSimpleName());
|
||||
builder.append(", BuildParseTree=").append(BUILD_PARSE_TREES);
|
||||
builder.append(", WalkBlankListener=").append(BLANK_LISTENER);
|
||||
|
||||
builder.append('\n');
|
||||
|
||||
builder.append("Lexer=").append(REUSE_LEXER ? "setInputStream" : "newInstance");
|
||||
builder.append(", Parser=").append(REUSE_PARSER ? "setInputStream" : "newInstance");
|
||||
builder.append(", AfterPass=").append(CLEAR_DFA ? "newInstance" : "setInputStream");
|
||||
|
||||
builder.append('\n');
|
||||
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -158,6 +266,8 @@ public class TestPerformance extends BaseTest {
|
|||
}
|
||||
}
|
||||
|
||||
int configOutputSize = 0;
|
||||
|
||||
protected void parseSources(JavaParserFactory factory, Collection<CharStream> sources) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
tokenCount = 0;
|
||||
|
|
@ -167,7 +277,7 @@ public class TestPerformance extends BaseTest {
|
|||
input.seek(0);
|
||||
inputSize += input.size();
|
||||
// this incurred a great deal of overhead and was causing significant variations in performance results.
|
||||
//System.out.format("Parsing file %s\n", file.getAbsolutePath());
|
||||
//System.out.format("Parsing file %s\n", input.getSourceName());
|
||||
try {
|
||||
factory.parseFile(input);
|
||||
} catch (IllegalStateException ex) {
|
||||
|
|
@ -180,6 +290,80 @@ public class TestPerformance extends BaseTest {
|
|||
inputSize / 1024,
|
||||
tokenCount,
|
||||
System.currentTimeMillis() - startTime);
|
||||
|
||||
if (RUN_PARSER) {
|
||||
// make sure the individual DFAState objects actually have unique ATNConfig arrays
|
||||
final ParserATNSimulator<?> interpreter = sharedParser.getInterpreter();
|
||||
final DFA[] decisionToDFA = interpreter.decisionToDFA;
|
||||
|
||||
if (SHOW_DFA_STATE_STATS) {
|
||||
int states = 0;
|
||||
|
||||
for (int i = 0; i < decisionToDFA.length; i++) {
|
||||
DFA dfa = decisionToDFA[i];
|
||||
if (dfa == null || dfa.states == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
states += dfa.states.size();
|
||||
}
|
||||
|
||||
System.out.format("There are %d DFAState instances.\n", states);
|
||||
}
|
||||
|
||||
int localDfaCount = 0;
|
||||
int globalDfaCount = 0;
|
||||
int localConfigCount = 0;
|
||||
int globalConfigCount = 0;
|
||||
int[] contextsInDFAState = new int[0];
|
||||
|
||||
for (int i = 0; i < decisionToDFA.length; i++) {
|
||||
DFA dfa = decisionToDFA[i];
|
||||
if (dfa == null || dfa.states == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (SHOW_CONFIG_STATS) {
|
||||
for (DFAState state : dfa.states.keySet()) {
|
||||
if (state.configset.size() >= contextsInDFAState.length) {
|
||||
contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configset.size() + 1);
|
||||
}
|
||||
|
||||
if (state.isAcceptState) {
|
||||
boolean hasGlobal = false;
|
||||
for (ATNConfig config : state.configset) {
|
||||
if (config.reachesIntoOuterContext > 0) {
|
||||
globalConfigCount++;
|
||||
hasGlobal = true;
|
||||
} else {
|
||||
localConfigCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (hasGlobal) {
|
||||
globalDfaCount++;
|
||||
} else {
|
||||
localDfaCount++;
|
||||
}
|
||||
}
|
||||
|
||||
contextsInDFAState[state.configset.size()]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (SHOW_CONFIG_STATS && currentPass == 0) {
|
||||
System.out.format(" DFA accept states: %d total, %d with only local context, %d with a global context\n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
|
||||
System.out.format(" Config stats: %d total, %d local, %d global\n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
|
||||
if (SHOW_DFA_STATE_STATS) {
|
||||
for (int i = 0; i < contextsInDFAState.length; i++) {
|
||||
if (contextsInDFAState[i] != 0) {
|
||||
System.out.format(" %d configs = %d\n", i, contextsInDFAState[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void compileParser(boolean leftRecursive) throws IOException {
|
||||
|
|
@ -187,8 +371,15 @@ public class TestPerformance extends BaseTest {
|
|||
String sourceName = leftRecursive ? "Java-LR.g" : "Java.g";
|
||||
String body = load(sourceName, null);
|
||||
@SuppressWarnings({"ConstantConditions"})
|
||||
String[] extraOptions = FORCE_ATN ? new String[] {"-Xforceatn"} : new String[0];
|
||||
boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptions);
|
||||
List<String> extraOptions = new ArrayList<String>();
|
||||
if (FORCE_ATN) {
|
||||
extraOptions.add("-Xforce-atn");
|
||||
}
|
||||
if (EXPORT_ATN_GRAPHS) {
|
||||
extraOptions.add("-atn");
|
||||
}
|
||||
String[] extraOptionsArray = extraOptions.toArray(new String[extraOptions.size()]);
|
||||
boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptionsArray);
|
||||
assertTrue(success);
|
||||
}
|
||||
|
||||
|
|
@ -228,7 +419,7 @@ public class TestPerformance extends BaseTest {
|
|||
final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("JavaParser");
|
||||
@SuppressWarnings({"unchecked"})
|
||||
final Class<? extends ParseTreeListener<Token>> listenerClass = (Class<? extends ParseTreeListener<Token>>)loader.loadClass("BlankJavaListener");
|
||||
this.sharedListener = listenerClass.newInstance();
|
||||
TestPerformance.sharedListener = listenerClass.newInstance();
|
||||
|
||||
final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
|
||||
final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
|
||||
|
|
@ -271,7 +462,7 @@ public class TestPerformance extends BaseTest {
|
|||
|
||||
Method parseMethod = parserClass.getMethod("compilationUnit");
|
||||
Object parseResult = parseMethod.invoke(sharedParser);
|
||||
assert parseResult instanceof ParseTree;
|
||||
Assert.assertTrue(parseResult instanceof ParseTree);
|
||||
|
||||
if (BUILD_PARSE_TREES && BLANK_LISTENER) {
|
||||
ParseTreeWalker.DEFAULT.walk(sharedListener, (ParseTree)parseResult);
|
||||
|
|
|
|||
Loading…
Reference in New Issue