jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge branch 'main'

This commit is contained in:
Terence Parr 2012-09-08 12:48:57 -07:00
parent 1155c40fc8 1f529e45b1
commit 36cdba494f
118 changed files with 5796 additions and 2473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

238
runtime/Java/doxyfile Normal file
View File

@ -0,0 +1,238 @@
# Doxyfile 1.5.2
#---------------------------------------------------------------------------
# Project related configuration options
#---------------------------------------------------------------------------
DOXYFILE_ENCODING = UTF-8
PROJECT_NAME = "ANTLR v4 API"
PROJECT_NUMBER = 4.0
OUTPUT_DIRECTORY = api
CREATE_SUBDIRS = NO
OUTPUT_LANGUAGE = English
BRIEF_MEMBER_DESC = YES
REPEAT_BRIEF = YES
ABBREVIATE_BRIEF = "The $name class" \
"The $name widget" \
"The $name file" \
is \
provides \
specifies \
contains \
represents \
a \
an \
the
ALWAYS_DETAILED_SEC = YES
INLINE_INHERITED_MEMB = NO
FULL_PATH_NAMES = YES
STRIP_FROM_PATH = /Applications/
STRIP_FROM_INC_PATH =
SHORT_NAMES = NO
JAVADOC_AUTOBRIEF = NO
MULTILINE_CPP_IS_BRIEF = NO
DETAILS_AT_TOP = NO
INHERIT_DOCS = YES
SEPARATE_MEMBER_PAGES = NO
TAB_SIZE = 8
ALIASES =
OPTIMIZE_OUTPUT_FOR_C = NO
OPTIMIZE_OUTPUT_JAVA = YES
BUILTIN_STL_SUPPORT = NO
CPP_CLI_SUPPORT = NO
DISTRIBUTE_GROUP_DOC = NO
SUBGROUPING = YES
#---------------------------------------------------------------------------
# Build related configuration options
#---------------------------------------------------------------------------
EXTRACT_ALL = YES
EXTRACT_PRIVATE = YES
EXTRACT_STATIC = YES
EXTRACT_LOCAL_CLASSES = YES
EXTRACT_LOCAL_METHODS = NO
HIDE_UNDOC_MEMBERS = NO
HIDE_UNDOC_CLASSES = NO
HIDE_FRIEND_COMPOUNDS = NO
HIDE_IN_BODY_DOCS = NO
INTERNAL_DOCS = NO
CASE_SENSE_NAMES = NO
HIDE_SCOPE_NAMES = NO
SHOW_INCLUDE_FILES = YES
INLINE_INFO = YES
SORT_MEMBER_DOCS = YES
SORT_BRIEF_DOCS = NO
SORT_BY_SCOPE_NAME = NO
GENERATE_TODOLIST = YES
GENERATE_TESTLIST = NO
GENERATE_BUGLIST = NO
GENERATE_DEPRECATEDLIST= NO
ENABLED_SECTIONS =
MAX_INITIALIZER_LINES = 30
SHOW_USED_FILES = YES
SHOW_DIRECTORIES = NO
FILE_VERSION_FILTER =
#---------------------------------------------------------------------------
# configuration options related to warning and progress messages
#---------------------------------------------------------------------------
QUIET = NO
WARNINGS = YES
WARN_IF_UNDOCUMENTED = YES
WARN_IF_DOC_ERROR = YES
WARN_NO_PARAMDOC = NO
WARN_FORMAT = "$file:$line: $text"
WARN_LOGFILE =
#---------------------------------------------------------------------------
# configuration options related to the input files
#---------------------------------------------------------------------------
INPUT = /Users/parrt/antlr/code/antlr4/runtime/Java/src
INPUT_ENCODING = UTF-8
FILE_PATTERNS = *.java
RECURSIVE = YES
EXCLUDE =
EXCLUDE_SYMLINKS = NO
EXCLUDE_PATTERNS =
EXCLUDE_SYMBOLS = java::util \
java::io
EXAMPLE_PATH =
EXAMPLE_PATTERNS = *
EXAMPLE_RECURSIVE = NO
IMAGE_PATH =
INPUT_FILTER =
FILTER_PATTERNS =
FILTER_SOURCE_FILES = NO
#---------------------------------------------------------------------------
# configuration options related to source browsing
#---------------------------------------------------------------------------
SOURCE_BROWSER = YES
INLINE_SOURCES = NO
STRIP_CODE_COMMENTS = YES
REFERENCED_BY_RELATION = NO
REFERENCES_RELATION = NO
REFERENCES_LINK_SOURCE = YES
USE_HTAGS = NO
VERBATIM_HEADERS = YES
#---------------------------------------------------------------------------
# configuration options related to the alphabetical class index
#---------------------------------------------------------------------------
ALPHABETICAL_INDEX = NO
COLS_IN_ALPHA_INDEX = 5
IGNORE_PREFIX =
#---------------------------------------------------------------------------
# configuration options related to the HTML output
#---------------------------------------------------------------------------
GENERATE_HTML = YES
HTML_OUTPUT = .
HTML_FILE_EXTENSION = .html
HTML_HEADER =
HTML_FOOTER =
HTML_STYLESHEET =
HTML_ALIGN_MEMBERS = YES
GENERATE_HTMLHELP = NO
CHM_FILE =
HHC_LOCATION =
GENERATE_CHI = NO
BINARY_TOC = NO
TOC_EXPAND = NO
DISABLE_INDEX = NO
ENUM_VALUES_PER_LINE = 4
GENERATE_TREEVIEW = NO
TREEVIEW_WIDTH = 250
#---------------------------------------------------------------------------
# configuration options related to the LaTeX output
#---------------------------------------------------------------------------
GENERATE_LATEX = NO
LATEX_OUTPUT = latex
LATEX_CMD_NAME = latex
MAKEINDEX_CMD_NAME = makeindex
COMPACT_LATEX = NO
PAPER_TYPE = a4wide
EXTRA_PACKAGES =
LATEX_HEADER =
PDF_HYPERLINKS = NO
USE_PDFLATEX = YES
LATEX_BATCHMODE = NO
LATEX_HIDE_INDICES = NO
#---------------------------------------------------------------------------
# configuration options related to the RTF output
#---------------------------------------------------------------------------
GENERATE_RTF = NO
RTF_OUTPUT = rtf
COMPACT_RTF = NO
RTF_HYPERLINKS = NO
RTF_STYLESHEET_FILE =
RTF_EXTENSIONS_FILE =
#---------------------------------------------------------------------------
# configuration options related to the man page output
#---------------------------------------------------------------------------
GENERATE_MAN = NO
MAN_OUTPUT = man
MAN_EXTENSION = .3
MAN_LINKS = NO
#---------------------------------------------------------------------------
# configuration options related to the XML output
#---------------------------------------------------------------------------
GENERATE_XML = NO
XML_OUTPUT = xml
XML_SCHEMA =
XML_DTD =
XML_PROGRAMLISTING = YES
#---------------------------------------------------------------------------
# configuration options for the AutoGen Definitions output
#---------------------------------------------------------------------------
GENERATE_AUTOGEN_DEF = NO
#---------------------------------------------------------------------------
# configuration options related to the Perl module output
#---------------------------------------------------------------------------
GENERATE_PERLMOD = NO
PERLMOD_LATEX = NO
PERLMOD_PRETTY = YES
PERLMOD_MAKEVAR_PREFIX =
#---------------------------------------------------------------------------
# Configuration options related to the preprocessor
#---------------------------------------------------------------------------
ENABLE_PREPROCESSING = YES
MACRO_EXPANSION = NO
EXPAND_ONLY_PREDEF = NO
SEARCH_INCLUDES = YES
INCLUDE_PATH =
INCLUDE_FILE_PATTERNS =
PREDEFINED =
EXPAND_AS_DEFINED =
SKIP_FUNCTION_MACROS = YES
#---------------------------------------------------------------------------
# Configuration::additions related to external references
#---------------------------------------------------------------------------
TAGFILES =
GENERATE_TAGFILE =
ALLEXTERNALS = NO
EXTERNAL_GROUPS = YES
PERL_PATH = /usr/bin/perl
#---------------------------------------------------------------------------
# Configuration options related to the dot tool
#---------------------------------------------------------------------------
CLASS_DIAGRAMS = NO
MSCGEN_PATH = /Applications/Doxygen.app/Contents/Resources/
HIDE_UNDOC_RELATIONS = YES
HAVE_DOT = YES
CLASS_GRAPH = YES
COLLABORATION_GRAPH = YES
GROUP_GRAPHS = YES
UML_LOOK = YES
TEMPLATE_RELATIONS = NO
INCLUDE_GRAPH = YES
INCLUDED_BY_GRAPH = YES
CALL_GRAPH = NO
CALLER_GRAPH = NO
GRAPHICAL_HIERARCHY = YES
DIRECTORY_GRAPH = YES
DOT_IMAGE_FORMAT = png
DOT_PATH = /Applications/Doxygen.app/Contents/Resources/
DOTFILE_DIRS =
DOT_GRAPH_MAX_NODES = 50
DOT_TRANSPARENT = NO
DOT_MULTI_TARGETS = NO
GENERATE_LEGEND = YES
DOT_CLEANUP = YES
#---------------------------------------------------------------------------
# Configuration::additions related to the search engine
#---------------------------------------------------------------------------
SEARCHENGINE = NO

22
runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java
View File

@ -30,16 +30,21 @@
package org.antlr.v4.runtime;
/** Bail out of parser at first syntax error. Do this to use it:
* myparser.setErrorHandler(new BailErrorStrategy<Token>());
* <p/>
* {@code myparser.setErrorHandler(new BailErrorStrategy());}
*/
public class BailErrorStrategy extends DefaultErrorStrategy {
/** Instead of recovering from exception e, Re-throw wrote it wrapped
* in a generic RuntimeException so it is not caught by the
* rule function catches. Exception e is the "cause" of the
* RuntimeException.
/** Instead of recovering from exception {@code e}, re-throw it wrapped
* in a generic {@link RuntimeException} so it is not caught by the
* rule function catches. Use {@link RuntimeException#getCause()} to get the
* original {@link RecognitionException}.
*/
@Override
public void recover(Parser recognizer, RecognitionException e) {
for (ParserRuleContext<?> context = recognizer.getContext(); context != null; context = context.getParent()) {
context.exception = e;
}
throw new RuntimeException(e);
}
@ -50,7 +55,12 @@ public class BailErrorStrategy extends DefaultErrorStrategy {
public Token recoverInline(Parser recognizer)
throws RecognitionException
{
throw new RuntimeException(new InputMismatchException(recognizer));
InputMismatchException e = new InputMismatchException(recognizer);
for (ParserRuleContext<?> context = recognizer.getContext(); context != null; context = context.getParent()) {
context.exception = e;
}
throw new RuntimeException(e);
}
/** Make sure we don't attempt to recover from problems in subrules. */

3
runtime/Java/src/org/antlr/v4/runtime/IntStream.java
View File

@ -30,9 +30,6 @@ package org.antlr.v4.runtime;
/** A simple stream of integers used when all I care about is the char
* or token type sequence (such as interpretation).
*
* Do like Java IO and have new BufferedStream(new TokenStream) rather than
* using inheritance?
*/
public interface IntStream {
void consume();

18
runtime/Java/src/org/antlr/v4/runtime/ParseListener.java
View File

@ -1,18 +0,0 @@
package org.antlr.v4.runtime;
/** We must distinguish between listeners triggered during the parse
* from listeners triggered during a subsequent tree walk. During
* the parse, the ctx object arg for enter methods don't have any labels set.
* We can only access the general ParserRuleContext<Symbol> ctx.
* Also, we can only call exit methods for left-recursive rules. Let's
* make the interface clear these semantics up. If you need the ctx,
* use Parser.getRuleContext().
*/
public interface ParseListener<Symbol extends Token> {
void visitTerminal(ParserRuleContext<Symbol> parent, Symbol token);
/** Enter all but left-recursive rules */
void enterNonLRRule(ParserRuleContext<Symbol> ctx);
void exitEveryRule(ParserRuleContext<Symbol> ctx);
}

152
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -36,41 +36,52 @@ import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.util.ArrayList;
import java.util.List;
/** This is all the parsing support code essentially; most of it is error recovery stuff. */
public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>> {
public class TraceListener implements ParseListener<Token> {
public class TraceListener implements ParseTreeListener<Token> {
@Override
public void enterNonLRRule(ParserRuleContext<Token> ctx) {
System.out.println("enter " + getRuleNames()[ctx.getRuleIndex()] + ", LT(1)=" + _input.LT(1).getText());
public void enterEveryRule(ParserRuleContext<Token> ctx) {
System.out.println("enter " + getRuleNames()[ctx.getRuleIndex()] +
", LT(1)=" + _input.LT(1).getText());
}
@Override
public void visitTerminal(TerminalNode<Token> node) {
System.out.println("consume "+node.getSymbol()+" rule "+
getRuleNames()[_ctx.getRuleIndex()]+
" alt="+_ctx.altNum);
}
@Override
public void visitErrorNode(ErrorNode<Token> node) {
}
@Override
public void exitEveryRule(ParserRuleContext<Token> ctx) {
System.out.println("exit "+getRuleNames()[ctx.getRuleIndex()]+", LT(1)="+_input.LT(1).getText());
}
@Override
public void visitTerminal(ParserRuleContext<Token> parent, Token token) {
System.out.println("consume "+token+" rule "+
getRuleNames()[parent.getRuleIndex()]+
" alt="+parent.altNum);
System.out.println("exit "+getRuleNames()[ctx.getRuleIndex()]+
", LT(1)="+_input.LT(1).getText());
}
}
public static class TrimToSizeListener implements ParseListener<Token> {
public static class TrimToSizeListener implements ParseTreeListener<Token> {
public static final TrimToSizeListener INSTANCE = new TrimToSizeListener();
@Override
public void visitTerminal(ParserRuleContext<Token> parent, Token token) {
}
public void enterEveryRule(ParserRuleContext<Token> ctx) { }
@Override
public void enterNonLRRule(ParserRuleContext<Token> ctx) {
}
public void visitTerminal(TerminalNode<Token> node) { }
@Override
public void visitErrorNode(ErrorNode<Token> node) { }
@Override
public void exitEveryRule(ParserRuleContext<Token> ctx) {
@ -78,7 +89,6 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
((ArrayList<?>)ctx.children).trimToSize();
}
}
}
protected ANTLRErrorStrategy _errHandler = new DefaultErrorStrategy();
@ -100,9 +110,11 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
* *during* the parse. This is typically done only when not building
* parse trees for later visiting. We either trigger events during
* the parse or during tree walks later. Both could be done.
* Not intended for tree parsing but would work.
* Not intended for average user!!! Most people should use
* ParseTreeListener with ParseTreeWalker.
* @see ParseTreeWalker
*/
protected List<ParseListener<Token>> _parseListeners;
protected List<ParseTreeListener<Token>> _parseListeners;
/** Did the recognizer encounter a syntax error? Track how many. */
protected int _syntaxErrors = 0;
@ -191,7 +203,6 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
}
/**
*
* @return {@code true} if the {@link ParserRuleContext#children} list is trimmed
* using the default {@link Parser.TrimToSizeListener} during the parse process.
*/
@ -208,19 +219,28 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
// return traceATNStates;
// }
public List<ParseListener<Token>> getParseListeners() {
public List<ParseTreeListener<Token>> getParseListeners() {
return _parseListeners;
}
public void addParseListener(ParseListener<Token> listener) {
/** Provide a listener that gets notified about token matches,
* and rule entry/exit events DURING the parse. It's a little bit
* weird for left recursive rule entry events but it's
* deterministic.
*
* THIS IS ONLY FOR ADVANCED USERS. Please give your
* ParseTreeListener to a ParseTreeWalker instead of giving it to
* the parser!!!!
*/
public void addParseListener(ParseTreeListener<Token> listener) {
if ( listener==null ) return;
if ( _parseListeners==null ) {
_parseListeners = new ArrayList<ParseListener<Token>>();
_parseListeners = new ArrayList<ParseTreeListener<Token>>();
}
this._parseListeners.add(listener);
}
public void removeParseListener(ParseListener<Token> l) {
public void removeParseListener(ParseTreeListener<Token> l) {
if ( l==null ) return;
if ( _parseListeners!=null ) {
_parseListeners.remove(l);
@ -234,17 +254,27 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
_parseListeners = null;
}
/** Notify any parse listeners (implemented as ParseTreeListener's)
* of an enter rule event. This is not involved with
* parse tree walking in any way; it's just reusing the
* ParseTreeListener interface. This is not for the average user.
*/
public void triggerEnterRuleEvent() {
for (ParseListener<Token> l : _parseListeners) {
l.enterNonLRRule(_ctx);
for (ParseTreeListener<Token> l : _parseListeners) {
l.enterEveryRule(_ctx);
_ctx.enterRule(l);
}
}
/** Notify any parse listeners (implemented as ParseTreeListener's)
* of an exit rule event. This is not involved with
* parse tree walking in any way; it's just reusing the
* ParseTreeListener interface. This is not for the average user.
*/
public void triggerExitRuleEvent() {
// reverse order walk of listeners
for (int i = _parseListeners.size()-1; i >= 0; i--) {
ParseListener<Token> l = _parseListeners.get(i);
ParseTreeListener<Token> l = _parseListeners.get(i);
_ctx.exitRule(l);
l.exitEveryRule(_ctx);
}
@ -311,10 +341,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
{
int line = -1;
int charPositionInLine = -1;
if (offendingToken instanceof Token) {
line = ((Token) offendingToken).getLine();
charPositionInLine = ((Token) offendingToken).getCharPositionInLine();
}
line = offendingToken.getLine();
charPositionInLine = offendingToken.getCharPositionInLine();
ANTLRErrorListener listener = getErrorListenerDispatch();
listener.syntaxError(this, offendingToken, line, charPositionInLine, msg, e);
@ -336,16 +364,19 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
public Token consume() {
Token o = getCurrentToken();
getInputStream().consume();
TerminalNode tnode = null;
if (_buildParseTrees) {
// TODO: tree parsers?
if ( _errHandler.inErrorRecoveryMode(this) ) {
// System.out.println("consume in error recovery mode for "+o);
_ctx.addErrorNode(o);
tnode = _ctx.addErrorNode(o);
}
else {
tnode = _ctx.addChild(o);
}
else _ctx.addChild(o);
}
if ( _parseListeners != null) {
for (ParseListener<Token> l : _parseListeners) l.visitTerminal(_ctx, o);
for (ParseTreeListener<Token> l : _parseListeners) {
l.visitTerminal(tnode);
}
}
return o;
}
@ -391,10 +422,13 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
_ctx.altNum = altNum;
}
/* like enterRule but for recursive rules; no enter events for recursive rules. */
/* like enterRule but for recursive rules */
public void pushNewRecursionContext(ParserRuleContext<Token> localctx, int ruleIndex) {
_ctx = localctx;
_ctx.start = _input.LT(1);
if ( _parseListeners != null ) {
triggerEnterRuleEvent(); // simulates rule entry for left-recursive rules
}
}
public void unrollRecursionContexts(ParserRuleContext<Token> _parentctx) {
@ -534,27 +568,31 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
}
/** For debugging and other purposes */
public List<String> getDFAStrings() {
List<String> s = new ArrayList<String>();
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
s.add( dfa.toString(getTokenNames()) );
}
return s;
public List<String> getDFAStrings() {
synchronized (_interp.decisionToDFA) {
List<String> s = new ArrayList<String>();
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
s.add( dfa.toString(getTokenNames()) );
}
return s;
}
}
/** For debugging and other purposes */
public void dumpDFA() {
boolean seenOne = false;
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
if ( dfa!=null ) {
if ( seenOne ) System.out.println();
System.out.println("Decision " + dfa.decision + ":");
System.out.print(dfa.toString(getTokenNames()));
seenOne = true;
}
}
/** For debugging and other purposes */
public void dumpDFA() {
synchronized (_interp.decisionToDFA) {
boolean seenOne = false;
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
if ( dfa!=null ) {
if ( seenOne ) System.out.println();
System.out.println("Decision " + dfa.decision + ":");
System.out.print(dfa.toString(getTokenNames()));
seenOne = true;
}
}
}
}
public String getSourceName() {
@ -586,7 +624,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
// if ( traceATNStates ) _ctx.trace(atnState);
}
/** During a parse is extremely useful to listen in on the rule entry and exit
/** During a parse is sometimes useful to listen in on the rule entry and exit
* events as well as token matches. This is for quick and dirty debugging.
*/
public void setTrace(boolean trace) {

47
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
View File

@ -28,10 +28,7 @@
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.ErrorNodeImpl;
@ -44,19 +41,19 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/** A rule invocation record for parsing and tree parsing.
/** A rule invocation record for parsing.
*
* Contains all of the information about the current rule not stored in the
* RuleContext. It handles parse tree children list, Any ATN state
* tracing, and the default values available for rule indications:
* start, stop, ST, rule index, current alt number, current
* start, stop, rule index, current alt number, current
* ATN state.
*
* Subclasses made for each rule and grammar track the parameters,
* return values, locals, and labels specific to that rule. These
* are the objects that are returned from rules.
*
* Note text is not an actual property of the return value, it is computed
* Note text is not an actual field of a rule return value; it is computed
* from start and stop using the input stream's toString() method. I
* could add a ctor to this so that we can pass in and store the input
* stream, but I'm not sure we want to do that. It would seem to be undefined
@ -68,11 +65,9 @@ import java.util.List;
* satisfy the superclass interface.
*/
public class ParserRuleContext<Symbol extends Token> extends RuleContext {
public static final ParserRuleContext<Token> EMPTY = new ParserRuleContext<Token>();
/** If we are debugging or building a parse tree for a visitor,
* we need to track all of the tokens and rule invocations associated
* with this rule's context. This is empty for normal parsing
* with this rule's context. This is empty for parsing w/o tree constr.
* operation because we don't the need to track the details about
* how we parse this rule.
*/
@ -140,23 +135,20 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
// Double dispatch methods for listeners
// parse listener
public void enterRule(ParseListener<Symbol> listener) { }
public void exitRule(ParseListener<Symbol> listener) { }
// parse tree listener
public void enterRule(ParseTreeListener<Symbol> listener) { }
public void exitRule(ParseTreeListener<Symbol> listener) { }
/** Does not set parent link; other add methods do */
public void addChild(TerminalNode<Symbol> t) {
/** Does not set parent link; other add methods do that */
public TerminalNode addChild(TerminalNode<Symbol> t) {
if ( children==null ) children = new ArrayList<ParseTree>();
children.add(t);
return t;
}
public void addChild(RuleContext ruleInvocation) {
public RuleContext addChild(RuleContext ruleInvocation) {
if ( children==null ) children = new ArrayList<ParseTree>();
children.add(ruleInvocation);
return ruleInvocation;
}
/** Used by enterOuterAlt to toss out a RuleContext previously added as
@ -174,10 +166,11 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
// states.add(s);
// }
public void addChild(Symbol matchedToken) {
public TerminalNode addChild(Symbol matchedToken) {
TerminalNodeImpl<Symbol> t = new TerminalNodeImpl<Symbol>(matchedToken);
addChild(t);
t.parent = this;
return t;
}
public ErrorNode<Symbol> addErrorNode(Symbol badToken) {
@ -304,24 +297,6 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
public Symbol getStart() { return start; }
public Symbol getStop() { return stop; }
@Override
public String toString(@NotNull Recognizer<?,?> recog, RuleContext stop) {
if ( recog==null ) return super.toString(recog, stop);
StringBuilder buf = new StringBuilder();
ParserRuleContext<?> p = this;
buf.append("[");
while ( p != null && p != stop ) {
ATN atn = recog.getATN();
ATNState s = atn.states.get(p.s);
String ruleName = recog.getRuleNames()[s.ruleIndex];
buf.append(ruleName);
if ( p.parent != null ) buf.append(" ");
p = (ParserRuleContext<?>)p.parent;
}
buf.append("]");
return buf.toString();
}
/** Used for rule context info debugging during parse-time, not so much for ATN debugging */
public String toInfoString(Parser recognizer) {
List<String> rules = recognizer.getRuleInvocationStack(this);

3
runtime/Java/src/org/antlr/v4/runtime/Recognizer.java
View File

@ -34,7 +34,6 @@ import org.antlr.v4.runtime.atn.ATNSimulator;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
@ -119,7 +118,7 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
@NotNull
public List<? extends ANTLRErrorListener> getErrorListeners() {
return new ArrayList<ANTLRErrorListener>(_listeners);
return _listeners;
}
public ANTLRErrorListener getErrorListenerDispatch() {

62
runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
View File

@ -38,6 +38,8 @@ import org.antlr.v4.runtime.tree.gui.TreeViewer;
import javax.print.PrintException;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
/** A rule context is a record of a single rule invocation. It knows
* which context invoked it, if any. If there is no parent context, then
@ -53,13 +55,15 @@ import java.io.IOException;
* The parent contexts are useful for computing lookahead sets and
* getting error information.
*
* These objects are used during lexing, parsing, and prediction.
* These objects are used during parsing and prediction.
* For the special case of parsers and tree parsers, we use the subclass
* ParserRuleContext.
*
* @see ParserRuleContext
*/
public class RuleContext implements RuleNode {
public static final ParserRuleContext<Token> EMPTY = new ParserRuleContext<Token>();
/** What context invoked this rule? */
public RuleContext parent;
@ -182,8 +186,8 @@ public class RuleContext implements RuleNode {
* that they are now going to track perfectly together. Once they
* converged on state 21, there is no way they can separate. In other
* words, the prior stack state is not consulted when computing where to
* go in the closure operation. ?$ and ??$ are considered the same stack.
* If ? is popped off then $ and ?$ remain; they are now an empty and
* go in the closure operation. x$ and xy$ are considered the same stack.
* If x is popped off then $ and y$ remain; they are now an empty and
* nonempty context comparison. So, if one stack is a suffix of
* another, then it will still degenerate to the simple empty stack
* comparison case.
@ -261,11 +265,13 @@ public class RuleContext implements RuleNode {
@Override
public <T> T accept(ParseTreeVisitor<? extends T> visitor) { return visitor.visitChildren(this); }
/** Call this method to view a parse tree in a dialog box visually. */
public void inspect(Parser parser) {
TreeViewer viewer = new TreeViewer(parser, this);
viewer.open();
}
/** Save this tree in a postscript file */
public void save(Parser parser, String fileName)
throws IOException, PrintException
{
@ -274,6 +280,7 @@ public class RuleContext implements RuleNode {
Trees.writePS(this, parser, fileName); // parrt routine
}
/** Save this tree in a postscript file using a particular font name and size */
public void save(Parser parser, String fileName,
String fontName, int fontSize)
throws IOException
@ -285,32 +292,65 @@ public class RuleContext implements RuleNode {
* (root child1 .. childN). Print just a node if this is a leaf.
* We have to know the recognizer so we can get rule names.
*/
public String toStringTree(Parser recog) {
public String toStringTree(@Nullable Parser recog) {
return Trees.toStringTree(this, recog);
}
/** Print out a whole tree, not just a node, in LISP format
* (root child1 .. childN). Print just a node if this is a leaf.
*/
public String toStringTree(@Nullable List<String> ruleNames) {
return Trees.toStringTree(this, ruleNames);
}
@Override
public String toStringTree() { return toStringTree(null); }
public String toStringTree() {
return toStringTree((List<String>)null);
}
@Override
public String toString() {
return toString(null);
return toString((List<String>)null, (RuleContext)null);
}
public String toString(@Nullable Recognizer<?,?> recog) {
public final String toString(@Nullable Recognizer<?,?> recog) {
return toString(recog, ParserRuleContext.EMPTY);
}
public final String toString(@Nullable List<String> ruleNames) {
return toString(ruleNames, null);
}
// recog null unless ParserRuleContext, in which case we use subclass toString(...)
public String toString(@Nullable Recognizer<?,?> recog, RuleContext stop) {
public String toString(@Nullable Recognizer<?,?> recog, @Nullable RuleContext stop) {
String[] ruleNames = recog != null ? recog.getRuleNames() : null;
List<String> ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null;
return toString(ruleNamesList, stop);
}
public String toString(@Nullable List<String> ruleNames, @Nullable RuleContext stop) {
StringBuilder buf = new StringBuilder();
RuleContext p = this;
buf.append("[");
while ( p != null && p != stop ) {
if ( !p.isEmpty() ) buf.append(p.invokingState);
if ( p.parent != null && !p.parent.isEmpty() ) buf.append(" ");
while (p != null && p != stop) {
if (ruleNames == null) {
if (!p.isEmpty()) {
buf.append(p.invokingState);
}
}
else {
int ruleIndex = p.getRuleIndex();
String ruleName = ruleIndex >= 0 && ruleIndex < ruleNames.size() ? ruleNames.get(ruleIndex) : Integer.toString(ruleIndex);
buf.append(ruleName);
}
if (p.parent != null && (ruleNames != null || !p.parent.isEmpty())) {
buf.append(" ");
}
p = p.parent;
}
buf.append("]");
return buf.toString();
}

2
runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
View File

@ -35,7 +35,7 @@ import org.antlr.v4.runtime.misc.Interval;
public interface TokenStream extends IntStream {
/** Get Token at current input pointer + i ahead where i=1 is next Token.
* i&lt;0 indicates tokens in the past. So -1 is previous token and -2 is
* two tokens ago. LT(0) is undefined. For i>=n, return Token.EOFToken.
* two tokens ago. LT(0) is undefined. For i>=n, return eof token.
* Return null for LT(0) and any index that results in an absolute address
* that is negative.
* TODO (Sam): Throw exception for invalid k?

1
runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java
View File

@ -71,6 +71,7 @@ public class ATN {
// runtime for lexer only
public int[] ruleToTokenType;
public int[] ruleToActionIndex;
@NotNull
public final List<TokensStartState> modeToStartState = new ArrayList<TokensStartState>();

31
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java
View File

@ -30,17 +30,15 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
/** An ATN state, predicted alt, and syntactic/semantic context.
* The syntactic context is a pointer into the rule invocation
/** A tuple: (ATN state, predicted alt, syntactic, semantic context).
* The syntactic context is a graph-structured stack node whose
* path(s) to the root is the rule invocation(s)
* chain used to arrive at the state. The semantic context is
* the unordered set semantic predicates encountered before reaching
* the tree of semantic predicates encountered before reaching
* an ATN state.
*
* (state, alt, rule context, semantic context)
*/
public class ATNConfig {
/** The ATN state associated with this configuration */
@ -55,7 +53,7 @@ public class ATNConfig {
* execution of the ATN simulator.
*/
@Nullable
public RuleContext context;
public PredictionContext context;
/**
* We cannot execute predicates dependent upon local context unless
@ -70,22 +68,19 @@ public class ATNConfig {
*/
public int reachesIntoOuterContext;
/** Capture lexer action we traverse */
public int lexerActionIndex = -1; // TOOD: move to subclass
@NotNull
public final SemanticContext semanticContext;
public ATNConfig(@NotNull ATNState state,
int alt,
@Nullable RuleContext context)
@Nullable PredictionContext context)
{
this(state, alt, context, SemanticContext.NONE);
}
public ATNConfig(@NotNull ATNState state,
int alt,
@Nullable RuleContext context,
@Nullable PredictionContext context,
@NotNull SemanticContext semanticContext)
{
this.state = state;
@ -102,19 +97,18 @@ public class ATNConfig {
this(c, state, c.context, semanticContext);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context) {
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable PredictionContext context) {
this(c, state, context, c.semanticContext);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context,
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable PredictionContext context,
@NotNull SemanticContext semanticContext)
{
this.state = state;
this.alt = c.alt;
this.context = context;
this.semanticContext = semanticContext;
this.reachesIntoOuterContext = c.reachesIntoOuterContext;
this.semanticContext = semanticContext;
this.lexerActionIndex = c.lexerActionIndex;
}
/** An ATN configuration is equal to another if both have
@ -171,8 +165,9 @@ public class ATNConfig {
buf.append(alt);
}
if ( context!=null ) {
buf.append(",");
buf.append(context.toString(recog));
buf.append(",[");
buf.append(context.toString());
buf.append("]");
}
if ( semanticContext!=null && semanticContext != SemanticContext.NONE ) {
buf.append(",");

429
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java
View File

@ -29,28 +29,277 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.Array2DHashSet;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/** Specialized OrderedHashSet that can track info about the set.
* Might be able to optimize later w/o affecting code that uses this set.
histogram of lexer DFA configset size:
206 30 <- 206 sets with size 30
47 1
17 31
12 2
10 3
7 32
4 4
3 35
2 9
2 6
2 5
2 34
1 7
1 33
1 29
1 12
1 119 <- max size
322 set size for SLL parser java.* in DFA states:
888 1
411 54
365 88
304 56
206 80
182 16
167 86
166 78
158 84
131 2
121 20
120 8
119 112
82 10
73 6
53 174
47 90
45 4
39 12
38 122
37 89
37 62
34 3
34 18
32 81
31 87
28 45
27 144
25 41
24 132
22 91
22 7
21 82
21 28
21 27
17 9
16 29
16 155
15 51
15 118
14 146
14 114
13 5
13 38
12 48
11 64
11 50
11 22
11 134
11 131
10 79
10 76
10 59
10 58
10 55
10 39
10 116
9 74
9 47
9 310
...
javalr, java.* configs with # preds histogram:
4569 0
57 1
27 27
5 76
4 28
3 72
3 38
3 30
2 6
2 32
1 9
1 2
javalr, java.* all atnconfigsets; max size = 322, num sets = 269088
114186 1 <-- optimize
35712 6
28081 78
15252 54
14171 56
13159 12
11810 88
6873 86
6158 80
5169 4
3773 118
2350 16
1002 112
915 28
898 44
734 2
632 62
575 8
566 59
474 20
388 84
343 48
333 55
328 47
311 41
306 38
277 81
263 79
255 66
245 90
245 87
234 50
224 10
220 60
194 64
186 32
184 82
150 18
125 7
121 132
116 30
103 51
95 114
84 36
82 40
78 22
77 89
55 9
53 174
48 152
44 67
44 5
42 115
41 58
38 122
37 134
34 13
34 116
29 45
29 3
29 24
27 144
26 146
25 91
24 113
20 27
...
number with 1-9 elements:
114186 1
35712 6
5169 4
734 2
575 8
125 7
55 9
44 5
29 3
Can cover 60% of sizes with size up to 6
Can cover 44% of sizes with size up to 4
Can cover 42% of sizes with size up to 1
*/
public class ATNConfigSet extends OrderedHashSet<ATNConfig> {
public class ATNConfigSet implements Set<ATNConfig> {
/*
The reason that we need this is because we don't want the hash map to use
the standard hash code and equals. We need all configurations with the same
(s,i,_,semctx) to be equal. Unfortunately, this key effectively doubles
the number of objects associated with ATNConfigs. The other solution is to
use a hash table that lets us specify the equals/hashcode operation.
*/
public static class ConfigHashSet extends Array2DHashSet<ATNConfig> {
public ConfigHashSet() {
super(16,2);
}
@Override
public int hashCode(ATNConfig o) {
int hashCode = 7;
hashCode = 31 * hashCode + o.state.stateNumber;
hashCode = 31 * hashCode + o.alt;
hashCode = 31 * hashCode + o.semanticContext.hashCode();
return hashCode;
}
@Override
public boolean equals(ATNConfig a, ATNConfig b) {
if ( a==b ) return true;
if ( a==null || b==null ) return false;
if ( hashCode(a) != hashCode(b) ) return false;
return a.state.stateNumber==b.state.stateNumber
&& a.alt==b.alt
&& b.semanticContext.equals(b.semanticContext);
}
}
/** Indicates that the set of configurations is read-only. Do not
* allow any code to manipulate the set; DFA states will point at
* the sets and they must not change. This does not protect the other
* fields; in particular, conflictingAlts is set after
* we've made this readonly.
*/
protected boolean readonly = false;
/** All configs but hashed by (s, i, _, pi) not incl context. Wiped out
* when we go readonly as this set becomes a DFA state.
*/
public ConfigHashSet configLookup;
/** Track the elements as they are added to the set; supports get(i) */
public final ArrayList<ATNConfig> configs = new ArrayList<ATNConfig>(7);
// TODO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation
// TODO: can we track conflicts as they are added to save scanning configs later?
public int uniqueAlt;
public IntervalSet conflictingAlts;
protected IntervalSet conflictingAlts;
// Used in parser and lexer. In lexer, it indicates we hit a pred
// while computing a closure operation. Don't make a DFA state from this.
public boolean hasSemanticContext;
public boolean dipsIntoOuterContext;
public ATNConfigSet() { }
/** Indicates that this configuration set is part of a full context
* LL prediction. It will be used to determine how to merge $. With SLL
* it's a wildcard whereas it is not for LL context merge.
*/
public final boolean fullCtx;
public ATNConfigSet(boolean fullCtx) {
configLookup = new ConfigHashSet();
this.fullCtx = fullCtx;
}
public ATNConfigSet() { this(true); }
public ATNConfigSet(ATNConfigSet old) {
this(old.fullCtx);
addAll(old);
this.uniqueAlt = old.uniqueAlt;
this.conflictingAlts = old.conflictingAlts;
@ -58,22 +307,190 @@ public class ATNConfigSet extends OrderedHashSet<ATNConfig> {
this.dipsIntoOuterContext = old.dipsIntoOuterContext;
}
@Override
public boolean add(ATNConfig config) {
return add(config, null);
}
/** Adding a new config means merging contexts with existing configs for
* (s, i, pi, _)
* We use (s,i,pi) as key
*/
public boolean add(
ATNConfig config,
DoubleKeyMap<PredictionContext,PredictionContext,PredictionContext> mergeCache)
{
if ( readonly ) throw new IllegalStateException("This set is readonly");
if ( config.semanticContext!=SemanticContext.NONE ) {
hasSemanticContext = true;
}
ATNConfig existing = configLookup.absorb(config);
if ( existing==config ) { // we added this new one
configs.add(config); // track order here
return true;
}
// a previous (s,i,pi,_), merge with it and save result
boolean rootIsWildcard = !fullCtx;
PredictionContext merged =
PredictionContext.merge(existing.context, config.context, rootIsWildcard, mergeCache);
// no need to check for existing.context, config.context in cache
// since only way to create new graphs is "call rule" and here. We
// cache at both places.
existing.reachesIntoOuterContext =
Math.max(existing.reachesIntoOuterContext, config.reachesIntoOuterContext);
existing.context = merged; // replace context; no need to alt mapping
return true;
}
/** Return a List holding list of configs */
public List<ATNConfig> elements() { return configs; }
public Set<ATNState> getStates() {
Set<ATNState> states = new HashSet<ATNState>();
for (ATNConfig c : this.elements) {
for (ATNConfig c : configs) {
states.add(c.state);
}
return states;
}
public List<SemanticContext> getPredicates() {
List<SemanticContext> preds = new ArrayList<SemanticContext>();
for (ATNConfig c : configs) {
if ( c.semanticContext!=SemanticContext.NONE ) {
preds.add(c.semanticContext);
}
}
return preds;
}
public ATNConfig get(int i) { return configs.get(i); }
// TODO: very expensive, used in lexer to kill after wildcard config
public void remove(int i) {
if ( readonly ) throw new IllegalStateException("This set is readonly");
ATNConfig c = elements().get(i);
configLookup.remove(c);
configs.remove(c); // slow linear search. ugh but not worse than it was
}
public void optimizeConfigs(ATNSimulator interpreter) {
if ( readonly ) throw new IllegalStateException("This set is readonly");
if ( configLookup.isEmpty() ) return;
for (ATNConfig config : configs) {
// int before = PredictionContext.getAllContextNodes(config.context).size();
config.context = interpreter.getCachedContext(config.context);
// int after = PredictionContext.getAllContextNodes(config.context).size();
// System.out.println("configs "+before+"->"+after);
}
}
public boolean addAll(Collection<? extends ATNConfig> coll) {
for (ATNConfig c : coll) add(c);
return false;
}
@Override
public boolean equals(Object o) {
// System.out.print("equals " + this + ", " + o+" = ");
ATNConfigSet other = (ATNConfigSet)o;
boolean same = configs!=null &&
configs.equals(other.configs) && // includes stack context
this.fullCtx == other.fullCtx &&
this.uniqueAlt == other.uniqueAlt &&
this.conflictingAlts == other.conflictingAlts &&
this.hasSemanticContext == other.hasSemanticContext &&
this.dipsIntoOuterContext == other.dipsIntoOuterContext;
// System.out.println(same);
return same;
}
@Override
public int hashCode() {
return configs.hashCode();
}
@Override
public int size() {
return configs.size();
}
@Override
public boolean isEmpty() {
return configs.isEmpty();
}
@Override
public boolean contains(Object o) {
if ( o instanceof ATNConfig ) {
return configLookup.contains(o);
}
return false;
}
@Override
public Iterator<ATNConfig> iterator() {
return configs.iterator();
}
@Override
public void clear() {
if ( readonly ) throw new IllegalStateException("This set is readonly");
configs.clear();
configLookup.clear();
}
public void setReadonly(boolean readonly) {
this.readonly = readonly;
configLookup = null; // can't mod, no need for lookup cache
}
@Override
public String toString() {
StringBuilder buf = new StringBuilder();
buf.append(super.toString());
buf.append(elements().toString());
if ( hasSemanticContext ) buf.append(",hasSemanticContext=").append(hasSemanticContext);
if ( uniqueAlt!=ATN.INVALID_ALT_NUMBER ) buf.append(",uniqueAlt=").append(uniqueAlt);
if ( conflictingAlts!=null ) buf.append(",conflictingAlts=").append(conflictingAlts);
if ( dipsIntoOuterContext ) buf.append(",dipsIntoOuterContext");
return buf.toString();
}
// satisfy interface
@Override
public Object[] toArray() {
ATNConfig[] configs = new ATNConfig[configLookup.size()];
int i = 0;
for (ATNConfig c : configLookup) configs[i++] = c;
return configs;
}
@Override
public <T> T[] toArray(T[] a) {
int i = 0;
for (ATNConfig c : configLookup) a[i++] = (T)c;
return a;
}
@Override
public boolean remove(Object o) {
throw new UnsupportedOperationException();
}
@Override
public boolean containsAll(Collection<?> c) {
throw new UnsupportedOperationException();
}
@Override
public boolean retainAll(Collection<?> c) {
throw new UnsupportedOperationException();
}
@Override
public boolean removeAll(Collection<?> c) {
throw new UnsupportedOperationException();
}
}

57
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -35,6 +35,7 @@ import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Pair;
import java.util.ArrayList;
import java.util.IdentityHashMap;
import java.util.List;
public abstract class ATNSimulator {
@ -44,17 +45,52 @@ public abstract class ATNSimulator {
@NotNull
public final ATN atn;
/** The context cache maps all PredictionContext objects that are equals()
* to a single cached copy. This cache is shared across all contexts
* in all ATNConfigs in all DFA states. We rebuild each ATNConfigSet
* to use only cached nodes/graphs in addDFAState(). We don't want to
* fill this during closure() since there are lots of contexts that
* pop up but are not used ever again. It also greatly slows down closure().
*
* This cache makes a huge difference in memory and a little bit in speed.
* For the Java grammar on java.*, it dropped the memory requirements
* at the end from 25M to 16M. We don't store any of the full context
* graphs in the DFA because they are limited to local context only,
* but apparently there's a lot of repetition there as well. We optimize
* the config contexts before storing the config set in the DFA states
* by literally rebuilding them with cached subgraphs only.
*
* I tried a cache for use during closure operations, that was
* whacked after each adaptivePredict(). It cost a little bit
* more time I think and doesn't save on the overall footprint
* so it's not worth the complexity.
*/
protected final PredictionContextCache sharedContextCache;
static {
ERROR = new DFAState(new ATNConfigSet());
ERROR.stateNumber = Integer.MAX_VALUE;
}
public ATNSimulator(@NotNull ATN atn) {
public ATNSimulator(@NotNull ATN atn,
@NotNull PredictionContextCache sharedContextCache)
{
this.atn = atn;
this.sharedContextCache = sharedContextCache;
}
public abstract void reset();
public PredictionContext getCachedContext(PredictionContext context) {
if ( sharedContextCache==null ) return context;
IdentityHashMap<PredictionContext, PredictionContext> visited =
new IdentityHashMap<PredictionContext, PredictionContext>();
return PredictionContext.getCachedContext(context,
sharedContextCache,
visited);
}
public static ATN deserialize(@NotNull char[] data) {
ATN atn = new ATN();
List<IntervalSet> sets = new ArrayList<IntervalSet>();
@ -174,17 +210,16 @@ public abstract class ATNSimulator {
p += 6;
}
if (atn.grammarType == ATN.LEXER) {
for (ATNState state : atn.states) {
for (int i = 0; i < state.getNumberOfTransitions(); i++) {
Transition t = state.transition(i);
if (!(t instanceof RuleTransition)) {
continue;
}
RuleTransition ruleTransition = (RuleTransition)t;
atn.ruleToStopState[ruleTransition.target.ruleIndex].addTransition(new EpsilonTransition(ruleTransition.followState));
// edges for rule stop states can be derived, so they aren't serialized
for (ATNState state : atn.states) {
for (int i = 0; i < state.getNumberOfTransitions(); i++) {
Transition t = state.transition(i);
if (!(t instanceof RuleTransition)) {
continue;
}
RuleTransition ruleTransition = (RuleTransition)t;
atn.ruleToStopState[ruleTransition.target.ruleIndex].addTransition(new EpsilonTransition(ruleTransition.followState));
}
}

7
runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java
View File

@ -31,7 +31,12 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.IntervalSet;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ATNState {
public static final int INITIAL_NUM_TRANSITIONS = 4;

194
runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java Normal file
View File

@ -0,0 +1,194 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import java.util.Arrays;
import java.util.Iterator;
public class ArrayPredictionContext extends PredictionContext {
/** Parent can be null only if full ctx mode and we make an array
* from EMPTY and non-empty. We merge EMPTY by using null parent and
* invokingState == EMPTY_FULL_INVOKING_STATE
*/
public final PredictionContext[] parents;
/** Sorted for merge, no duplicates; if present,
* EMPTY_FULL_INVOKING_STATE is always first
*/
public final int[] invokingStates;
public ArrayPredictionContext(SingletonPredictionContext a) {
this(new PredictionContext[] {a.parent},
new int[] {a==PredictionContext.EMPTY ? EMPTY_FULL_CTX_INVOKING_STATE : a.invokingState});
}
public ArrayPredictionContext(PredictionContext[] parents, int[] invokingStates) {
super(calculateHashCode(parents, invokingStates));
assert parents!=null && parents.length>0;
assert invokingStates!=null && invokingStates.length>0;
// System.err.println("CREATE ARRAY: "+Arrays.toString(parents)+", "+Arrays.toString(invokingStates));
this.parents = parents;
this.invokingStates = invokingStates;
}
//ArrayPredictionContext(@NotNull PredictionContext[] parents, int[] invokingStates, int parentHashCode, int invokingStateHashCode) {
// super(calculateHashCode(parentHashCode, invokingStateHashCode));
// assert parents.length == invokingStates.length;
// assert invokingStates.length > 1 || invokingStates[0] != EMPTY_FULL_STATE_KEY : "Should be using PredictionContext.EMPTY instead.";
//
// this.parents = parents;
// this.invokingStates = invokingStates;
// }
//
//ArrayPredictionContext(@NotNull PredictionContext[] parents, int[] invokingStates, int hashCode) {
// super(hashCode);
// assert parents.length == invokingStates.length;
// assert invokingStates.length > 1 || invokingStates[0] != EMPTY_FULL_STATE_KEY : "Should be using PredictionContext.EMPTY instead.";
//
// this.parents = parents;
// this.invokingStates = invokingStates;
// }
protected static int calculateHashCode(PredictionContext[] parents, int[] invokingStates) {
return calculateHashCode(calculateParentHashCode(parents),
calculateInvokingStatesHashCode(invokingStates));
}
protected static int calculateParentHashCode(PredictionContext[] parents) {
int hashCode = 1;
for (PredictionContext p : parents) {
if ( p!=null ) { // can be null for full ctx stack in ArrayPredictionContext
hashCode = hashCode * 31 ^ p.hashCode();
}
}
return hashCode;
}
protected static int calculateInvokingStatesHashCode(int[] invokingStates) {
int hashCode = 1;
for (int state : invokingStates) {
hashCode = hashCode * 31 ^ state;
}
return hashCode;
}
@Override
public Iterator<SingletonPredictionContext> iterator() {
return new Iterator<SingletonPredictionContext>() {
int i = 0;
@Override
public boolean hasNext() { return i < parents.length; }
@Override
public SingletonPredictionContext next() {
SingletonPredictionContext ctx =
new SingletonPredictionContext(parents[i], invokingStates[i]);
i++;
return ctx;
}
@Override
public void remove() { throw new UnsupportedOperationException(); }
};
}
@Override
public boolean isEmpty() {
return size()==1 &&
invokingStates[0]==EmptyPredictionContext.EMPTY_INVOKING_STATE;
}
@Override
public int size() {
return invokingStates.length;
}
@Override
public PredictionContext getParent(int index) {
return parents[index];
}
@Override
public int getInvokingState(int index) {
return invokingStates[index];
}
// @Override
// public int findInvokingState(int invokingState) {
// return Arrays.binarySearch(invokingStates, invokingState);
// }
/** Find invokingState parameter (call it x) in this.invokingStates,
* if present. Call pop on all x's parent(s) and then pull other
* elements from this context and merge into new context.
*/
@Override
public PredictionContext popAll(
int invokingState,
boolean fullCtx,
DoubleKeyMap<PredictionContext,PredictionContext,PredictionContext> mergeCache)
{
int index = Arrays.binarySearch(this.invokingStates, invokingState);
if ( index < 0 ) {
return this;
}
PredictionContext newCtx =
this.parents[index].popAll(invokingState, fullCtx, mergeCache);
for (int i = 0; i < this.invokingStates.length; i++) {
if (i == index) continue;
PredictionContext next;
if ( this.invokingStates[i] == EMPTY_FULL_CTX_INVOKING_STATE ) {
next = PredictionContext.EMPTY;
}
else {
next = new SingletonPredictionContext(this.parents[i],
this.invokingStates[i]);
}
boolean rootIsWildcard = fullCtx;
newCtx = merge(newCtx, next, rootIsWildcard, mergeCache);
}
return newCtx;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
else if ( !(o instanceof ArrayPredictionContext) ) {
return false;
}
if ( this.hashCode() != o.hashCode() ) {
return false; // can't be same if hash is different
}
ArrayPredictionContext a = (ArrayPredictionContext)o;
return Arrays.equals(invokingStates, a.invokingStates) &&
Arrays.equals(parents, a.parents);
}
@Override
public String toString() {
if ( isEmpty() ) return "[]";
StringBuilder buf = new StringBuilder();
buf.append("[");
for (int i=0; i<invokingStates.length; i++) {
if ( i>0 ) buf.append(", ");
buf.append(invokingStates[i]);
if ( parents[i]!=null ) {
buf.append(' ');
buf.append(parents[i].toString());
}
else {
buf.append("null");
}
}
buf.append("]");
return buf.toString();
}
}

2
runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java
View File

@ -29,8 +29,8 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
/** TODO: make all transitions sets? no, should remove set edges */
public final class AtomTransition extends Transition {

35
runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java Normal file
View File

@ -0,0 +1,35 @@
package org.antlr.v4.runtime.atn;
public class EmptyPredictionContext extends SingletonPredictionContext {
public static final int EMPTY_INVOKING_STATE = ATNState.INVALID_STATE_NUMBER - 1;
public EmptyPredictionContext() {
super(null, EMPTY_INVOKING_STATE);
}
public boolean isEmpty() { return true; }
@Override
public int size() {
return 1;
}
@Override
public PredictionContext getParent(int index) {
return null;
}
@Override
public int getInvokingState(int index) {
return invokingState;
}
@Override
public boolean equals(Object o) {
return this == o;
}
@Override
public String toString() {
return "$";
}
}

33
runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java
View File

@ -29,7 +29,6 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.IntervalSet;
@ -63,7 +62,7 @@ public class LL1Analyzer {
Set<ATNConfig> lookBusy = new HashSet<ATNConfig>();
boolean seeThruPreds = false; // fail to get lookahead upon pred
_LOOK(s.transition(alt - 1).target,
ParserRuleContext.EMPTY,
PredictionContext.EMPTY,
look[alt], lookBusy, seeThruPreds);
// Wipe out lookahead for this alternative if we found nothing
// or we had a predicate when we !seeThruPreds
@ -81,18 +80,19 @@ public class LL1Analyzer {
public IntervalSet LOOK(@NotNull ATNState s, @Nullable RuleContext ctx) {
IntervalSet r = new IntervalSet();
boolean seeThruPreds = true; // ignore preds; get all lookahead
_LOOK(s, ctx, r, new HashSet<ATNConfig>(), seeThruPreds);
_LOOK(s, PredictionContext.fromRuleContext(ctx),
r, new HashSet<ATNConfig>(), seeThruPreds);
return r;
}
/** Computer set of tokens that can come next. If the context is EMPTY,
/** Compute set of tokens that can come next. If the context is EMPTY,
* then we don't go anywhere when we hit the end of the rule. We have
* the correct set. If the context is null, that means that we did not want
* any tokens following this rule--just the tokens that could be found within this
* rule. Add EPSILON to the set indicating we reached the end of the ruled out having
* to match a token.
*/
protected void _LOOK(@NotNull ATNState s, @Nullable RuleContext ctx,
protected void _LOOK(@NotNull ATNState s, @Nullable PredictionContext ctx,
@NotNull IntervalSet look,
@NotNull Set<ATNConfig> lookBusy,
boolean seeThruPreds)
@ -106,21 +106,25 @@ public class LL1Analyzer {
look.add(Token.EPSILON);
return;
}
if ( ctx.invokingState!=-1 ) {
ATNState invokingState = atn.states.get(ctx.invokingState);
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
// System.out.println("popping back to "+retState);
_LOOK(retState, ctx.parent, look, lookBusy, seeThruPreds);
return;
}
if ( ctx != PredictionContext.EMPTY ) {
// run thru all possible stack tops in ctx
for (SingletonPredictionContext p : ctx) {
ATNState invokingState = atn.states.get(p.invokingState);
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
// System.out.println("popping back to "+retState);
_LOOK(retState, p.parent, look, lookBusy, seeThruPreds);
}
return;
}
}
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( t.getClass() == RuleTransition.class ) {
RuleContext newContext = new RuleContext(ctx, s.stateNumber);
PredictionContext newContext =
new SingletonPredictionContext(ctx, s.stateNumber);
_LOOK(t.target, newContext, look, lookBusy, seeThruPreds);
}
else if ( t instanceof PredicateTransition ) {
@ -149,5 +153,4 @@ public class LL1Analyzer {
}
}
}
}

41
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java Normal file
View File

@ -0,0 +1,41 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
public class LexerATNConfig extends ATNConfig {
/** Capture lexer action we traverse */
public int lexerActionIndex = -1; // TOOD: move to subclass
public LexerATNConfig(@NotNull ATNState state,
int alt,
@Nullable PredictionContext context)
{
super(state, alt, context, SemanticContext.NONE);
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state) {
super(c, state, c.context, c.semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@NotNull SemanticContext semanticContext) {
super(c, state, c.context, semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
int actionIndex)
{
super(c, state, c.context, c.semanticContext);
this.lexerActionIndex = actionIndex;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@Nullable PredictionContext context) {
super(c, state, context, c.semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
}
}

265
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -33,7 +33,6 @@ import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.LexerNoViableAltException;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
@ -46,8 +45,6 @@ import java.io.OutputStream;
/** "dup" of ParserInterpreter */
public class LexerATNSimulator extends ATNSimulator {
public static final RuleContext EMPTY_LEXER_RULE_CONTEXT = new RuleContext();
public static boolean debug = false;
public static boolean dfa_debug = false;
public static final int MAX_DFA_EDGE = 127; // forces unicode to stay in ATN
@ -76,7 +73,7 @@ public class LexerATNSimulator extends ATNSimulator {
protected int line = 0;
protected int charPos = -1;
protected DFAState dfaState;
protected ATNConfig config;
protected LexerATNConfig config;
protected void reset() {
index = -1;
@ -104,7 +101,7 @@ public class LexerATNSimulator extends ATNSimulator {
protected int charPositionInLine = 0;
@NotNull
public final DFA[] dfa;
public final DFA[] decisionToDFA;
protected int mode = Lexer.DEFAULT_MODE;
/** Used during DFA/ATN exec to record the most recent accept configuration info */
@ -114,15 +111,26 @@ public class LexerATNSimulator extends ATNSimulator {
public static int ATN_failover = 0;
public static int match_calls = 0;
public LexerATNSimulator(@NotNull ATN atn) {
this(null, atn);
public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA,
@NotNull PredictionContextCache sharedContextCache)
{
this(null, atn, decisionToDFA,sharedContextCache);
}
public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn) {
super(atn);
dfa = new DFA[atn.modeToStartState.size()];
for (int i=0; i<atn.modeToStartState.size(); i++) {
dfa[i] = new DFA(atn.modeToStartState.get(i));
public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn,
@NotNull DFA[] decisionToDFA,
@NotNull PredictionContextCache sharedContextCache)
{
super(atn,sharedContextCache);
this.decisionToDFA = decisionToDFA;
if ( decisionToDFA[Lexer.DEFAULT_MODE]==null ) { // create all mode dfa
synchronized (this.decisionToDFA) {
if ( decisionToDFA[Lexer.DEFAULT_MODE]==null ) { // create all mode dfa
for (int i=0; i<atn.modeToStartState.size(); i++) {
this.decisionToDFA[i] = new DFA(atn.modeToStartState.get(i));
}
}
}
}
this.recog = recog;
}
@ -154,14 +162,15 @@ public class LexerATNSimulator extends ATNSimulator {
int mark = input.mark();
traceBeginMatch(input, mode);
try {
if ( dfa[mode].s0==null ) {
DFA dfa = decisionToDFA[mode];
if ( dfa.s0==null ) {
return matchATN(input);
}
else {
return execDFA(input, dfa[mode].s0);
return execDFA(input, dfa.s0);
}
}
finally {
finally {
traceEndMatch();
input.release(mark);
}
@ -189,11 +198,11 @@ public class LexerATNSimulator extends ATNSimulator {
ATNConfigSet s0_closure = computeStartState(input, startState);
int old_mode = mode;
dfa[mode].s0 = addDFAState(s0_closure);
int predict = execATN(input, s0_closure, dfa[mode].s0);
decisionToDFA[mode].s0 = addDFAState(s0_closure);
int predict = execATN(input, s0_closure, decisionToDFA[mode].s0);
if ( debug ) {
System.out.format("DFA after matchATN: %s\n", dfa[old_mode].toLexerString());
System.out.format("DFA after matchATN: %s\n", decisionToDFA[old_mode].toLexerString());
}
tracePredict(predict);
@ -247,7 +256,7 @@ public class LexerATNSimulator extends ATNSimulator {
t = input.LA(1);
}
ATNConfigSet reach = prevAccept.dfaState != null ? prevAccept.dfaState.configset : null;
ATNConfigSet reach = prevAccept.dfaState != null ? prevAccept.dfaState.configs : null;
return failOrAccept(prevAccept, input, reach, t);
}
@ -289,7 +298,7 @@ public class LexerATNSimulator extends ATNSimulator {
ATNConfigSet reach = null;
if (s != null) {
if ( s.edges != null && t < s.edges.length && t > CharStream.EOF ) {
closure = s.configset;
closure = s.configs;
target = s.edges[t];
if (target == ERROR) {
break;
@ -299,7 +308,7 @@ public class LexerATNSimulator extends ATNSimulator {
System.out.println("reuse state "+s.stateNumber+
" edge to "+target.stateNumber);
}
reach = target.configset;
reach = target.configs;
}
}
}
@ -325,7 +334,7 @@ public class LexerATNSimulator extends ATNSimulator {
}
// Did we hit a stop state during reach op?
processAcceptConfigs(input, reach);
reach = processAcceptConfigs(input, reach);
// Add an edge from s to target DFA found/created for reach
target = addDFAEdge(s, t, reach);
@ -387,19 +396,19 @@ public class LexerATNSimulator extends ATNSimulator {
Transition trans = c.state.transition(ti);
ATNState target = getReachableTarget(trans, t);
if ( target!=null ) {
closure(new ATNConfig(c, target), reach);
closure(new LexerATNConfig((LexerATNConfig)c, target), reach);
}
}
}
}
protected void processAcceptConfigs(@NotNull CharStream input, @NotNull ATNConfigSet reach) {
protected ATNConfigSet processAcceptConfigs(@NotNull CharStream input, @NotNull ATNConfigSet reach) {
if ( debug ) {
System.out.format("processAcceptConfigs: reach=%s, prevAccept=%s, prevIndex=%d\n",
reach, prevAccept.config, prevAccept.index);
}
for (int ci=0; ci<reach.size(); ci++) {
ATNConfig c = reach.get(ci);
LexerATNConfig c = (LexerATNConfig)reach.get(ci);
if ( c.state instanceof RuleStopState) {
if ( debug ) {
System.out.format("processAcceptConfigs: hit accept config %s index %d\n",
@ -419,18 +428,19 @@ public class LexerATNSimulator extends ATNSimulator {
if ( debug ) {
System.out.format("markExecSettings for %s @ index=%d, line %d:%d\n", c, index, prevAccept.line, prevAccept.charPos);
}
captureSimState(prevAccept, input, reach, c);
captureSimState(prevAccept, input, c);
}
// if we reach lexer accept state, toss out any configs in rest
// of configs work list associated with this rule (config.alt);
// that rule is done. this is how we cut off nongreedy .+ loops.
deleteWildcardConfigsForAlt(reach, ci, c.alt); // CAUSES INF LOOP if reach not closure
reach = deleteWildcardConfigsForAlt(reach, ci, c.alt);
// move to next char, looking for longer match
// (we continue processing if there are states in reach)
}
}
return reach;
}
protected void accept(@NotNull CharStream input, int ruleIndex, int actionIndex,
@ -476,7 +486,7 @@ public class LexerATNSimulator extends ATNSimulator {
}
return null;
case Transition.SET:
SetTransition st = (SetTransition)trans;
if ( st.set.contains(t) ) {
@ -501,51 +511,54 @@ public class LexerATNSimulator extends ATNSimulator {
}
return null;
case Transition.WILDCARD:
if (t != CharStream.EOF) {
return trans.target;
}
return null;
default:
return null;
}
}
public void deleteWildcardConfigsForAlt(@NotNull ATNConfigSet closure, int ci, int alt) {
/** Delete configs for alt following ci. Closure is unmodified; copy returned. */
public ATNConfigSet deleteWildcardConfigsForAlt(@NotNull ATNConfigSet closure, int ci, int alt) {
ATNConfigSet dup = new ATNConfigSet(closure);
int j=ci+1;
while ( j<closure.size() ) {
ATNConfig c = closure.get(j);
boolean isWildcard = c.state.getClass() == ATNState.class &&
c.state.transition(0).getClass() == WildcardTransition.class;
while ( j < dup.size() ) {
ATNConfig c = dup.get(j);
boolean isWildcard = c.state.getClass() == ATNState.class && // plain state only, not rulestop etc..
c.state.transition(0) instanceof WildcardTransition;
if ( c.alt == alt && isWildcard ) {
if ( debug ) {
System.out.format("deleteWildcardConfigsForAlt %s\n", c);
}
closure.remove(j);
dup.remove(j);
}
else j++;
}
return dup;
}
@NotNull
protected ATNConfigSet computeStartState(@NotNull IntStream input,
@NotNull ATNState p)
{
RuleContext initialContext = EMPTY_LEXER_RULE_CONTEXT;
PredictionContext initialContext = PredictionContext.EMPTY;
ATNConfigSet configs = new ATNConfigSet();
for (int i=0; i<p.getNumberOfTransitions(); i++) {
ATNState target = p.transition(i).target;
ATNConfig c = new ATNConfig(target, i+1, initialContext);
LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
closure(c, configs);
}
return configs;
}
protected void closure(@NotNull ATNConfig config, @NotNull ATNConfigSet configs) {
protected void closure(@NotNull LexerATNConfig config, @NotNull ATNConfigSet configs) {
if ( debug ) {
System.out.println("closure("+config.toString(recog, true)+")");
}
@ -566,12 +579,18 @@ public class LexerATNSimulator extends ATNSimulator {
configs.add(config);
return;
}
RuleContext newContext = config.context.parent; // "pop" invoking state
ATNState invokingState = atn.states.get(config.context.invokingState);
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
ATNConfig c = new ATNConfig(retState, config.alt, newContext);
closure(c, configs);
if ( config.context!=null && !config.context.isEmpty() ) {
for (SingletonPredictionContext ctx : config.context) {
if ( !ctx.isEmpty() ) {
PredictionContext newContext = ctx.parent; // "pop" invoking state
ATNState invokingState = atn.states.get(ctx.invokingState);
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
LexerATNConfig c = new LexerATNConfig(retState, config.alt, newContext);
closure(c, configs);
}
}
}
return;
}
@ -583,77 +602,65 @@ public class LexerATNSimulator extends ATNSimulator {
ATNState p = config.state;
for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i);
ATNConfig c = getEpsilonTarget(config, t, configs);
LexerATNConfig c = getEpsilonTarget(config, t, configs);
if ( c!=null ) closure(c, configs);
}
}
// side-effect: can alter configs.hasSemanticContext
@Nullable
public ATNConfig getEpsilonTarget(@NotNull ATNConfig config,
public LexerATNConfig getEpsilonTarget(@NotNull LexerATNConfig config,
@NotNull Transition t,
@NotNull ATNConfigSet configs)
{
ATNState p = config.state;
ATNConfig c;
LexerATNConfig c = null;
switch (t.getSerializationType()) {
case Transition.RULE:
RuleContext newContext =
new RuleContext(config.context, p.stateNumber);
c = new ATNConfig(config, t.target, newContext);
break;
case Transition.PREDICATE:
if (recog == null) {
System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");
}
case Transition.RULE:
PredictionContext newContext =
new SingletonPredictionContext(config.context, p.stateNumber);
c = new LexerATNConfig(config, t.target, newContext);
break;
case Transition.PREDICATE:
// if (recog == null) {
// System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");
// }
/* Track traversing semantic predicates. If we traverse,
we cannot add a DFA state for this "reach" computation
because the DFA would not test the predicate again in the
future. Rather than creating collections of semantic predicates
like v3 and testing them on prediction, v4 will test them on the
fly all the time using the ATN not the DFA. This is slower but
semantically it's not used that often. One of the key elements to
this predicate mechanism is not adding DFA states that see
predicates immediately afterwards in the ATN. For example,
/* Track traversing semantic predicates. If we traverse,
we cannot add a DFA state for this "reach" computation
because the DFA would not test the predicate again in the
future. Rather than creating collections of semantic predicates
like v3 and testing them on prediction, v4 will test them on the
fly all the time using the ATN not the DFA. This is slower but
semantically it's not used that often. One of the key elements to
this predicate mechanism is not adding DFA states that see
predicates immediately afterwards in the ATN. For example,
a : ID {p1}? | ID {p2}? ;
a : ID {p1}? | ID {p2}? ;
should create the start state for rule 'a' (to save start state
competition), but should not create target of ID state. The
collection of ATN states the following ID references includes
states reached by traversing predicates. Since this is when we
test them, we cannot cash the DFA state target of ID.
*/
PredicateTransition pt = (PredicateTransition)t;
if ( debug ) {
System.out.println("EVAL rule "+pt.ruleIndex+":"+pt.predIndex);
}
configs.hasSemanticContext = true;
if ( recog == null || recog.sempred(null, pt.ruleIndex, pt.predIndex) ) {
c = new ATNConfig(config, t.target, pt.getPredicate());
}
else {
c = null;
}
break;
case Transition.ACTION:
should create the start state for rule 'a' (to save start state
competition), but should not create target of ID state. The
collection of ATN states the following ID references includes
states reached by traversing predicates. Since this is when we
test them, we cannot cash the DFA state target of ID.
*/
PredicateTransition pt = (PredicateTransition)t;
if ( debug ) {
System.out.println("EVAL rule "+pt.ruleIndex+":"+pt.predIndex);
}
configs.hasSemanticContext = true;
if ( recog == null || recog.sempred(null, pt.ruleIndex, pt.predIndex) ) {
c = new LexerATNConfig(config, t.target, pt.getPredicate());
}
break;
// ignore actions; just exec one per rule upon accept
c = new ATNConfig(config, t.target);
c.lexerActionIndex = ((ActionTransition)t).actionIndex;
break;
case Transition.EPSILON:
c = new ATNConfig(config, t.target);
break;
default:
c = null;
break;
case Transition.ACTION:
c = new LexerATNConfig(config, t.target, ((ActionTransition)t).actionIndex);
break;
case Transition.EPSILON:
c = new LexerATNConfig(config, t.target);
break;
}
return c;
@ -665,14 +672,14 @@ public class LexerATNSimulator extends ATNSimulator {
if ( dfa_debug ) {
System.out.format("no edge for %s\n", getTokenName(input.LA(1)));
System.out.format("ATN exec upon %s at DFA state %d = %s\n",
input.getText(Interval.of(startIndex, input.index())), s.stateNumber, s.configset);
input.getText(Interval.of(startIndex, input.index())), s.stateNumber, s.configs);
}
int ttype = execATN(input, s.configset, s);
int ttype = execATN(input, s.configs, s);
if ( dfa_debug ) {
System.out.format("back from DFA update, ttype=%d, dfa[mode %d]=\n%s\n",
ttype, mode, dfa[mode].toLexerString());
ttype, mode, decisionToDFA[mode].toLexerString());
}
// action already executed by ATN
@ -694,8 +701,7 @@ public class LexerATNSimulator extends ATNSimulator {
protected void captureSimState(@NotNull SimState settings,
@NotNull CharStream input,
@NotNull ATNConfigSet ATNConfigs,
@NotNull ATNConfig config)
@NotNull LexerATNConfig config)
{
settings.index = input.index();
settings.line = line;
@ -727,11 +733,14 @@ public class LexerATNSimulator extends ATNSimulator {
protected void addDFAEdge(@NotNull DFAState p, int t, @NotNull DFAState q) {
if (t < 0 || t > MAX_DFA_EDGE) return; // Only track edges within the DFA bounds
if ( p.edges==null ) {
// make room for tokens 1..n and -1 masquerading as index 0
p.edges = new DFAState[MAX_DFA_EDGE+1]; // TODO: make adaptive
DFA dfa = decisionToDFA[mode];
synchronized (dfa) {
if ( p.edges==null ) {
// make room for tokens 1..n and -1 masquerading as index 0
p.edges = new DFAState[MAX_DFA_EDGE+1]; // TODO: make adaptive
}
p.edges[t] = q; // connect
}
p.edges[t] = q; // connect
}
/** Add a new DFA state if there isn't one with this set of
@ -745,11 +754,6 @@ public class LexerATNSimulator extends ATNSimulator {
if ( configs.hasSemanticContext ) return null;
DFAState proposed = new DFAState(configs);
DFAState existing = dfa[mode].states.get(proposed);
if ( existing!=null ) return existing;
DFAState newState = proposed;
ATNConfig firstConfigWithRuleStopState = null;
for (ATNConfig c : configs) {
if ( c.state instanceof RuleStopState ) {
@ -759,22 +763,31 @@ public class LexerATNSimulator extends ATNSimulator {
}
if ( firstConfigWithRuleStopState!=null ) {
newState.isAcceptState = true;
newState.lexerRuleIndex = firstConfigWithRuleStopState.state.ruleIndex;
newState.lexerActionIndex = firstConfigWithRuleStopState.lexerActionIndex;
newState.prediction = atn.ruleToTokenType[newState.lexerRuleIndex];
proposed.isAcceptState = true;
proposed.lexerRuleIndex = firstConfigWithRuleStopState.state.ruleIndex;
proposed.lexerActionIndex =
((LexerATNConfig)firstConfigWithRuleStopState).lexerActionIndex;
proposed.prediction = atn.ruleToTokenType[proposed.lexerRuleIndex];
}
newState.stateNumber = dfa[mode].states.size();
newState.configset = new ATNConfigSet();
newState.configset.addAll(configs);
dfa[mode].states.put(newState, newState);
return newState;
DFA dfa = decisionToDFA[mode];
synchronized (dfa) {
DFAState existing = dfa.states.get(proposed);
if ( existing!=null ) return existing;
DFAState newState = proposed;
newState.stateNumber = dfa.states.size();
configs.setReadonly(true);
newState.configs = configs;
decisionToDFA[mode].states.put(newState, newState);
return newState;
}
}
@Nullable
public DFA getDFA(int mode) {
return dfa[mode];
return decisionToDFA[mode];
}
/** Get the text of the current token from an *action* in lexer not
@ -896,7 +909,7 @@ public class LexerATNSimulator extends ATNSimulator {
public final void tracePushMode(int mode) {
if (trace) {
traceByteSlow(LexerOpCode.PushMode, (byte)mode);
traceByteSlow(LexerOpCode.PushMode, (byte) mode);
}
}

294
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNPathFinder.java
View File

@ -29,160 +29,146 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.TraceTree;
public class ParserATNPathFinder /*extends ParserATNSimulator<Token>*/ {
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class ParserATNPathFinder extends ParserATNSimulator<Token> {
public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn) {
super(parser, atn);
}
/** Given an input sequence, as a subset of the input stream, trace the path through the
* ATN starting at s. The path returned includes s and the final target of the last input
* symbol. If there are multiple paths through the ATN to the final state, it uses the first
* method finds. This is used to figure out how input sequence is matched in more than one
* way between the alternatives of a decision. It's only that decision we are concerned with
* and so if there are ambiguous decisions further along, we will ignore them for the
* purposes of computing the path to the final state. To figure out multiple paths for
* decision, use this method on the left edge of the alternatives of the decision in question.
*
* TODO: I haven't figured out what to do with nongreedy decisions yet
* TODO: preds. unless i create rule specific ctxs, i can't eval preds. also must eval args!
*/
public TraceTree trace(@NotNull ATNState s, @Nullable RuleContext ctx,
TokenStream input, int start, int stop)
{
System.out.println("REACHES "+s.stateNumber+" start state");
List<TraceTree> leaves = new ArrayList<TraceTree>();
HashSet<ATNState>[] busy = new HashSet[stop-start+1];
for (int i = 0; i < busy.length; i++) {
busy[i] = new HashSet<ATNState>();
}
TraceTree path = _trace(s, ctx, ctx, input, start, start, stop, leaves, busy);
if ( path!=null ) path.leaves = leaves;
return path;
}
/** Returns true if we found path */
public TraceTree _trace(@NotNull ATNState s, RuleContext initialContext, RuleContext ctx,
TokenStream input, int start, int i, int stop,
List<TraceTree> leaves, @NotNull Set<ATNState>[] busy)
{
TraceTree root = new TraceTree(s);
if ( i>stop ) {
leaves.add(root); // track final states
System.out.println("leaves=" + leaves);
return root;
}
if ( !busy[i-start].add(s) ) {
System.out.println("already visited "+s.stateNumber+" at input "+i+"="+input.get(i).getText());
return null;
}
busy[i-start].add(s);
System.out.println("TRACE "+s.stateNumber+" at input "+input.get(i).getText());
if ( s instanceof RuleStopState) {
// We hit rule end. If we have context info, use it
if ( ctx!=null && !ctx.isEmpty() ) {
System.out.println("stop state "+s.stateNumber+", ctx="+ctx);
ATNState invokingState = atn.states.get(ctx.invokingState);
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
return _trace(retState, initialContext, ctx.parent, input, start, i, stop, leaves, busy);
}
else {
// else if we have no context info, just chase follow links (if greedy)
System.out.println("FALLING off rule "+getRuleName(s.ruleIndex));
}
}
int n = s.getNumberOfTransitions();
boolean aGoodPath = false;
TraceTree found;
for (int j=0; j<n; j++) {
Transition t = s.transition(j);
if ( t.getClass() == RuleTransition.class ) {
RuleContext newContext =
new RuleContext(ctx, s.stateNumber);
found = _trace(t.target, initialContext, newContext, input, start, i, stop, leaves, busy);
if ( found!=null ) {aGoodPath=true; root.addChild(found);}
continue;
}
if ( t instanceof PredicateTransition ) {
found = predTransition(initialContext, ctx, input, start, i, stop, leaves, busy, root, t);
if ( found!=null ) {aGoodPath=true; root.addChild(found);}
continue;
}
if ( t.isEpsilon() ) {
found = _trace(t.target, initialContext, ctx, input, start, i, stop, leaves, busy);
if ( found!=null ) {aGoodPath=true; root.addChild(found);}
continue;
}
if ( t.getClass() == WildcardTransition.class ) {
System.out.println("REACHES " + t.target.stateNumber + " matching input " + input.get(i).getText());
found = _trace(t.target, initialContext, ctx, input, start, i+1, stop, leaves, busy);
if ( found!=null ) {aGoodPath=true; root.addChild(found);}
continue;
}
IntervalSet set = t.label();
if ( set!=null ) {
if ( t instanceof NotSetTransition ) {
if ( !set.contains(input.get(i).getType()) ) {
System.out.println("REACHES " + t.target.stateNumber + " matching input " + input.get(i).getText());
found = _trace(t.target, initialContext, ctx, input, start, i+1, stop, leaves, busy);
if ( found!=null ) {aGoodPath=true; root.addChild(found);}
}
}
else {
if ( set.contains(input.get(i).getType()) ) {
System.out.println("REACHES " + t.target.stateNumber + " matching input " + input.get(i).getText());
found = _trace(t.target, initialContext, ctx, input, start, i+1, stop, leaves, busy);
if ( found!=null ) {aGoodPath=true; root.addChild(found);}
}
}
}
}
if ( aGoodPath ) return root; // found at least one transition leading to success
return null;
}
public TraceTree predTransition(RuleContext initialContext, RuleContext ctx, TokenStream input, int start,
int i, int stop, List<TraceTree> leaves, Set<ATNState>[] busy,
TraceTree root, Transition t)
{
SemanticContext.Predicate pred = ((PredicateTransition) t).getPredicate();
boolean pass;
if ( pred.isCtxDependent ) {
if ( ctx instanceof ParserRuleContext && ctx==initialContext ) {
System.out.println("eval pred "+pred+"="+pred.eval(parser, ctx));
pass = pred.eval(parser, ctx);
}
else {
pass = true; // see thru ctx dependent when out of context
}
}
else {
System.out.println("eval pred "+pred+"="+pred.eval(parser, initialContext));
pass = pred.eval(parser, ctx);
}
if ( pass ) {
return _trace(t.target, initialContext, ctx, input, start, i, stop, leaves, busy);
}
return null;
}
// public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
// super(parser, atn, decisionToDFA);
// }
//
// /** Given an input sequence, as a subset of the input stream, trace the path through the
// * ATN starting at s. The path returned includes s and the final target of the last input
// * symbol. If there are multiple paths through the ATN to the final state, it uses the first
// * method finds. This is used to figure out how input sequence is matched in more than one
// * way between the alternatives of a decision. It's only that decision we are concerned with
// * and so if there are ambiguous decisions further along, we will ignore them for the
// * purposes of computing the path to the final state. To figure out multiple paths for
// * decision, use this method on the left edge of the alternatives of the decision in question.
// *
// * TODO: I haven't figured out what to do with nongreedy decisions yet
// * TODO: preds. unless i create rule specific ctxs, i can't eval preds. also must eval args!
// */
// public TraceTree trace(@NotNull ATNState s, @Nullable RuleContext ctx,
// TokenStream input, int start, int stop)
// {
// System.out.println("REACHES "+s.stateNumber+" start state");
// List<TraceTree> leaves = new ArrayList<TraceTree>();
// HashSet<ATNState>[] busy = new HashSet[stop-start+1];
// for (int i = 0; i < busy.length; i++) {
// busy[i] = new HashSet<ATNState>();
// }
// TraceTree path = _trace(s, ctx, ctx, input, start, start, stop, leaves, busy);
// if ( path!=null ) path.leaves = leaves;
// return path;
// }
//
// /** Returns true if we found path */
// public TraceTree _trace(@NotNull ATNState s, RuleContext initialContext, RuleContext ctx,
// TokenStream input, int start, int i, int stop,
// List<TraceTree> leaves, @NotNull Set<ATNState>[] busy)
// {
// TraceTree root = new TraceTree(s);
// if ( i>stop ) {
// leaves.add(root); // track final states
// System.out.println("leaves=" + leaves);
// return root;
// }
//
// if ( !busy[i-start].add(s) ) {
// System.out.println("already visited "+s.stateNumber+" at input "+i+"="+input.get(i).getText());
// return null;
// }
// busy[i-start].add(s);
//
// System.out.println("TRACE "+s.stateNumber+" at input "+input.get(i).getText());
//
// if ( s instanceof RuleStopState) {
// // We hit rule end. If we have context info, use it
// if ( ctx!=null && !ctx.isEmpty() ) {
// System.out.println("stop state "+s.stateNumber+", ctx="+ctx);
// ATNState invokingState = atn.states.get(ctx.invokingState);
// RuleTransition rt = (RuleTransition)invokingState.transition(0);
// ATNState retState = rt.followState;
// return _trace(retState, initialContext, ctx.parent, input, start, i, stop, leaves, busy);
// }
// else {
// // else if we have no context info, just chase follow links (if greedy)
// System.out.println("FALLING off rule "+getRuleName(s.ruleIndex));
// }
// }
//
// int n = s.getNumberOfTransitions();
// boolean aGoodPath = false;
// TraceTree found;
// for (int j=0; j<n; j++) {
// Transition t = s.transition(j);
// if ( t.getClass() == RuleTransition.class ) {
// RuleContext newContext =
// new RuleContext(ctx, s.stateNumber);
// found = _trace(t.target, initialContext, newContext, input, start, i, stop, leaves, busy);
// if ( found!=null ) {aGoodPath=true; root.addChild(found);}
// continue;
// }
// if ( t instanceof PredicateTransition ) {
// found = predTransition(initialContext, ctx, input, start, i, stop, leaves, busy, root, t);
// if ( found!=null ) {aGoodPath=true; root.addChild(found);}
// continue;
// }
// if ( t.isEpsilon() ) {
// found = _trace(t.target, initialContext, ctx, input, start, i, stop, leaves, busy);
// if ( found!=null ) {aGoodPath=true; root.addChild(found);}
// continue;
// }
// if ( t.getClass() == WildcardTransition.class ) {
// System.out.println("REACHES " + t.target.stateNumber + " matching input " + input.get(i).getText());
// found = _trace(t.target, initialContext, ctx, input, start, i+1, stop, leaves, busy);
// if ( found!=null ) {aGoodPath=true; root.addChild(found);}
// continue;
// }
// IntervalSet set = t.label();
// if ( set!=null ) {
// if ( t instanceof NotSetTransition ) {
// if ( !set.contains(input.get(i).getType()) ) {
// System.out.println("REACHES " + t.target.stateNumber + " matching input " + input.get(i).getText());
// found = _trace(t.target, initialContext, ctx, input, start, i+1, stop, leaves, busy);
// if ( found!=null ) {aGoodPath=true; root.addChild(found);}
// }
// }
// else {
// if ( set.contains(input.get(i).getType()) ) {
// System.out.println("REACHES " + t.target.stateNumber + " matching input " + input.get(i).getText());
// found = _trace(t.target, initialContext, ctx, input, start, i+1, stop, leaves, busy);
// if ( found!=null ) {aGoodPath=true; root.addChild(found);}
// }
// }
// }
// }
// if ( aGoodPath ) return root; // found at least one transition leading to success
// return null;
// }
//
// public TraceTree predTransition(RuleContext initialContext, RuleContext ctx, TokenStream input, int start,
// int i, int stop, List<TraceTree> leaves, Set<ATNState>[] busy,
// TraceTree root, Transition t)
// {
// SemanticContext.Predicate pred = ((PredicateTransition) t).getPredicate();
// boolean pass;
// if ( pred.isCtxDependent ) {
// if ( ctx instanceof ParserRuleContext && ctx==initialContext ) {
// System.out.println("eval pred "+pred+"="+pred.eval(parser, ctx));
// pass = pred.eval(parser, ctx);
// }
// else {
// pass = true; // see thru ctx dependent when out of context
// }
// }
// else {
// System.out.println("eval pred "+pred+"="+pred.eval(parser, initialContext));
// pass = pred.eval(parser, ctx);
// }
// if ( pass ) {
// return _trace(t.target, initialContext, ctx, input, start, i, stop, leaves, busy);
// }
// return null;
// }
}

1150
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

File diff suppressed because it is too large Load Diff

638
runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java Normal file
View File

@ -0,0 +1,638 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public abstract class PredictionContext implements Iterable<SingletonPredictionContext>,
Comparable<PredictionContext> // to sort node lists by id
{
/** Represents $ in local ctx prediction, which means wildcard. *+x = *. */
public static final EmptyPredictionContext EMPTY = new EmptyPredictionContext();
/** Represents $ in an array in full ctx mode, when $ doesn't mean wildcard:
* $ + x = [$,x]. Here, $ = EMPTY_FULL_CTX_INVOKING_STATE.
*/
public static final int EMPTY_FULL_CTX_INVOKING_STATE = Integer.MAX_VALUE;
public static int globalNodeCount = 0;
public final int id = globalNodeCount++;
public final int cachedHashCode;
protected PredictionContext(int cachedHashCode) {
this.cachedHashCode = cachedHashCode;
}
/** Convert a RuleContext tree to a PredictionContext graph.
* Return EMPTY if outerContext is empty or null.
*/
public static PredictionContext fromRuleContext(RuleContext outerContext) {
if ( outerContext==null ) outerContext = RuleContext.EMPTY;
// if we are in RuleContext of start rule, s, then PredictionContext
// is EMPTY. Nobody called us. (if we are empty, return empty)
if ( outerContext.parent==null || outerContext==RuleContext.EMPTY ) {
return PredictionContext.EMPTY;
}
// If we have a parent, convert it to a PredictionContext graph
PredictionContext parent = EMPTY;
if ( outerContext.parent != null ) {
parent = PredictionContext.fromRuleContext(outerContext.parent);
}
return new SingletonPredictionContext(parent, outerContext.invokingState);
}
@Override
public abstract Iterator<SingletonPredictionContext> iterator();
public abstract int size();
public abstract PredictionContext getParent(int index);
public abstract int getInvokingState(int index);
/** This means only the EMPTY context is in set */
public boolean isEmpty() {
return this == EMPTY;
}
public abstract PredictionContext popAll(
int invokingState,
boolean fullCtx,
DoubleKeyMap<PredictionContext,PredictionContext,PredictionContext> mergeCache);
@Override
public int compareTo(PredictionContext o) { // used for toDotString to print nodes in order
return id - o.id;
}
@Override
public int hashCode() {
return cachedHashCode;
}
protected static int calculateHashCode(int parentHashCode, int invokingStateHashCode) {
return 5 * 5 * 7 + 5 * parentHashCode + invokingStateHashCode;
}
/** Two contexts conflict() if they are equals() or one is a stack suffix
* of the other. For example, contexts [21 12 $] and [21 9 $] do not
* conflict, but [21 $] and [21 12 $] do conflict. Note that I should
* probably not show the $ in this case. There is a dummy node for each
* stack that just means empty; $ is a marker that's all.
*
* This is used in relation to checking conflicts associated with a
* single NFA state's configurations within a single DFA state.
* If there are configurations s and t within a DFA state such that
* s.state=t.state && s.alt != t.alt && s.ctx conflicts t.ctx then
* the DFA state predicts more than a single alt--it's nondeterministic.
* Two contexts conflict if they are the same or if one is a suffix
* of the other.
*
* When comparing contexts, if one context has a stack and the other
* does not then they should be considered the same context. The only
* way for an NFA state p to have an empty context and a nonempty context
* is the case when closure falls off end of rule without a call stack
* and re-enters the rule with a context. This resolves the issue I
* discussed with Sriram Srinivasan Feb 28, 2005 about not terminating
* fast enough upon nondeterminism.
*
* UPDATE FOR GRAPH STACK; no suffix
*/
// public boolean conflictsWith(PredictionContext other) {
// return this.equals(other);
// }
// dispatch
public static PredictionContext merge(
PredictionContext a, PredictionContext b,
boolean rootIsWildcard,
DoubleKeyMap<PredictionContext,PredictionContext,PredictionContext> mergeCache)
{
if ( (a==null&&b==null) || a==b || a.equals(b) ) return a; // share same graph if both same
if ( a instanceof SingletonPredictionContext && b instanceof SingletonPredictionContext) {
return mergeSingletons((SingletonPredictionContext)a,
(SingletonPredictionContext)b,
rootIsWildcard, mergeCache);
}
// At least one of a or b is array
// If one is $ and rootIsWildcard, return $ as * wildcard
if ( rootIsWildcard ) {
if ( a instanceof EmptyPredictionContext ) return a;
if ( b instanceof EmptyPredictionContext ) return b;
}
// convert singleton so both are arrays to normalize
if ( a instanceof SingletonPredictionContext ) {
a = new ArrayPredictionContext((SingletonPredictionContext)a);
}
if ( b instanceof SingletonPredictionContext) {
b = new ArrayPredictionContext((SingletonPredictionContext)b);
}
return mergeArrays((ArrayPredictionContext) a, (ArrayPredictionContext) b,
rootIsWildcard, mergeCache);
}
// http://www.antlr.org/wiki/download/attachments/32014352/singleton-merge.png
public static PredictionContext mergeSingletons(
SingletonPredictionContext a,
SingletonPredictionContext b,
boolean rootIsWildcard,
DoubleKeyMap<PredictionContext,PredictionContext,PredictionContext> mergeCache)
{
if ( mergeCache!=null ) {
PredictionContext previous = mergeCache.get(a,b);
if ( previous!=null ) return previous;
previous = mergeCache.get(b,a);
if ( previous!=null ) return previous;
}
PredictionContext rootMerge = mergeRoot(a, b, rootIsWildcard);
if ( rootMerge!=null ) {
if ( mergeCache!=null ) mergeCache.put(a, b, rootMerge);
return rootMerge;
}
if ( a.invokingState==b.invokingState ) { // a == b
PredictionContext parent = merge(a.parent, b.parent, rootIsWildcard, mergeCache);
// if parent is same as existing a or b parent or reduced to a parent, return it
if ( parent == a.parent ) return a; // ax + bx = ax, if a=b
if ( parent == b.parent ) return b; // ax + bx = bx, if a=b
// else: ax + ay = a'[x,y]
// merge parents x and y, giving array node with x,y then remainders
// of those graphs. dup a, a' points at merged array
// new joined parent so create new singleton pointing to it, a'
PredictionContext a_ = new SingletonPredictionContext(parent, a.invokingState);
if ( mergeCache!=null ) mergeCache.put(a, b, a_);
return a_;
}
else { // a != b payloads differ
// see if we can collapse parents due to $+x parents if local ctx
PredictionContext singleParent = null;
if ( rootIsWildcard ) {
if ( a.parent == EMPTY ) singleParent = EMPTY; // $ + b = $
if ( b.parent == EMPTY ) singleParent = EMPTY; // a + $ = $
}
if ( a==b || a.parent.equals(b.parent) ) { // ax + bx = [a,b]x
singleParent = a.parent;
}
if ( singleParent!=null ) { // parents are same
// sort payloads and use same parent
int[] payloads = {a.invokingState, b.invokingState};
if ( a.invokingState > b.invokingState ) {
payloads[0] = b.invokingState;
payloads[1] = a.invokingState;
}
PredictionContext[] parents = {singleParent, singleParent};
PredictionContext a_ = new ArrayPredictionContext(parents, payloads);
if ( mergeCache!=null ) mergeCache.put(a, b, a_);
return a_;
}
// parents differ and can't merge them. Just pack together
// into array; can't merge.
// ax + by = [ax,by]
int[] payloads = {a.invokingState, b.invokingState};
PredictionContext[] parents = {a.parent, b.parent};
if ( a.invokingState > b.invokingState ) { // sort by payload
payloads[0] = b.invokingState;
payloads[1] = a.invokingState;
parents = new PredictionContext[] {b.parent, a.parent};
}
PredictionContext a_ = new ArrayPredictionContext(parents, payloads);
if ( mergeCache!=null ) mergeCache.put(a, b, a_);
return a_;
}
}
// http://www.antlr.org/wiki/download/attachments/32014352/local-ctx-root-merge.png
// http://www.antlr.org/wiki/download/attachments/32014352/full-ctx-root-merge.png
/** Handle case where at least one of a or b is $ (EMPTY) */
public static PredictionContext mergeRoot(SingletonPredictionContext a,
SingletonPredictionContext b,
boolean rootIsWildcard)
{
if ( rootIsWildcard ) {
if ( a == EMPTY ) return EMPTY; // * + b = *
if ( b == EMPTY ) return EMPTY; // a + * = *
}
else {
if ( a == EMPTY && b == EMPTY ) return EMPTY; // $ + $ = $
if ( a == EMPTY ) { // $ + x = [$,x]
int[] payloads = {EMPTY_FULL_CTX_INVOKING_STATE, b.invokingState};
PredictionContext[] parents = {null, b.parent};
PredictionContext joined =
new ArrayPredictionContext(parents, payloads);
return joined;
}
if ( b == EMPTY ) { // x + $ = [$,x] ($ is always first if present)
int[] payloads = {EMPTY_FULL_CTX_INVOKING_STATE, a.invokingState};
PredictionContext[] parents = {null, a.parent};
PredictionContext joined =
new ArrayPredictionContext(parents, payloads);
return joined;
}
}
return null;
}
// http://www.antlr.org/wiki/download/attachments/32014352/array-merge.png
public static PredictionContext mergeArrays(
ArrayPredictionContext a,
ArrayPredictionContext b,
boolean rootIsWildcard,
DoubleKeyMap<PredictionContext,PredictionContext,PredictionContext> mergeCache)
{
if ( mergeCache!=null ) {
PredictionContext previous = mergeCache.get(a,b);
if ( previous!=null ) return previous;
previous = mergeCache.get(b,a);
if ( previous!=null ) return previous;
}
// merge sorted payloads a + b => M
int i = 0; // walks a
int j = 0; // walks b
int k = 0; // walks target M array
int[] mergedInvokingStates =
new int[a.invokingStates.length + b.invokingStates.length];
PredictionContext[] mergedParents =
new PredictionContext[a.invokingStates.length + b.invokingStates.length];
// walk and merge to yield mergedParents, mergedInvokingStates
while ( i<a.invokingStates.length && j<b.invokingStates.length ) {
PredictionContext a_parent = a.parents[i];
PredictionContext b_parent = b.parents[j];
if ( a.invokingStates[i]==b.invokingStates[j] ) {
// same payload (stack tops are equal), must yield merged singleton
int payload = a.invokingStates[i];
// $+$ = $
boolean both$ = payload == EMPTY_FULL_CTX_INVOKING_STATE &&
a_parent == null && b_parent == null;
boolean ax_ax = (a_parent!=null && b_parent!=null) &&
a_parent.equals(b_parent); // ax+ax -> ax
if ( both$ || ax_ax ) {
mergedParents[k] = a_parent; // choose left
mergedInvokingStates[k] = payload;
}
else { // ax+ay -> a'[x,y]
PredictionContext mergedParent =
merge(a_parent, b_parent, rootIsWildcard, mergeCache);
mergedParents[k] = mergedParent;
mergedInvokingStates[k] = payload;
}
i++; // hop over left one as usual
j++; // but also skip one in right side since we merge
}
else if ( a.invokingStates[i]<b.invokingStates[j] ) { // copy a[i] to M
mergedParents[k] = a_parent;
mergedInvokingStates[k] = a.invokingStates[i];
i++;
}
else { // b > a, copy b[j] to M
mergedParents[k] = b_parent;
mergedInvokingStates[k] = b.invokingStates[j];
j++;
}
k++;
}
// copy over any payloads remaining in either array
if (i < a.invokingStates.length) {
for (int p = i; p < a.invokingStates.length; p++) {
mergedParents[k] = a.parents[p];
mergedInvokingStates[k] = a.invokingStates[p];
k++;
}
}
else {
for (int p = j; p < b.invokingStates.length; p++) {
mergedParents[k] = b.parents[p];
mergedInvokingStates[k] = b.invokingStates[p];
k++;
}
}
// trim merged if we combined a few that had same stack tops
if ( k < mergedParents.length ) { // write index < last position; trim
int lastSlot = mergedParents.length - 1;
int p = lastSlot; // walk backwards from last index until we find non-null parent
while ( p>=0 && mergedParents[p]==null ) { p--; }
// p is now last non-null index
assert p>0; // could only happen to be <0 if two arrays with $
if ( p < lastSlot ) {
int n = p+1; // how many slots we really used in merge
if ( n == 1 ) { // for just one merged element, return singleton top
PredictionContext a_ = new SingletonPredictionContext(mergedParents[0],
mergedInvokingStates[0]);
if ( mergeCache!=null ) mergeCache.put(a,b,a_);
return a_;
}
mergedParents = Arrays.copyOf(mergedParents, n);
mergedInvokingStates = Arrays.copyOf(mergedInvokingStates, n);
}
}
PredictionContext M =
new ArrayPredictionContext(mergedParents, mergedInvokingStates);
// if we created same array as a or b, return that instead
// TODO: track whether this is possible above during merge sort for speed
if ( M.equals(a) ) {
if ( mergeCache!=null ) mergeCache.put(a,b,a);
return a;
}
if ( M.equals(b) ) {
if ( mergeCache!=null ) mergeCache.put(a,b,b);
return b;
}
combineCommonParents(mergedParents);
if ( mergeCache!=null ) mergeCache.put(a,b,M);
return M;
}
/** make pass over all M parents; merge any equals() ones */
protected static void combineCommonParents(PredictionContext[] parents) {
Map<PredictionContext, PredictionContext> uniqueParents =
new HashMap<PredictionContext, PredictionContext>();
for (int p = 0; p < parents.length; p++) {
PredictionContext parent = parents[p];
if ( !uniqueParents.containsKey(parent) ) { // don't replace
uniqueParents.put(parent, parent);
}
}
for (int p = 0; p < parents.length; p++) {
parents[p] = uniqueParents.get(parents[p]);
}
}
public static String toDOTString(PredictionContext context) {
if ( context==null ) return "";
StringBuilder buf = new StringBuilder();
buf.append("digraph G {\n");
buf.append("rankdir=LR;\n");
List<PredictionContext> nodes = getAllContextNodes(context);
Collections.sort(nodes);
for (PredictionContext current : nodes) {
if ( current instanceof SingletonPredictionContext ) {
String s = String.valueOf(current.id);
buf.append(" s").append(s);
String invokingState = String.valueOf(current.getInvokingState(0));
if ( current instanceof EmptyPredictionContext ) invokingState = "$";
buf.append(" [label=\"").append(invokingState).append("\"];\n");
continue;
}
ArrayPredictionContext arr = (ArrayPredictionContext)current;
buf.append(" s").append(arr.id);
buf.append(" [shape=box, label=\"");
buf.append("[");
boolean first = true;
for (int inv : arr.invokingStates) {
if ( !first ) buf.append(", ");
if ( inv == EMPTY_FULL_CTX_INVOKING_STATE ) buf.append("$");
else buf.append(inv);
first = false;
}
buf.append("]");
buf.append("\"];\n");
}
for (PredictionContext current : nodes) {
if ( current==EMPTY ) continue;
for (int i = 0; i < current.size(); i++) {
if ( current.getParent(i)==null ) continue;
String s = String.valueOf(current.id);
buf.append(" s").append(s);
buf.append("->");
buf.append("s");
buf.append(current.getParent(i).id);
if ( current.size()>1 ) buf.append(" [label=\"parent["+i+"]\"];\n");
else buf.append(";\n");
}
}
buf.append("}\n");
return buf.toString();
}
// From Sam
public static PredictionContext getCachedContext(
@NotNull PredictionContext context,
@NotNull PredictionContextCache contextCache,
@NotNull IdentityHashMap<PredictionContext, PredictionContext> visited)
{
if (context.isEmpty()) {
return context;
}
PredictionContext existing = visited.get(context);
if (existing != null) {
return existing;
}
synchronized (contextCache) {
existing = contextCache.get(context);
if (existing != null) {
visited.put(context, existing);
return existing;
}
}
boolean changed = false;
PredictionContext[] parents = new PredictionContext[context.size()];
for (int i = 0; i < parents.length; i++) {
PredictionContext parent = getCachedContext(context.getParent(i), contextCache, visited);
if (changed || parent != context.getParent(i)) {
if (!changed) {
parents = new PredictionContext[context.size()];
for (int j = 0; j < context.size(); j++) {
parents[j] = context.getParent(j);
}
changed = true;
}
parents[i] = parent;
}
}
if (!changed) {
synchronized (contextCache) {
contextCache.add(context);
}
visited.put(context, context);
return context;
}
PredictionContext updated;
if (parents.length == 0) {
updated = EMPTY;
}
else if (parents.length == 1) {
updated = new SingletonPredictionContext(parents[0], context.getInvokingState(0));
}
else {
ArrayPredictionContext arrayPredictionContext = (ArrayPredictionContext)context;
updated = new ArrayPredictionContext(parents, arrayPredictionContext.invokingStates);
}
synchronized (contextCache) {
contextCache.add(updated);
}
visited.put(updated, updated);
visited.put(context, updated);
return updated;
}
// // extra structures, but cut/paste/morphed works, so leave it.
// // seems to do a breadth-first walk
// public static List<PredictionContext> getAllNodes(PredictionContext context) {
// Map<PredictionContext, PredictionContext> visited =
// new IdentityHashMap<PredictionContext, PredictionContext>();
// Deque<PredictionContext> workList = new ArrayDeque<PredictionContext>();
// workList.add(context);
// visited.put(context, context);
// List<PredictionContext> nodes = new ArrayList<PredictionContext>();
// while (!workList.isEmpty()) {
// PredictionContext current = workList.pop();
// nodes.add(current);
// for (int i = 0; i < current.size(); i++) {
// PredictionContext parent = current.getParent(i);
// if ( parent!=null && visited.put(parent, parent) == null) {
// workList.push(parent);
// }
// }
// }
// return nodes;
// }
// ter's recursive version of Sam's getAllNodes()
public static List<PredictionContext> getAllContextNodes(PredictionContext context) {
List<PredictionContext> nodes = new ArrayList<PredictionContext>();
Map<PredictionContext, PredictionContext> visited =
new IdentityHashMap<PredictionContext, PredictionContext>();
getAllContextNodes_(context, nodes, visited);
return nodes;
}
public static void getAllContextNodes_(PredictionContext context,
List<PredictionContext> nodes,
Map<PredictionContext, PredictionContext> visited)
{
if ( context==null || visited.containsKey(context) ) return;
visited.put(context, context);
nodes.add(context);
for (int i = 0; i < context.size(); i++) {
getAllContextNodes_(context.getParent(i), nodes, visited);
}
}
public String toString(@Nullable Recognizer<?,?> recog) {
return toString();
// return toString(recog, ParserRuleContext.EMPTY);
}
// recog null unless ParserRuleContext, in which case we use subclass toString(...)
public String toString(@Nullable Recognizer<?,?> recog, RuleContext stop) {
StringBuilder buf = new StringBuilder();
PredictionContext p = this;
buf.append("[");
// while ( p != null && p != stop ) {
// if ( !p.isEmpty() ) buf.append(p.invokingState);
// if ( p.parent != null && !p.parent.isEmpty() ) buf.append(" ");
// p = p.parent;
// }
buf.append("]");
return buf.toString();
}
public String[] toStrings(Recognizer<?, ?> recognizer, int currentState) {
return toStrings(recognizer, EMPTY, currentState);
}
// FROM SAM
public String[] toStrings(Recognizer<?, ?> recognizer, PredictionContext stop, int currentState) {
List<String> result = new ArrayList<String>();
outer:
for (int perm = 0; ; perm++) {
int offset = 0;
boolean last = true;
PredictionContext p = this;
int stateNumber = currentState;
StringBuilder localBuffer = new StringBuilder();
localBuffer.append("[");
while ( !p.isEmpty() && p != stop ) {
int index = 0;
if (p.size() > 0) {
int bits = 1;
while ((1 << bits) < p.size()) {
bits++;
}
int mask = (1 << bits) - 1;
index = (perm >> offset) & mask;
last &= index >= p.size() - 1;
if (index >= p.size()) {
continue outer;
}
offset += bits;
}
if ( recognizer!=null ) {
if (localBuffer.length() > 1) {
// first char is '[', if more than that this isn't the first rule
localBuffer.append(' ');
}
ATN atn = recognizer.getATN();
ATNState s = atn.states.get(stateNumber);
String ruleName = recognizer.getRuleNames()[s.ruleIndex];
localBuffer.append(ruleName);
}
else if ( p.getInvokingState(index)!= EMPTY_FULL_CTX_INVOKING_STATE) {
if ( !p.isEmpty() ) {
if (localBuffer.length() > 1) {
// first char is '[', if more than that this isn't the first rule
localBuffer.append(' ');
}
localBuffer.append(p.getInvokingState(index));
}
}
stateNumber = p.getInvokingState(index);
p = p.getParent(index);
}
localBuffer.append("]");
result.add(localBuffer.toString());
if (last) {
break;
}
}
return result.toArray(new String[result.size()]);
}
}

36
runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java Normal file
View File

@ -0,0 +1,36 @@
package org.antlr.v4.runtime.atn;
import java.util.HashMap;
import java.util.Map;
/** Used to cache PredictionContext objects. Its used for the shared
* context cash associated with contexts in DFA states. This cache
* can be used for both lexers and parsers.
*/
public class PredictionContextCache {
protected Map<PredictionContext, PredictionContext> cache =
new HashMap<PredictionContext, PredictionContext>();
/** Add a context to the cache and return it. If the context already exists,
* return that one instead and do not add a new context to the cache.
* Protect shared cache from unsafe thread access.
*/
public PredictionContext add(PredictionContext ctx) {
if ( ctx==PredictionContext.EMPTY ) return PredictionContext.EMPTY;
PredictionContext existing = cache.get(ctx);
if ( existing!=null ) {
// System.out.println(name+" reuses "+existing);
return existing;
}
cache.put(ctx, ctx);
return ctx;
}
public PredictionContext get(PredictionContext ctx) {
return cache.get(ctx);
}
public int size() {
return cache.size();
}
}

2
runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java
View File

@ -29,8 +29,8 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
public final class RangeTransition extends Transition {
public final int from;

2
runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java
View File

@ -64,8 +64,6 @@ public abstract class SemanticContext {
*/
public abstract boolean eval(Recognizer<?,?> parser, RuleContext outerContext);
public SemanticContext optimize() { return this; }
public static class Predicate extends SemanticContext {
public final int ruleIndex;
public final int predIndex;

4
runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java
View File

@ -29,10 +29,10 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
/** A transition containing a set of values */
public class SetTransition extends Transition {

88
runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java Normal file
View File

@ -0,0 +1,88 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import java.util.Iterator;
public class SingletonPredictionContext extends PredictionContext {
public final PredictionContext parent;
public final int invokingState;
public SingletonPredictionContext(PredictionContext parent, int invokingState) {
super(calculateHashCode(parent!=null ? 31 ^ parent.hashCode() : 1,
31 ^ invokingState));
assert invokingState!=EMPTY_FULL_CTX_INVOKING_STATE &&
invokingState!=ATNState.INVALID_STATE_NUMBER;
this.parent = parent;
this.invokingState = invokingState;
}
@Override
public Iterator<SingletonPredictionContext> iterator() {
final SingletonPredictionContext self = this;
return new Iterator<SingletonPredictionContext>() {
int i = 0;
@Override
public boolean hasNext() { return i==0; }
@Override
public SingletonPredictionContext next() { i++; return self; }
@Override
public void remove() { throw new UnsupportedOperationException(); }
};
}
@Override
public int size() {
return 1;
}
@Override
public PredictionContext getParent(int index) {
assert index == 0;
return parent;
}
@Override
public int getInvokingState(int index) {
assert index == 0;
return invokingState;
}
@Override
public PredictionContext popAll(
int invokingState,
boolean fullCtx,
DoubleKeyMap<PredictionContext,PredictionContext,PredictionContext> mergeCache)
{
if ( invokingState == this.invokingState ) {
return parent.popAll(invokingState, fullCtx, mergeCache);
}
return this;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
else if ( !(o instanceof SingletonPredictionContext) ) {
return false;
}
if ( this.hashCode() != o.hashCode() ) {
return false; // can't be same if hash is different
}
SingletonPredictionContext s = (SingletonPredictionContext)o;
return invokingState == s.invokingState && parent.equals(s.parent);
}
@Override
public String toString() {
String up = parent!=null ? parent.toString() : "";
if ( up.length()==0 ) return String.valueOf(invokingState);
return String.valueOf(invokingState)+" "+up;
}
}

6
runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java
View File

@ -33,7 +33,11 @@ import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/** An ATN transition between any two ATN states. Subclasses define
* atom, set, epsilon, action, predicate, rule transitions.

14
runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java
View File

@ -29,11 +29,19 @@
package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class DFA {
/** A set of all DFA states. Use Map so we can get old state back
@ -96,7 +104,7 @@ public class DFA {
List<Set<ATNState>> atnStates = new ArrayList<Set<ATNState>>();
int i = start;
for (DFAState D : dfaStates) {
Set<ATNState> fullSet = D.configset.getStates();
Set<ATNState> fullSet = D.configs.getStates();
Set<ATNState> statesInvolved = new HashSet<ATNState>();
for (ATNState astate : fullSet) {
Transition t = astate.transition(0);

96
runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java
View File

@ -65,65 +65,50 @@ import java.util.Set;
public class DFAState {
public int stateNumber = -1;
/** The set of ATN configurations (state,alt,context) for this DFA state */
// @Nullable
// public OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>();
public ATNConfigSet configs = new ATNConfigSet();
// TODO: rename to configs after flipping to new ATN sim
public ATNConfigSet configset = new ATNConfigSet();
/** edges[symbol] points to target of symbol */
/** edges[symbol] points to target of symbol. Shift up by 1 so (-1)
* EOF maps to edges[0].
*/
@Nullable
public DFAState[] edges;
public boolean isAcceptState = false;
public int prediction; // if accept state, what ttype do we match? is "else" clause if predicated
/** if accept state, what ttype do we match or alt do we predict?
* This is set to ATN.INVALID_ALT_NUMBER when predicates!=null or
* isCtxSensitive.
*/
public int prediction;
public int lexerRuleIndex = -1; // if accept, exec action in what rule?
public int lexerActionIndex = -1; // if accept, exec what action?
// todo: rename as unique?
public boolean complete; // all alts predict "prediction"
/** Indicates that this state was created during SLL prediction
* that discovered a conflict between the configurations in the state.
* Future execDFA() invocations immediately jumped doing full context
* prediction if this field is true.
*/
public boolean isCtxSensitive;
/** DFA accept states use predicates in two situations:
* disambiguating and validating predicates. If an accept state
* predicts more than one alternative, It's ambiguous and we
* try to resolve with predicates. Disambiguating predicates
* are evaluated when there is a unique prediction for this accept state.
* This array tracks the list of predicates to test in either case;
* there will only be one in the case of a disambiguating predicate.
/** During SLL parsing, this is a list of predicates associated with the
* ATN configurations of the DFA state. When we have predicates,
* isCtxSensitive=false since full context prediction evaluates predicates
* on-the-fly. If this is not null, then this.prediction is
* ATN.INVALID_ALT_NUMBER.
*
* Because there could be 20 alternatives for a decision,
* we don't want to map alt to predicates; we might have to walk
* all of the early alternatives just to get to the predicates.
* We only use these for non isCtxSensitive but conflicting states. That
* means we know from the context (it's $ or we don't dip into outer
* ctx) that it's an ambiguity not a conflict.
*
* If this is null then there are no predicates involved in
* decision-making for this state.
*
* As an example, we might have:
*
* predicates = [(p,3), (q,4), (null, 2)]
*
* This means that there are 2 predicates for 3 ambiguous alternatives.
* If the first 2 predicates fail, then we default to the last
* PredPrediction pair, which predicts alt 2. This comes from:
*
* r : B
* | A
* | {p}? A
* | {q}? A
* ;
*
* This is used only when isCtxSensitive = false;
* This list is computed by predicateDFAState() in ATN simulator.
*/
@Nullable
public List<PredPrediction> predicates;
/** Map a predicate to a predicted alternative */
public static class PredPrediction {
public SemanticContext pred;
public SemanticContext pred; // never null; at least SemanticContext.NONE
public int alt;
public PredPrediction(SemanticContext pred, int alt) {
this.alt = alt;
@ -139,38 +124,31 @@ public class DFAState {
public DFAState(int stateNumber) { this.stateNumber = stateNumber; }
public DFAState(ATNConfigSet configs) { this.configset = configs; }
public DFAState(ATNConfigSet configs) { this.configs = configs; }
/** Get the set of all alts mentioned by all ATN configurations in this
* DFA state.
*/
public Set<Integer> getAltSet() {
// TODO (sam): what to do when configs==null?
Set<Integer> alts = new HashSet<Integer>();
for (ATNConfig c : configset) {
alts.add(c.alt);
if ( configs!=null ) {
for (ATNConfig c : configs) {
alts.add(c.alt);
}
}
if ( alts.isEmpty() ) return null;
return alts;
}
/*
public void setContextSensitivePrediction(RuleContext ctx, int predictedAlt) {
isCtxSensitive = true;
if ( ctxToPrediction==null ) {
ctxToPrediction = new LinkedHashMap<RuleContext, Integer>();
}
ctxToPrediction.put(ctx, predictedAlt);
}
*/
/** A decent hash for a DFA state is the sum of the ATN state/alt pairs. */
@Override
public int hashCode() {
// TODO (sam): what to do when configs==null?
int h = 0;
for (ATNConfig c : configset) {
h += c.alt;
int h = 7;
if ( configs!=null ) {
for (ATNConfig c : configs) {
h = h * 31 ^ c.alt;
h = h * 31 ^ c.state.stateNumber;
}
}
return h;
}
@ -197,7 +175,7 @@ public class DFAState {
DFAState other = (DFAState)o;
// TODO (sam): what to do when configs==null?
boolean sameSet = this.configset.equals(other.configset);
boolean sameSet = this.configs.equals(other.configs);
// System.out.println("DFAState.equals: "+configs+(sameSet?"==":"!=")+other.configs);
return sameSet;
}
@ -205,7 +183,7 @@ public class DFAState {
@Override
public String toString() {
StringBuilder buf = new StringBuilder();
buf.append(stateNumber).append(":").append(configset);
buf.append(stateNumber).append(":").append(configs);
if ( isAcceptState ) {
buf.append("=>");
if ( predicates!=null ) {

330
runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java Normal file
View File

@ -0,0 +1,330 @@
package org.antlr.v4.runtime.misc;
import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
/** Set impl with closed hashing (open addressing). */
public class Array2DHashSet<T> implements EquivalenceSet<T> {
public static final int INITAL_CAPACITY = 16; // must be power of 2
public static final int INITAL_BUCKET_CAPACITY = 8;
public static final double LOAD_FACTOR = 0.75;
protected T[][] buckets;
/** How many elements in set */
protected int n = 0;
protected int threshold = (int)(INITAL_CAPACITY * LOAD_FACTOR); // when to expand
protected int currentPrime = 1; // jump by 4 primes each expand or whatever
protected int initialBucketCapacity = INITAL_BUCKET_CAPACITY;
public Array2DHashSet() {
this(INITAL_CAPACITY, INITAL_BUCKET_CAPACITY);
}
public Array2DHashSet(int initialCapacity, int initialBucketCapacity) {
buckets = (T[][])new Object[initialCapacity][];
this.initialBucketCapacity = initialBucketCapacity;
}
/** Add o to set if not there; return existing value if already there.
* Absorb is used as synonym for add.
*/
public T absorb(T o) {
if ( n > threshold ) expand();
return absorb_(o);
}
protected T absorb_(T o) {
int b = getBucket(o);
T[] bucket = buckets[b];
// NEW BUCKET
if ( bucket==null ) {
buckets[b] = (T[])new Object[initialBucketCapacity];
buckets[b][0] = o;
n++;
return o;
}
// LOOK FOR IT IN BUCKET
for (int i=0; i<bucket.length; i++) {
T existing = bucket[i];
if ( existing==null ) { // empty slot; not there, add.
bucket[i] = o;
n++;
return o;
}
if ( equals(existing, o) ) return existing; // found existing, quit
}
// FULL BUCKET, expand and add to end
T[] old = bucket;
bucket = (T[])new Object[old.length * 2];
buckets[b] = bucket;
System.arraycopy(old, 0, bucket, 0, old.length);
bucket[old.length] = o; // add to end
n++;
return o;
}
public T get(T o) {
if ( o==null ) return o;
int b = getBucket(o);
T[] bucket = buckets[b];
if ( bucket==null ) return null; // no bucket
for (T e : bucket) {
if ( e==null ) return null; // empty slot; not there
if ( equals(e, o) ) return e;
}
return null;
}
protected int getBucket(T o) {
int hash = hashCode(o);
int b = hash & (buckets.length-1); // assumes len is power of 2
return b;
}
@Override
public int hashCode() {
int h = 0;
for (T[] bucket : buckets) {
if ( bucket==null ) continue;
for (T o : bucket) {
if ( o==null ) break;
h += hashCode(o);
}
}
return h;
}
@Override
public boolean equals(Object o) {
if (o == this) return true;
if ( !(o instanceof Array2DHashSet) || o==null ) return false;
Array2DHashSet<T> other = (Array2DHashSet<T>)o;
if ( other.size() != size() ) return false;
boolean same = this.containsAll(other);
return same;
}
protected void expand() {
T[][] old = buckets;
currentPrime += 4;
int newCapacity = buckets.length * 2;
T[][] newTable = (T[][])new Object[newCapacity][];
buckets = newTable;
threshold = (int)(newCapacity * LOAD_FACTOR);
// System.out.println("new size="+newCapacity+", thres="+threshold);
// rehash all existing entries
int oldSize = size();
for (T[] bucket : old) {
if ( bucket==null ) continue;
for (T o : bucket) {
if ( o==null ) break;
absorb_(o);
}
}
n = oldSize;
}
public int hashCode(T o) {
return o.hashCode();
}
public boolean equals(T a, T b) {
return a.equals(b);
}
@Override
public boolean add(T t) {
T existing = absorb(t);
return existing==t;
}
@Override
public int size() {
return n;
}
@Override
public boolean isEmpty() {
return n==0;
}
@Override
public boolean contains(Object o) {
return get((T)o) != null;
}
@Override
public Iterator<T> iterator() {
// return new Iterator<T>() {
// int i = -1;
// @Override
// public boolean hasNext() { return (i+1) < table.length; }
//
// @Override
// public T next() {
// i++;
// if ( i > table.length ) throw new NoSuchElementException();
// while ( table[i]==null ) i++;
// return table[i];
// }
//
// @Override
// public void remove() {
// }
// }
throw new UnsupportedOperationException();
}
@Override
public Object[] toArray() {
Object[] a = new Object[size()];
int i = 0;
for (T[] bucket : buckets) {
if ( bucket==null ) continue;
for (T o : bucket) {
if ( o==null ) break;
a[i++] = o;
}
}
return a;
}
@Override
public <U> U[] toArray(U[] a) {
int i = 0;
for (T[] bucket : buckets) {
if ( bucket==null ) continue;
for (T o : bucket) {
if ( o==null ) break;
a[i++] = (U)o;
}
}
return a;
}
@Override
public boolean remove(Object o) {
if ( o==null ) return false;
int b = getBucket((T)o);
T[] bucket = buckets[b];
if ( bucket==null ) return false; // no bucket
for (int i=0; i<bucket.length; i++) {
T e = bucket[i];
if ( e==null ) return false; // empty slot; not there
if ( equals(e, (T) o) ) { // found it
// shift all elements to the right down one
// for (int j=i; j<bucket.length-1; j++) bucket[j] = bucket[j+1];
System.arraycopy(bucket, i+1, bucket, i, bucket.length-i-1);
n--;
return true;
}
}
return false;
}
@Override
public boolean containsAll(Collection<?> collection) {
if ( collection instanceof Array2DHashSet ) {
Array2DHashSet<T> s = (Array2DHashSet<T>)collection;
for (T[] bucket : s.buckets) {
if ( bucket==null ) continue;
for (T o : bucket) {
if ( o==null ) break;
if ( !this.contains(o) ) return false;
}
}
}
else {
for (Object o : collection) {
if ( !this.contains(o) ) return false;
}
}
return true;
}
@Override
public boolean addAll(Collection<? extends T> c) {
boolean changed = false;
for (T o : c) {
T existing = absorb(o);
if ( existing!=o ) changed=true;
}
return changed;
}
@Override
public boolean retainAll(Collection<?> c) {
throw new UnsupportedOperationException();
}
@Override
public boolean removeAll(Collection<?> c) {
throw new UnsupportedOperationException();
}
@Override
public void clear() {
buckets = (T[][])new Object[INITAL_CAPACITY][];
n = 0;
}
public String toString() {
if ( size()==0 ) return "{}";
StringBuilder buf = new StringBuilder();
buf.append('{');
boolean first = true;
for (T[] bucket : buckets) {
if ( bucket==null ) continue;
for (T o : bucket) {
if ( o==null ) break;
if ( first ) first=false;
else buf.append(", ");
buf.append(o.toString());
}
}
buf.append('}');
return buf.toString();
}
public String toTableString() {
StringBuilder buf = new StringBuilder();
for (T[] bucket : buckets) {
if ( bucket==null ) {
buf.append("null\n");
continue;
}
buf.append('[');
boolean first = true;
for (T o : bucket) {
if ( first ) first=false;
else buf.append(" ");
if ( o==null ) buf.append("_");
else buf.append(o.toString());
}
buf.append("]\n");
}
return buf.toString();
}
public static void main(String[] args) {
Array2DHashSet<String> clset = new Array2DHashSet<String>();
Set<String> set = clset;
set.add("hi");
set.add("mom");
set.add("foo");
set.add("ach");
set.add("cbba");
set.add("d");
set.add("edf");
set.add("f");
set.add("gab");
set.remove("ach");
System.out.println(set);
System.out.println(clset.toTableString());
}
}

7
tool/src/org/antlr/v4/misc/DoubleKeyMap.java → runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java
View File

@ -27,9 +27,12 @@
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.misc;
package org.antlr.v4.runtime.misc;
import java.util.*;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
/** Sometimes we need to map a key to a value but key is two pieces of data.
* This nested hash table saves creating a single key each time we access

7
runtime/Java/src/org/antlr/v4/runtime/misc/EquivalenceRelation.java Normal file
View File

@ -0,0 +1,7 @@
package org.antlr.v4.runtime.misc;
/** What does it mean for two objects to be equivalent? */
public interface EquivalenceRelation<T> {
public int hashCode(T o);
public boolean equals(T a, T b);
}

12
runtime/Java/src/org/antlr/v4/runtime/misc/EquivalenceSet.java Normal file
View File

@ -0,0 +1,12 @@
package org.antlr.v4.runtime.misc;
import java.util.Set;
/** A set that allows us to override equivalence. For a single set, we might
* want multiple subset perspectives as defined by different hash code
* and equivalence methods. HashSet does not allow us to subclass and
* override the equivalence operations, so we have to implement our own
* sets that are flexible in terms of equivalence.
*/
public interface EquivalenceSet<T> extends Set<T>, EquivalenceRelation<T> {
}

107
runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java
View File

@ -39,12 +39,17 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import javax.print.PrintException;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
/** Run a lexer/parser combo, optionally printing tree string or generating
* postscript file. Optionally taking input file.
@ -54,28 +59,31 @@ import java.lang.reflect.Method;
* [-tokens] [-gui] [-ps file.ps]
* [-trace]
* [-diagnostics]
* [input-filename]
* [-SLL]
* [input-filename(s)]
*/
public class TestRig {
static String grammarName;
static String startRuleName;
static List<String> inputFiles = new ArrayList<String>();
static boolean printTree = false;
static boolean gui = false;
static String psFile = null;
static boolean showTokens = false;
static boolean trace = false;
static boolean diagnostics = false;
static String encoding = null;
static boolean SLL = false;
public static final String LEXER_START_RULE_NAME = "tokens";
public static void main(String[] args) throws Exception {
String grammarName;
String startRuleName;
String inputFile = null;
boolean printTree = false;
boolean gui = false;
String psFile = null;
boolean showTokens = false;
boolean trace = false;
boolean diagnostics = false;
String encoding = null;
if ( args.length < 2 ) {
System.err.println("java org.antlr.v4.runtime.misc.TestRig GrammarName startRuleName\n" +
" [-tokens] [-tree] [-gui] [-ps file.ps] [-encoding encodingname]\n" +
" [-trace] [-diagnostics]\n"+
" [input-filename]");
" [-trace] [-diagnostics] [-SLL]\n"+
" [input-filename(s)]");
System.err.println("Use startRuleName='tokens' if GrammarName is a lexer grammar.");
System.err.println("Omitting input-filename makes rig read from stdin.");
return;
@ -89,7 +97,7 @@ public class TestRig {
String arg = args[i];
i++;
if ( arg.charAt(0)!='-' ) { // input file name
inputFile = arg;
inputFiles.add(arg);
continue;
}
if ( arg.equals("-tree") ) {
@ -104,6 +112,9 @@ public class TestRig {
else if ( arg.equals("-trace") ) {
trace = true;
}
else if ( arg.equals("-SLL") ) {
SLL = true;
}
else if ( arg.equals("-diagnostics") ) {
diagnostics = true;
}
@ -141,23 +152,53 @@ public class TestRig {
return;
}
InputStream is = System.in;
if ( inputFile!=null ) {
is = new FileInputStream(inputFile);
}
Reader r;
if ( encoding!=null ) {
r = new InputStreamReader(is, encoding);
}
else {
r = new InputStreamReader(is);
Constructor<Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
Lexer lexer = lexerCtor.newInstance((CharStream)null);
String parserName = grammarName+"Parser";
Class parserClass = cl.loadClass(parserName);
if ( parserClass==null ) {
System.err.println("Can't load "+parserName);
}
Constructor<Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
Parser parser = parserCtor.newInstance((TokenStream)null);
if ( inputFiles.size()==0 ) {
InputStream is = System.in;
Reader r;
if ( encoding!=null ) {
r = new InputStreamReader(is, encoding);
}
else {
r = new InputStreamReader(is);
}
process(lexer, parserClass, parser, is, r);
return;
}
for (String inputFile : inputFiles) {
InputStream is = System.in;
if ( inputFile!=null ) {
is = new FileInputStream(inputFile);
}
Reader r;
if ( encoding!=null ) {
r = new InputStreamReader(is, encoding);
}
else {
r = new InputStreamReader(is);
}
if ( inputFiles.size()>1 ) {
System.err.println(inputFile);
}
process(lexer, parserClass, parser, is, r);
}
}
static void process(Lexer lexer, Class parserClass, Parser parser, InputStream is, Reader r) throws IOException, IllegalAccessException, InvocationTargetException, PrintException {
try {
ANTLRInputStream input = new ANTLRInputStream(r);
Constructor<Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
Lexer lexer = lexerCtor.newInstance(input);
lexer.setInputStream(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
@ -170,13 +211,6 @@ public class TestRig {
if ( startRuleName.equals(LEXER_START_RULE_NAME) ) return;
String parserName = grammarName+"Parser";
Class parserClass = cl.loadClass(parserName);
if ( parserClass==null ) {
System.err.println("Can't load "+parserName);
}
Constructor<Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
Parser parser = parserCtor.newInstance(tokens);
if ( diagnostics ) parser.addErrorListener(new DiagnosticErrorListener());
@ -184,6 +218,11 @@ public class TestRig {
parser.setBuildParseTree(true);
}
if ( SLL ) {
parser.getInterpreter().setSLL(true);
}
parser.setTokenStream(tokens);
parser.setTrace(trace);
try {

11
runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java
View File

@ -29,6 +29,10 @@
package org.antlr.v4.runtime.misc;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.util.Collection;
import java.util.Iterator;
@ -70,4 +74,11 @@ public class Utils {
}
return buf.toString();
}
public static void writeFile(String fileName, String content) throws IOException {
FileWriter fw = new FileWriter(fileName);
Writer w = new BufferedWriter(fw);
w.write(content);
w.close();
}
}

3
runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java
View File

@ -30,9 +30,6 @@
package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.Interval;
/** An interface to access the tree of RuleContext objects created
* during a parse that makes the data structure look like a simple parse tree.

23
runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java
View File

@ -3,9 +3,19 @@ package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
/** {@code T} is return type of {@code visit} methods. Use {@link Void} for no return type.
/** This class defines the basic notion of a parse tree visitor
* object. Generated visitors extend this class and implement the XVisitor
* interface for grammar X.
*
* @param <T> The return type of the visit operation. Use {@link Void} for
* operations with no return type.
*/
public class ParseTreeVisitor<T> {
/** Visit a parse tree, and return a user-defined result of the operation.
*
* @param tree The {@link ParseTree} to visit.
* @return The result of visiting the parse tree.
*/
public T visit(ParseTree tree) {
return tree.accept(this);
}
@ -27,6 +37,17 @@ public class ParseTreeVisitor<T> {
return result;
}
/** Visit a terminal node, and return a user-defined result of the operation.
*
* @param node The {@link TerminalNode} to visit.
* @return The result of visiting the node.
*/
public T visitTerminal(TerminalNode<? extends Token> node) { return null; }
/** Visit an error node, and return a user-defined result of the operation.
*
* @param node The {@link ErrorNode} to visit.
* @return The result of visiting the node.
*/
public T visitErrorNode(ErrorNode<? extends Token> node) { return null; }
}

104
runtime/Java/src/org/antlr/v4/runtime/tree/TraceTree.java
View File

@ -1,104 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.atn.ATNState;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
public class TraceTree implements Tree {
public TraceTree parent;
public List<TraceTree> children;
/** If this node is root, it has list of all leaves of tree after ParserATNPathFinder.trace() */
public List<TraceTree> leaves;
public ATNState state; // payload
public TraceTree(ATNState s) { state = s; }
public void addChild(TraceTree t) {
if ( children==null ) {
children = new ArrayList<TraceTree>();
}
children.add(t);
t.parent = this;
}
public void addChild(ATNState s) { addChild(new TraceTree(s)); }
@Override
public Tree getChild(int i) {
if ( children==null ) {
throw new IndexOutOfBoundsException(i+"<0 or >"+getChildCount());
}
return children.get(i);
}
@Override
public Tree getParent() {
return parent;
}
@Override
public Object getPayload() {
return state;
}
@Override
public int getChildCount() {
if ( children==null ) return 0;
return children.size();
}
public List<ATNState> getPathToNode(TraceTree s) {
List<ATNState> states = new LinkedList<ATNState>();
TraceTree p = s;
while ( p!=null ) {
states.add(0, p.state);
p = p.parent;
}
if ( states.isEmpty() ) return null;
return states;
}
@Override
public String toString() {
if ( state==null ) return "null";
return state.toString();
}
@Override
public String toStringTree() {
return Trees.toStringTree(this, null);
}
}

42
runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java
View File

@ -32,6 +32,7 @@ package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.runtime.tree.gui.TreePostScriptGenerator;
@ -39,6 +40,7 @@ import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@ -75,31 +77,55 @@ public class Trees {
writePS(t, recog, fileName, "Helvetica", 11);
}
/** Print out a whole tree in LISP form. getNodeText is used on the
/** Print out a whole tree in LISP form. {@link #getNodeText} is used on the
* node payloads to get the text for the nodes. Detect
* parse trees and extract data appropriately.
*/
public static String toStringTree(Tree t, Parser recog) {
String s = Utils.escapeWhitespace(getNodeText(t, recog), false);
public static String toStringTree(@NotNull Tree t) {
return toStringTree(t, (List<String>)null);
}
/** Print out a whole tree in LISP form. {@link #getNodeText} is used on the
* node payloads to get the text for the nodes. Detect
* parse trees and extract data appropriately.
*/
public static String toStringTree(@NotNull Tree t, @Nullable Parser recog) {
String[] ruleNames = recog != null ? recog.getRuleNames() : null;
List<String> ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null;
return toStringTree(t, ruleNamesList);
}
/** Print out a whole tree in LISP form. {@link #getNodeText} is used on the
* node payloads to get the text for the nodes. Detect
* parse trees and extract data appropriately.
*/
public static String toStringTree(@NotNull Tree t, @Nullable List<String> ruleNames) {
String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false);
if ( t.getChildCount()==0 ) return s;
StringBuilder buf = new StringBuilder();
buf.append("(");
s = Utils.escapeWhitespace(getNodeText(t, recog), false);
s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false);
buf.append(s);
buf.append(' ');
for (int i = 0; i<t.getChildCount(); i++) {
if ( i>0 ) buf.append(' ');
buf.append(toStringTree(t.getChild(i), recog));
buf.append(toStringTree(t.getChild(i), ruleNames));
}
buf.append(")");
return buf.toString();
}
public static String getNodeText(Tree t, Parser recog) {
if ( recog!=null ) {
public static String getNodeText(@NotNull Tree t, @Nullable Parser recog) {
String[] ruleNames = recog != null ? recog.getRuleNames() : null;
List<String> ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null;
return getNodeText(t, ruleNamesList);
}
public static String getNodeText(@NotNull Tree t, @Nullable List<String> ruleNames) {
if ( ruleNames!=null ) {
if ( t instanceof RuleNode ) {
int ruleIndex = ((RuleNode)t).getRuleContext().getRuleIndex();
String ruleName = recog.getRuleNames()[ruleIndex];
String ruleName = ruleNames.get(ruleIndex);
return ruleName;
}
else if ( t instanceof ErrorNode) {

4
tool/playground/A.g4
View File

@ -2,8 +2,8 @@ grammar A;
s : e ;
e : e '*' e -> Mult
| INT -> primary
e : e '*' e # Mult
| INT # primary
;
INT : [0-9]+ ;

2
tool/playground/AA.g
View File

@ -14,7 +14,7 @@ member
stat: expr ';'
{System.out.println("found expr: "+$stat.text);}
| ID '=' expr ';'
{System.out.println("found assign: "+$stat.text);}
{System.out.println("found assign: "+$stat.text+$ID.text);}
;
expr: INT

11
tool/playground/Bar.java Normal file
View File

@ -0,0 +1,11 @@
class Bar {
private int bitsOrSingle(int bits, int ch) {
boolean d =
!(3==4 &&
(ch == 0xff || ch == 0xb5 ||
// ch == 0x53 || ch == 0x73 ||
// ch == 0x4b || ch == 0x6b ||
ch == 0xc5 || ch == 0xe5));
return 9;
}
}

54
tool/playground/ExtractInheritance.java Normal file
View File

@ -0,0 +1,54 @@
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.util.List;
public class ExtractInheritance extends JavaLRBaseListener {
JavaLRParser parser;
public ExtractInheritance(JavaLRParser parser) { this.parser = parser; }
/*
normalClassDeclaration
: 'class' Identifier typeParameters?
('extends' type)?
('implements' typeList)?
classBody
;
*/
@Override
public void enterNormalClassDeclaration(JavaLRParser.NormalClassDeclarationContext ctx) {
TerminalNode<Token> id = ctx.Identifier();
String sup = null;
if ( ctx.type()!=null ) {
sup = ctx.type().getText();
System.out.println("\""+id+"\" -> \""+sup+"\"");
}
if ( ctx.typeList()!=null ) {
List<? extends JavaLRParser.TypeContext> type = ctx.typeList().type();
for (JavaLRParser.TypeContext t : type) {
System.out.println("\""+id+"\" -> \""+t.getText()+"\"");
}
}
}
/*
normalInterfaceDeclaration
: 'interface' Identifier typeParameters? ('extends' typeList)? interfaceBody
;
*/
@Override
public void enterNormalInterfaceDeclaration(JavaLRParser.NormalInterfaceDeclarationContext ctx) {
TerminalNode<Token> id = ctx.Identifier();
System.out.println("###### interface "+id);
String args = null;
if ( ctx.typeList()!=null ) {
List<? extends JavaLRParser.TypeContext> type = ctx.typeList().type();
for (JavaLRParser.TypeContext t : type) {
System.out.println("\""+id+"\" -> \""+t.getText()+"\"");
}
}
}
}

14
tool/playground/Foo.java Normal file
View File

@ -0,0 +1,14 @@
class Foo {
private int bitsOrSingle(int bits, int ch) {
int d;
if (ch < 256 &&
!(3==4 && 5==6 &&
(ch == 0xff || ch == 0xb5 ||
ch == 0x49 || ch == 0x69 || //I and i // cmt out and it continues!
ch == 0x53 || ch == 0x73 || //S and s
ch == 0x4b || ch == 0x6b || //K and k
ch == 0xc5 || ch == 0xe5))) //A+ring
return 0;
return 9;
}
}

61
tool/playground/GenHierarchy.java Normal file
View File

@ -0,0 +1,61 @@
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
public class GenHierarchy {
public static void main(String[] args) throws Exception {
// START: input
String inputFile = null;
if ( args.length>0 ) inputFile = args[0];
List<String> files = getFilenames(new File(inputFile));
for (String file : files) {
InputStream is = new FileInputStream(file);
ANTLRInputStream input = new ANTLRInputStream(is);
// END: input
// System.out.println(file);
// START: launch
JavaLRLexer lexer = new JavaLRLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
JavaLRParser parser = new JavaLRParser(tokens);
ParserRuleContext<Token> tree = parser.compilationUnit(); // parse
ParseTreeWalker walker = new ParseTreeWalker(); // create standard walker
ExtractInheritance extractor = new ExtractInheritance(parser);
walker.walk(extractor, tree); // initiate walk of tree with listener
}
// END: launch
}
public static List<String> getFilenames(File f) throws Exception {
List<String> files = new ArrayList<String>();
getFilenames_(f, files);
return files;
}
public static void getFilenames_(File f, List<String> files) throws Exception {
// If this is a directory, walk each file/dir in that directory
if (f.isDirectory()) {
String flist[] = f.list();
for(int i=0; i < flist.length; i++) {
getFilenames_(new File(f, flist[i]), files);
}
}
// otherwise, if this is a java file, parse it!
else if ( ((f.getName().length()>5) &&
f.getName().substring(f.getName().length()-5).equals(".java")) )
{
files.add(f.getAbsolutePath());
}
}
}

14
tool/playground/LL1.g
View File

@ -1,14 +0,0 @@
grammar LL1;
b : B | C ;
c1 : A? B ;
c2 : (B|C)? D ;
d1 : A* B ;
d2 : ({true}? B | {true}? A)* D {System.out.println("works!");} ;
e1 : A+ B ;
e2 : (B|A)+ D ;

36
tool/playground/T.g
View File

@ -1,34 +1,2 @@
grammar T;
s : stat ;
stat: expr[0] NEWLINE
| ID '=' expr[0] NEWLINE
| NEWLINE
;
expr[int _p]
: ( INT
| ID
| '(' expr[0] ')'
)
( {5 >= $_p}? ('*'|'/') expr[6]
| {4 >= $_p}? ('+'|'-') expr[5]
)*
;
/*
expr: expr ('*'|'/') expr # MulDiv
| expr ('+'|'-') expr # AddSub
| INT # int
| ID # id
| '(' expr ')' # parens
;
*/
MUL : '*' ; // assigns token name to '*' used above in grammar
DIV : '/' ;
ADD : '+' ;
SUB : '-' ;
ID : [a-zA-Z]+ ; // match identifiers
INT : [0-9]+ ; // match integers
NEWLINE:'\r'? '\n' ; // return newlines to parser (is end-statement signal)
WS : [ \t]+ -> skip ; // toss out whitespace
lexer grammar T;
A : 'a';

19
tool/playground/TestJava.java
View File

@ -1,6 +1,9 @@
import org.antlr.runtime.debug.BlankDebugEventListener;
import org.antlr.v4.runtime.ANTLRFileStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.DiagnosticErrorListener;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
@ -14,6 +17,10 @@ class TestJava {
public static boolean profile = false;
public static JavaLexer lexer;
public static JavaParser parser = null;
public static boolean showTree = false;
public static boolean printTree = false;
public static boolean SLL = false;
public static boolean diag = false;
public static void main(String[] args) {
doAll(args);
@ -28,6 +35,10 @@ class TestJava {
if (args.length > 0 ) {
// for each directory/file specified on the command line
for(int i=0; i< args.length;i++) {
if ( args[i].equals("-tree") ) showTree = true;
else if ( args[i].equals("-ptree") ) printTree = true;
else if ( args[i].equals("-SLL") ) SLL = true;
else if ( args[i].equals("-diag") ) diag = true;
doFile(new File(args[i])); // parse it
}
}
@ -114,8 +125,14 @@ class TestJava {
// parser.getInterpreter().setContextSensitive(true);
}
parser.setTokenStream(tokens);
if ( diag ) parser.addErrorListener(new DiagnosticErrorListener());
if ( SLL ) parser.getInterpreter().setSLL(true);
// start parsing at the compilationUnit rule
parser.compilationUnit();
ParserRuleContext<Token> tree = parser.compilationUnit();
if ( showTree ) tree.inspect(parser);
if ( printTree ) System.out.println(tree.toStringTree(parser));
//System.err.println("finished "+f);
// System.out.println("cache size = "+DefaultErrorStrategy.cache.size());
}

259
tool/playground/TestJavaLR.java
View File

@ -27,132 +27,251 @@
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
import org.antlr.runtime.debug.BlankDebugEventListener;
import org.antlr.v4.runtime.ANTLRFileStream;
import org.antlr.v4.runtime.BailErrorStrategy;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.DiagnosticErrorListener;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
class TestJavaLR {
public static long lexerTime = 0;
// public static long lexerTime = 0;
public static boolean profile = false;
public static JavaLRLexer lexer;
public static JavaLRParser parser = null;
public static boolean showTree = false;
public static boolean notree = false;
public static boolean gui = false;
public static boolean printTree = false;
public static boolean SLL = false;
public static boolean diag = false;
public static boolean bail = false;
public static boolean x2 = false;
public static boolean threaded = false;
public static boolean quiet = false;
// public static long parserStart;
// public static long parserStop;
public static Worker[] workers = new Worker[3];
static int windex = 0;
public static CyclicBarrier barrier;
public static volatile boolean firstPassDone = false;
public static class Worker implements Runnable {
public long parserStart;
public long parserStop;
List<String> files;
public Worker(List<String> files) {
this.files = files;
}
@Override
public void run() {
parserStart = System.currentTimeMillis();
for (String f : files) {
parseFile(f);
}
parserStop = System.currentTimeMillis();
try {
barrier.await();
}
catch (InterruptedException ex) {
return;
}
catch (BrokenBarrierException ex) {
return;
}
}
}
public static void main(String[] args) {
doAll(args);
// doAll(args);
}
public static void doAll(String[] args) {
List<String> inputFiles = new ArrayList<String>();
long start = System.currentTimeMillis();
try {
lexerTime = 0;
long start = System.currentTimeMillis();
if (args.length > 0 ) {
// for each directory/file specified on the command line
for(int i=0; i< args.length;i++) {
if ( args[i].equals("-tree") ) showTree = true;
doFile(new File(args[i])); // parse it
if ( args[i].equals("-notree") ) notree = true;
else if ( args[i].equals("-gui") ) gui = true;
else if ( args[i].equals("-ptree") ) printTree = true;
else if ( args[i].equals("-SLL") ) SLL = true;
else if ( args[i].equals("-bail") ) bail = true;
else if ( args[i].equals("-diag") ) diag = true;
else if ( args[i].equals("-2x") ) x2 = true;
else if ( args[i].equals("-threaded") ) threaded = true;
else if ( args[i].equals("-quiet") ) quiet = true;
if ( args[i].charAt(0)!='-' ) { // input file name
inputFiles.add(args[i]);
}
}
List<String> javaFiles = new ArrayList<String>();
for (String fileName : inputFiles) {
List<String> files = getFilenames(new File(fileName));
javaFiles.addAll(files);
}
doFiles(javaFiles);
// DOTGenerator gen = new DOTGenerator(null);
// String dot = gen.getDOT(JavaLRParser._decisionToDFA[112], false);
// System.out.println(dot);
// dot = gen.getDOT(JavaLRParser._decisionToDFA[81], false);
// System.out.println(dot);
if ( x2 ) {
System.gc();
System.out.println("waiting for 1st pass");
if ( threaded ) while ( !firstPassDone ) { } // spin
System.out.println("2nd pass");
doFiles(javaFiles);
}
}
else {
System.err.println("Usage: java Main <directory or file name>");
}
long stop = System.currentTimeMillis();
System.out.println("Lexer total time " + lexerTime + "ms.");
System.out.println("Total time " + (stop - start) + "ms.");
System.out.println("finished parsing OK");
System.out.println(LexerATNSimulator.ATN_failover+" lexer failovers");
System.out.println(LexerATNSimulator.match_calls+" lexer match calls");
System.out.println(ParserATNSimulator.ATN_failover+" parser failovers");
System.out.println(ParserATNSimulator.predict_calls +" parser predict calls");
System.out.println(ParserATNSimulator.retry_with_context +" retry_with_context after SLL conflict");
System.out.println(ParserATNSimulator.retry_with_context_indicates_no_conflict +" retry sees no conflict");
if ( profile ) {
System.out.println("num decisions "+profiler.numDecisions);
}
}
catch(Exception e) {
System.err.println("exception: "+e);
e.printStackTrace(System.err); // so we can get stack trace
}
long stop = System.currentTimeMillis();
// System.out.println("Overall time " + (stop - start) + "ms.");
System.gc();
}
public static void doFiles(List<String> files) throws Exception {
long parserStart = System.currentTimeMillis();
// lexerTime = 0;
if ( threaded ) {
barrier = new CyclicBarrier(3,new Runnable() {
public void run() {
report(); firstPassDone = true;
}
});
int chunkSize = files.size() / 3; // 10/3 = 3
int p1 = chunkSize; // 0..3
int p2 = 2 * chunkSize; // 4..6, then 7..10
workers[0] = new Worker(files.subList(0,p1+1));
workers[1] = new Worker(files.subList(p1+1,p2+1));
workers[2] = new Worker(files.subList(p2+1,files.size()));
new Thread(workers[0], "worker-"+windex++).start();
new Thread(workers[1], "worker-"+windex++).start();
new Thread(workers[2], "worker-"+windex++).start();
}
else {
for (String f : files) {
parseFile(f);
}
long parserStop = System.currentTimeMillis();
System.out.println("Total lexer+parser time " + (parserStop - parserStart) + "ms.");
}
}
// This method decides what action to take based on the type of
// file we are looking at
public static void doFile(File f) throws Exception {
private static void report() {
// parserStop = System.currentTimeMillis();
// System.out.println("Lexer total time " + lexerTime + "ms.");
long time = 0;
if ( workers!=null ) {
// compute max as it's overlapped time
for (Worker w : workers) {
long wtime = w.parserStop - w.parserStart;
time = Math.max(time,wtime);
System.out.println("worker time " + wtime + "ms.");
}
}
System.out.println("Total lexer+parser time " + time + "ms.");
System.out.println("finished parsing OK");
System.out.println(LexerATNSimulator.ATN_failover+" lexer failovers");
System.out.println(LexerATNSimulator.match_calls+" lexer match calls");
System.out.println(ParserATNSimulator.ATN_failover+" parser failovers");
System.out.println(ParserATNSimulator.predict_calls +" parser predict calls");
System.out.println(ParserATNSimulator.retry_with_context +" retry_with_context after SLL conflict");
System.out.println(ParserATNSimulator.retry_with_context_indicates_no_conflict +" retry sees no conflict");
System.out.println(ParserATNSimulator.retry_with_context_predicts_same_as_alt +" retry predicts same alt as resolving conflict");
System.out.println(ParserATNSimulator.retry_with_context_from_dfa +" retry from DFA");
}
public static List<String> getFilenames(File f) throws Exception {
List<String> files = new ArrayList<String>();
getFilenames_(f, files);
return files;
}
public static void getFilenames_(File f, List<String> files) throws Exception {
// If this is a directory, walk each file/dir in that directory
if (f.isDirectory()) {
String files[] = f.list();
for(int i=0; i < files.length; i++)
doFile(new File(f, files[i]));
String flist[] = f.list();
for(int i=0; i < flist.length; i++) {
getFilenames_(new File(f, flist[i]), files);
}
}
// otherwise, if this is a java file, parse it!
else if ( ((f.getName().length()>5) &&
f.getName().substring(f.getName().length()-5).equals(".java"))
|| f.getName().equals("input") )
f.getName().substring(f.getName().length()-5).equals(".java")) )
{
System.err.println(f.getAbsolutePath());
parseFile(f.getAbsolutePath());
files.add(f.getAbsolutePath());
}
}
static class CountDecisions extends BlankDebugEventListener {
public int numDecisions = 0;
public void enterDecision(int decisionNumber) {
numDecisions++;
}
}
static CountDecisions profiler = new CountDecisions();
// This method decides what action to take based on the type of
// file we are looking at
// public static void doFile_(File f) throws Exception {
// // If this is a directory, walk each file/dir in that directory
// if (f.isDirectory()) {
// String files[] = f.list();
// for(int i=0; i < files.length; i++) {
// doFile_(new File(f, files[i]));
// }
// }
//
// // otherwise, if this is a java file, parse it!
// else if ( ((f.getName().length()>5) &&
// f.getName().substring(f.getName().length()-5).equals(".java")) )
// {
// System.err.println(f.getAbsolutePath());
// parseFile(f.getAbsolutePath());
// }
// }
// Here's where we do the real work...
public static void parseFile(String f)
throws Exception {
public static void parseFile(String f) {
try {
if ( !quiet ) System.err.println(f);
// Create a scanner that reads from the input stream passed to us
if ( lexer==null ) {
lexer = new JavaLRLexer(null);
}
lexer.setInputStream(new ANTLRFileStream(f));
Lexer lexer = new JavaLRLexer(new ANTLRFileStream(f));
CommonTokenStream tokens = new CommonTokenStream(lexer);
long start = System.currentTimeMillis();
tokens.fill(); // load all and check time
// System.out.println(tokens.getTokens());
long stop = System.currentTimeMillis();
lexerTime += stop-start;
// long start = System.currentTimeMillis();
// tokens.fill(); // load all and check time
// long stop = System.currentTimeMillis();
// lexerTime += stop-start;
if ( true ) {
// Create a parser that reads from the scanner
if ( parser==null ) {
parser = new JavaLRParser(null);
if ( showTree ) parser.setBuildParseTree(true);
// parser.setErrorHandler(new BailErrorStrategy<Token>());
// parser.getInterpreter().setContextSensitive(true);
}
// Create a parser that reads from the scanner
JavaLRParser parser = new JavaLRParser(tokens);
if ( diag ) parser.addErrorListener(new DiagnosticErrorListener());
if ( bail ) parser.setErrorHandler(new BailErrorStrategy());
if ( SLL ) parser.getInterpreter().setSLL(true);
parser.setTokenStream(tokens);
// start parsing at the compilationUnit rule
ParserRuleContext<Token> tree = parser.compilationUnit();
if ( showTree ) tree.inspect(parser);
//System.err.println("finished "+f);
// System.out.println("cache size = "+DefaultErrorStrategy.cache.size());
}
// start parsing at the compilationUnit rule
ParserRuleContext<Token> t = parser.compilationUnit();
if ( notree ) parser.setBuildParseTree(false);
if ( gui ) t.inspect(parser);
if ( printTree ) System.out.println(t.toStringTree(parser));
}
catch (Exception e) {
System.err.println("parser exception: "+e);
e.printStackTrace(); // so we can get stack trace
}
}
}

4
tool/playground/TestT.java
View File

@ -26,7 +26,11 @@ public class TestT {
parser.addErrorListener(new DiagnosticErrorListener());
// parser.getInterpreter().setSLL(true);
// parser.setTrace(true);
ParserRuleContext tree = parser.s();
System.out.println(tree.toStringTree(parser));
// tree.save(parser, "/tmp/t.ps");
}
}

33
tool/playground/U.g
View File

@ -1,32 +1,5 @@
grammar U;
@members {public static boolean java5 = true;}
prog: ( enumDecl
| stat
)*
EOF
;
enumDecl
: {java5}? 'enum' ID '{' ID (',' ID)* '}'
;
args
: arg (',' arg )*
;
arg
: INT
;
stat: ID '=' expr ';' ;
expr: ID {System.out.println("ID "+$ID.text);}
| {!java5}? 'enum' {System.out.println("ID enum");}
| INT
;
ID : [a-zA-Z]+ ;
INT : [0-9]+ ;
WS : [ \t\n\r]+ -> skip ;
a : X ;
X : 'a' -> skip ;
Y : 'z' -> skip, more ;

61
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -59,36 +59,6 @@ public class <file.grammarName>BaseListener implements <file.grammarName>Listene
}
>>
ParseListenerFile(file, header) ::= <<
<header>
import org.antlr.v4.runtime.tree.*;
import org.antlr.v4.runtime.*;
public interface <file.grammarName>ParseListener extends ParseListener\<<InputSymbolType()>\> {
<file.listenerEnterNames:{lname |
void enter<lname; format="cap">(ParserRuleContext\<<InputSymbolType()>\> ctx);}; separator="\n">
<file.listenerExitNames:{lname |
void exit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx);}; separator="\n">
}
>>
BaseParseListenerFile(file, header) ::= <<
<header>
import org.antlr.v4.runtime.*;
public class <file.grammarName>BaseParseListener implements <file.grammarName>ParseListener {
<file.listenerEnterNames:{lname |
@Override public void enter<lname; format="cap">(ParserRuleContext\<<InputSymbolType()>\> ctx) { \}}; separator="\n">
<file.listenerExitNames:{lname |
@Override public void exit<lname; format="cap">(<file.parserName>.<lname; format="cap">Context ctx) { \}}; separator="\n">
@Override public void enterNonLRRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
@Override public void exitEveryRule(ParserRuleContext\<<InputSymbolType()>\> ctx) { }
@Override public void visitTerminal(ParserRuleContext\<<InputSymbolType()>\> ctx, <InputSymbolType()> symbol) { }
}
>>
VisitorFile(file, header) ::= <<
<header>
import org.antlr.v4.runtime.tree.*;
@ -112,13 +82,16 @@ public class <file.grammarName>BaseVisitor\<T> extends ParseTreeVisitor\<T> impl
}
>>
Parser(parser, funcs, atn, sempredFuncs, superclass) ::= <<
Parser(parser, funcs, atn, sempredFuncs, superClass) ::= <<
<Parser_(ctor="parser_ctor", ...)>
>>
Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= <<
Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superClass) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public <if(parser.abstractRecognizer)>abstract <endif>class <parser.name> extends <superclass> {
public class <parser.name> extends <superClass> {
protected static final DFA[] _decisionToDFA;
protected static final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
<if(parser.tokens)>
public static final int
<parser.tokens:{k | <k>=<parser.tokens.(k)>}; separator=", ", wrap, anchor>;
@ -191,7 +164,7 @@ case <f.ruleIndex> : return <f.name>_sempred((<f.ctxType>)_localctx, predIndex);
parser_ctor(p) ::= <<
public <p.name>(TokenStream input) {
super(input);
_interp = new ParserATNSimulator\<Token>(this,_ATN);
_interp = new ParserATNSimulator\<Token>(this,_ATN,_decisionToDFA,_sharedContextCache);
}
>>
@ -511,6 +484,7 @@ LexerModeCommand(arg) ::= "_mode = <arg>;"
LexerPushModeCommand(arg) ::= "pushMode(<arg>);"
DefaultParserSuperClass(s) ::= "Parser"
DefaultLexerSuperClass(s) ::= "Lexer"
ActionText(t) ::= "<t.text>"
ActionTemplate(t) ::= "<t.st>"
@ -552,7 +526,7 @@ SetNonLocalAttr(s, rhsChunks) ::=
AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.add(<labelref(a.label)>);"
TokenDecl(t) ::= "public Token <t.name>;"
TokenDecl(t) ::= "public <TokenLabelType()> <t.name>;"
TokenTypeDecl(t) ::= "int <t.name>;"
TokenListDecl(t) ::= "public List\<Token> <t.name> = new ArrayList\<Token>();"
RuleContextDecl(r) ::= "public <r.ctxName> <r.name>;"
@ -645,13 +619,6 @@ public \<T> T accept(ParseTreeVisitor\<? extends T> visitor) {
}
>>
ParseListenerDispatchMethod(method) ::= <<
@Override
public void <if(method.isEnter)>enter<else>exit<endif>Rule(ParseListener\<<InputSymbolType()>\> listener) {
if ( listener instanceof <parser.grammarName>ParseListener ) ((<parser.grammarName>ParseListener)listener).<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(this);
}
>>
AttributeDecl(d) ::= "public <d.decl>;"
/** If we don't know location of label def x, use this template */
@ -712,9 +679,12 @@ import org.antlr.v4.runtime.misc.*;
<lexer>
>>
Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= <<
Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends Lexer {
public class <lexer.name> extends <superClass> {
protected static final DFA[] _decisionToDFA;
protected static final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
public static final int
<lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>;
<rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n">
@ -734,7 +704,7 @@ public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends
public <lexer.name>(CharStream input) {
super(input);
_interp = new LexerATNSimulator(this,_ATN);
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
}
@Override
@ -766,6 +736,7 @@ public static final String _serializedATN =
public static final ATN _ATN =
ATNSimulator.deserialize(_serializedATN.toCharArray());
static {
_decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
//org.antlr.v4.tool.DOTGenerator dot = new org.antlr.v4.tool.DOTGenerator(null);
//System.out.println(dot.getDOT(_ATN.decisionToState.get(0), ruleNames, false));
//System.out.println(dot.getDOT(_ATN.ruleToStartState[2], ruleNames, false));

154
tool/src/org/antlr/v4/Tool.java
View File

@ -29,6 +29,7 @@
package org.antlr.v4;
import org.antlr.misc.Graph;
import org.antlr.runtime.ANTLRFileStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
@ -64,20 +65,22 @@ import org.stringtemplate.v4.STGroup;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
public class Tool {
public String VERSION = "4.0-"+new Date();
public static final String getVersion() { return "4.0b1"; }
public static final String GRAMMAR_EXTENSION = ".g4";
public static final String LEGACY_GRAMMAR_EXTENSION = ".g";
@ -122,27 +125,27 @@ public class Tool {
public boolean log = false;
public boolean verbose_dfa = false;
public boolean gen_listener = true;
public boolean gen_parse_listener = false;
public boolean gen_visitor = false;
public boolean abstract_recognizer = false;
public Map<String, String> grammarOptions = null;
public static Option[] optionDefs = {
new Option("outputDirectory", "-o", OptionArgType.STRING, "specify output directory where all output is generated"),
new Option("libDirectory", "-lib", OptionArgType.STRING, "specify location of grammars, tokens files"),
/*
new Option("report", "-report", "print out a report about the grammar(s) processed"),
new Option("printGrammar", "-print", "print out the grammar without actions"),
new Option("debug", "-debug", "generate a parser that emits debugging events"),
new Option("profile", "-profile", "generate a parser that computes profiling information"),
*/
new Option("generate_ATN_dot", "-atn", "generate rule augmented transition network diagrams"),
new Option("grammarEncoding", "-encoding", OptionArgType.STRING, "specify grammar file encoding; e.g., euc-jp"),
new Option("msgFormat", "-message-format", OptionArgType.STRING, "specify output style for messages"),
new Option("msgFormat", "-message-format", OptionArgType.STRING, "specify output style for messages in antlr, gnu, vs2005"),
new Option("gen_listener", "-listener", "generate parse tree listener (default)"),
new Option("gen_listener", "-no-listener", "don't generate parse tree listener"),
new Option("gen_parse_listener", "-parse-listener", "generate parse listener"),
new Option("gen_parse_listener", "-no-parse-listener", "don't generate parse listener (default)"),
new Option("gen_visitor", "-visitor", "generate parse tree visitor"),
new Option("gen_visitor", "-no-visitor", "don't generate parse tree visitor (default)"),
new Option("abstract_recognizer", "-abstract", "generate abstract recognizer classes"),
new Option("", "-D<option>=value", "set/override a grammar-level option"),
new Option("saveLexer", "-Xsave-lexer", "save temp lexer file created for combined grammars"),
new Option("launch_ST_inspector", "-XdbgST", "launch StringTemplate visualizer on generated code"),
@ -164,7 +167,7 @@ public class Tool {
protected List<String> grammarFiles = new ArrayList<String>();
public ErrorManager errMgr = new ErrorManager(this);
public ErrorManager errMgr;
public LogManager logMgr = new LogManager();
List<ANTLRToolListener> listeners = new CopyOnWriteArrayList<ANTLRToolListener>();
@ -204,6 +207,8 @@ public class Tool {
public Tool(String[] args) {
this.args = args;
errMgr = new ErrorManager(this);
errMgr.setFormat(msgFormat);
handleArgs();
}
@ -212,8 +217,12 @@ public class Tool {
while ( args!=null && i<args.length ) {
String arg = args[i];
i++;
if ( arg.startsWith("-D") ) { // -Dlanguage=Java syntax
handleOptionSetArg(arg);
continue;
}
if ( arg.charAt(0)!='-' ) { // file name
grammarFiles.add(arg);
if ( !grammarFiles.contains(arg) ) grammarFiles.add(arg);
continue;
}
boolean found = false;
@ -280,15 +289,39 @@ public class Tool {
}
}
public void processGrammarsOnCommandLine() {
for (String fileName : grammarFiles) {
GrammarAST t = loadGrammar(fileName);
if ( t==null || t instanceof GrammarASTErrorNode) return; // came back as error node
if ( ((GrammarRootAST)t).hasErrors ) return;
GrammarRootAST ast = (GrammarRootAST)t;
protected void handleOptionSetArg(String arg) {
int eq = arg.indexOf('=');
if ( eq>0 && arg.length()>3 ) {
String option = arg.substring("-D".length(), eq);
String value = arg.substring(eq+1);
if ( value.length()==0 ) {
errMgr.toolError(ErrorType.BAD_OPTION_SET_SYNTAX, arg);
return;
}
if ( Grammar.parserOptions.contains(option) ||
Grammar.lexerOptions.contains(option) )
{
if ( grammarOptions==null ) grammarOptions = new HashMap<String, String>();
grammarOptions.put(option, value);
}
else {
errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
null,
null,
option);
}
}
else {
errMgr.toolError(ErrorType.BAD_OPTION_SET_SYNTAX, arg);
}
}
final Grammar g = createGrammar(ast);
g.fileName = fileName;
public void processGrammarsOnCommandLine() {
List<GrammarRootAST> sortedGrammars = sortGrammarByTokenVocab(grammarFiles);
for (GrammarRootAST t : sortedGrammars) {
final Grammar g = createGrammar(t);
g.fileName = t.fileName;
process(g, true);
}
}
@ -302,7 +335,6 @@ public class Tool {
public void process(Grammar g, boolean gencode) {
g.loadImportedGrammars();
GrammarTransformPipeline transform = new GrammarTransformPipeline(g, this);
transform.process();
@ -332,13 +364,12 @@ public class Tool {
if ( g.ast!=null && internalOption_PrintGrammarTree ) System.out.println(g.ast.toStringTree());
//g.ast.inspect();
if ( errMgr.getNumErrors()>0 ) return;
int prevErrors = errMgr.getNumErrors();
// MAKE SURE GRAMMAR IS SEMANTICALLY CORRECT (FILL IN GRAMMAR OBJECT)
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if ( errMgr.getNumErrors()>0 ) return;
if ( errMgr.getNumErrors()>prevErrors ) return;
// BUILD ATN FROM AST
ATNFactory factory;
@ -354,7 +385,7 @@ public class Tool {
//if ( generate_DFA_dot ) generateDFAs(g);
if ( g.tool.getNumErrors()>0 ) return;
if ( g.tool.getNumErrors()>prevErrors ) return;
// GENERATE CODE
if ( gencode ) {
@ -363,6 +394,63 @@ public class Tool {
}
}
public List<GrammarRootAST> sortGrammarByTokenVocab(List<String> fileNames) {
// System.out.println(fileNames);
Graph<String> g = new Graph<String>();
List<GrammarRootAST> roots = new ArrayList<GrammarRootAST>();
for (String fileName : fileNames) {
GrammarAST t = loadGrammar(fileName);
if ( t==null || t instanceof GrammarASTErrorNode) continue; // came back as error node
if ( ((GrammarRootAST)t).hasErrors ) continue;
GrammarRootAST root = (GrammarRootAST)t;
roots.add(root);
root.fileName = fileName;
String grammarName = root.getChild(0).getText();
GrammarAST tokenVocabNode = findOptionValueAST(root, "tokenVocab");
// Make grammars depend on any tokenVocab options
if ( tokenVocabNode!=null ) {
String vocabName = tokenVocabNode.getText();
g.addEdge(grammarName, vocabName);
}
// add cycle to graph so we always process a grammar if no error
// even if no dependency
g.addEdge(grammarName, grammarName);
}
List<String> sortedGrammarNames = g.sort();
// System.out.println("sortedGrammarNames="+sortedGrammarNames);
List<GrammarRootAST> sortedRoots = new ArrayList<GrammarRootAST>();
for (String grammarName : sortedGrammarNames) {
for (GrammarRootAST root : roots) {
if ( root.getGrammarName().equals(grammarName) ) {
sortedRoots.add(root);
break;
}
}
}
return sortedRoots;
}
/** Manually get option node from tree; return null if no defined. */
public static GrammarAST findOptionValueAST(GrammarRootAST root, String option) {
GrammarAST options = (GrammarAST)root.getFirstChildWithType(ANTLRParser.OPTIONS);
if ( options!=null ) {
for (Object o : options.getChildren()) {
GrammarAST c = (GrammarAST)o;
if ( c.getType() == ANTLRParser.ASSIGN &&
c.getChild(0).getText().equals(option) )
{
return (GrammarAST)c.getChild(1);
}
}
}
return null;
}
/** Given the raw AST of a grammar, create a grammar object
associated with the AST. Once we have the grammar object, ensure
that all nodes in tree referred to this grammar. Later, we will
@ -435,6 +523,9 @@ public class Tool {
if ( root instanceof GrammarRootAST) {
((GrammarRootAST)root).hasErrors = p.getNumberOfSyntaxErrors()>0;
((GrammarRootAST)root).tokens = tokens;
if ( grammarOptions!=null ) {
((GrammarRootAST)root).cmdLineOptions = grammarOptions;
}
return ((GrammarRootAST)root);
}
return null;
@ -505,8 +596,15 @@ public class Tool {
if (!outputDir.exists()) {
outputDir.mkdirs();
}
FileWriter fw = new FileWriter(outputFile);
return new BufferedWriter(fw);
FileOutputStream fos = new FileOutputStream(outputFile);
OutputStreamWriter osw;
if ( grammarEncoding!=null ) {
osw = new OutputStreamWriter(fos, grammarEncoding);
}
else {
osw = new OutputStreamWriter(fos);
}
return new BufferedWriter(osw);
}
public File getImportedGrammarFile(Grammar g, String fileName) {
@ -595,7 +693,7 @@ public class Tool {
}
public void help() {
info("ANTLR Parser Generator Version " + new Tool().VERSION);
info("ANTLR Parser Generator Version " + Tool.getVersion());
for (Option o : optionDefs) {
String name = o.name + (o.argType!=OptionArgType.NONE? " ___" : "");
String s = String.format(" %-19s %s", name, o.description);
@ -638,7 +736,7 @@ public class Tool {
}
public void version() {
info("ANTLR Parser Generator Version " + new Tool().VERSION);
info("ANTLR Parser Generator Version " + getVersion());
}
public void exit(int e) { System.exit(e); }

15
tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java
View File

@ -29,11 +29,20 @@
package org.antlr.v4.analysis;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.RuleStartState;
import org.antlr.v4.runtime.atn.RuleStopState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.util.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class LeftRecursionDetector {
Grammar g;

9
tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java
View File

@ -69,17 +69,18 @@ import java.util.List;
public class LeftRecursiveRuleTransformer {
public GrammarRootAST ast;
public Collection<Rule> rules;
public Grammar g;
public Tool tool;
public LeftRecursiveRuleTransformer(GrammarRootAST ast, Collection<Rule> rules, Tool tool) {
public LeftRecursiveRuleTransformer(GrammarRootAST ast, Collection<Rule> rules, Grammar g) {
this.ast = ast;
this.rules = rules;
this.tool = tool;
this.g = g;
this.tool = g.tool;
}
public void translateLeftRecursiveRules() {
// TODO: what about -language foo cmd line?
String language = Grammar.getLanguageOption(ast);
String language = g.getOptionString("language");
// translate all recursive rules
List<String> leftRecursiveRuleNames = new ArrayList<String>();
for (Rule r : rules) {

6
tool/src/org/antlr/v4/automata/ATNFactory.java
View File

@ -31,7 +31,11 @@ package org.antlr.v4.automata;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.PredAST;
import org.antlr.v4.tool.ast.TerminalAST;
import java.util.List;

33
tool/src/org/antlr/v4/automata/ATNOptimizer.java
View File

@ -46,6 +46,7 @@ import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.util.ArrayList;
import java.util.List;
/**
@ -144,30 +145,24 @@ public class ATNOptimizer {
}
}
System.out.println("ATN optimizer removed " + removedStates + " states by collapsing sets.");
// System.out.println("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
private static void optimizeStates(ATN atn) {
List<ATNState> states = atn.states;
int current = 0;
for (int i = 0; i < states.size(); i++) {
ATNState state = states.get(i);
if (state == null) {
continue;
// System.out.println(atn.states);
List<ATNState> compressed = new ArrayList<ATNState>();
int i = 0; // new state number
for (ATNState s : atn.states) {
if ( s!=null ) {
compressed.add(s);
s.stateNumber = i; // reset state number as we shift to new position
i++;
}
if (i != current) {
state.stateNumber = current;
states.set(current, state);
states.set(i, null);
}
current++;
}
System.out.println("ATN optimizer removed " + (states.size() - current) + " null states.");
states.subList(current, states.size()).clear();
// System.out.println(compressed);
// System.out.println("ATN optimizer removed " + (atn.states.size() - compressed.size()) + " null states.");
atn.states.clear();
atn.states.addAll(compressed);
}
private ATNOptimizer() {

18
tool/src/org/antlr/v4/automata/ATNPrinter.java
View File

@ -29,7 +29,23 @@
package org.antlr.v4.automata;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ActionTransition;
import org.antlr.v4.runtime.atn.AtomTransition;
import org.antlr.v4.runtime.atn.BlockEndState;
import org.antlr.v4.runtime.atn.BlockStartState;
import org.antlr.v4.runtime.atn.EpsilonTransition;
import org.antlr.v4.runtime.atn.NotSetTransition;
import org.antlr.v4.runtime.atn.PlusBlockStartState;
import org.antlr.v4.runtime.atn.PlusLoopbackState;
import org.antlr.v4.runtime.atn.RuleStartState;
import org.antlr.v4.runtime.atn.RuleStopState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.SetTransition;
import org.antlr.v4.runtime.atn.StarBlockStartState;
import org.antlr.v4.runtime.atn.StarLoopEntryState;
import org.antlr.v4.runtime.atn.StarLoopbackState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.tool.Grammar;
import java.util.ArrayList;

40
tool/src/org/antlr/v4/automata/ATNSerializer.java
View File

@ -31,7 +31,19 @@ package org.antlr.v4.automata;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNSimulator;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ActionTransition;
import org.antlr.v4.runtime.atn.AtomTransition;
import org.antlr.v4.runtime.atn.BlockStartState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LoopEndState;
import org.antlr.v4.runtime.atn.PredicateTransition;
import org.antlr.v4.runtime.atn.RangeTransition;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.SetTransition;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet;
@ -79,9 +91,10 @@ public class ATNSerializer {
else if ( g.getType()== ANTLRParser.PARSER ) data.add(ATN.PARSER);
else data.add(ATN.TREE_PARSER);
data.add(g.getMaxTokenType());
data.add(atn.states.size());
int nedges = 0;
// dump states, count edges and collect sets while doing so
data.add(atn.states.size());
for (ATNState s : atn.states) {
if ( s==null ) { // might be optimized away
data.add(ATNState.INVALID_TYPE);
@ -95,7 +108,12 @@ public class ATNSerializer {
else if ( s instanceof BlockStartState ) {
data.add(((BlockStartState)s).endState.stateNumber);
}
nedges += s.getNumberOfTransitions();
if (s.getStateType() != ATNState.RULE_STOP) {
// the deserializer can trivially derive these edges, so there's no need to serialize them
nedges += s.getNumberOfTransitions();
}
for (int i=0; i<s.getNumberOfTransitions(); i++) {
Transition t = s.transition(i);
int edgeType = Transition.serializationTypes.get(t.getClass());
@ -105,6 +123,7 @@ public class ATNSerializer {
}
}
}
int nrules = atn.ruleToStartState.length;
data.add(nrules);
for (int r=0; r<nrules; r++) {
@ -117,6 +136,7 @@ public class ATNSerializer {
data.add(rule.actionIndex);
}
}
int nmodes = atn.modeToStartState.size();
data.add(nmodes);
if ( nmodes>0 ) {
@ -124,6 +144,7 @@ public class ATNSerializer {
data.add(modeStartState.stateNumber);
}
}
int nsets = sets.size();
data.add(nsets);
for (IntervalSet set : sets) {
@ -133,10 +154,19 @@ public class ATNSerializer {
data.add(I.b);
}
}
data.add(nedges);
int setIndex = 0;
for (ATNState s : atn.states) {
if ( s==null ) continue; // might be optimized away
if ( s==null ) {
// might be optimized away
continue;
}
if (s.getStateType() == ATNState.RULE_STOP) {
continue;
}
for (int i=0; i<s.getNumberOfTransitions(); i++) {
Transition t = s.transition(i);
@ -217,7 +247,7 @@ public class ATNSerializer {
int loopBackStateNumber = ATNSimulator.toInt(data[p++]);
arg = " "+loopBackStateNumber;
}
else if ( stype == ATNState.PLUS_BLOCK_START || stype == ATNState.STAR_BLOCK_START ) {
else if ( stype == ATNState.PLUS_BLOCK_START || stype == ATNState.STAR_BLOCK_START || stype == ATNState.BLOCK_START ) {
int endStateNumber = ATNSimulator.toInt(data[p++]);
arg = " "+endStateNumber;
}

3
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -46,7 +46,6 @@ import org.antlr.v4.runtime.atn.TokensStartState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
@ -63,7 +62,7 @@ public class LexerATNFactory extends ParserATNFactory {
public LexerATNFactory(LexerGrammar g) {
super(g);
// use codegen to get correct language templates for lexer commands
String language = Grammar.getLanguageOption(g.ast);
String language = g.getOptionString("language");
CodeGenerator gen = new CodeGenerator(g.tool, null, language);
codegenTemplates = gen.templates;
}

31
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -38,7 +38,27 @@ import org.antlr.v4.misc.CharSupport;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.ATNBuilder;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ActionTransition;
import org.antlr.v4.runtime.atn.AtomTransition;
import org.antlr.v4.runtime.atn.BlockEndState;
import org.antlr.v4.runtime.atn.BlockStartState;
import org.antlr.v4.runtime.atn.EpsilonTransition;
import org.antlr.v4.runtime.atn.LoopEndState;
import org.antlr.v4.runtime.atn.NotSetTransition;
import org.antlr.v4.runtime.atn.PlusBlockStartState;
import org.antlr.v4.runtime.atn.PlusLoopbackState;
import org.antlr.v4.runtime.atn.PredicateTransition;
import org.antlr.v4.runtime.atn.RuleStartState;
import org.antlr.v4.runtime.atn.RuleStopState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.SetTransition;
import org.antlr.v4.runtime.atn.StarBlockStartState;
import org.antlr.v4.runtime.atn.StarLoopEntryState;
import org.antlr.v4.runtime.atn.StarLoopbackState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.atn.WildcardTransition;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
@ -47,7 +67,12 @@ import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.PredAST;
import org.antlr.v4.tool.ast.TerminalAST;
import java.lang.reflect.Constructor;
import java.util.Collection;
@ -205,8 +230,6 @@ public class ParserATNFactory implements ATNFactory {
@Override
public Handle ruleRef(GrammarAST node) {
Handle h = _ruleRef(node);
// Rule r = g.getRule(node.getText());
// addFollowLink(r, h.right);
return h;
}

6
tool/src/org/antlr/v4/codegen/CodeGenPipeline.java
View File

@ -42,6 +42,8 @@ public class CodeGenPipeline {
public void process() {
CodeGenerator gen = new CodeGenerator(g);
if ( gen.templates==null ) return;
if ( g.isLexer() ) {
ST lexer = gen.generateLexer();
if ( g.tool.launch_ST_inspector ) lexer.inspect();
@ -55,10 +57,6 @@ public class CodeGenPipeline {
gen.writeListener(gen.generateListener());
gen.writeBaseListener(gen.generateBaseListener());
}
if ( g.tool.gen_parse_listener ) {
gen.writeParseListener(gen.generateParseListener());
gen.writeBaseParseListener(gen.generateBaseParseListener());
}
if ( g.tool.gen_visitor ) {
gen.writeVisitor(gen.generateVisitor());
gen.writeBaseVisitor(gen.generateBaseVisitor());

49
tool/src/org/antlr/v4/codegen/CodeGenerator.java
View File

@ -33,13 +33,22 @@ import org.antlr.v4.Tool;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.tool.*;
import org.stringtemplate.v4.*;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.stringtemplate.v4.AutoIndentWriter;
import org.stringtemplate.v4.NumberRenderer;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupFile;
import org.stringtemplate.v4.STWriter;
import org.stringtemplate.v4.StringRenderer;
import java.io.*;
import java.lang.reflect.*;
import java.util.*;
import org.antlr.v4.misc.Utils.Func1;
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
/** General controller for code gen. Can instantiate sub generator(s).
*/
@ -61,7 +70,7 @@ public class CodeGenerator {
public int lineWidth = 72;
public CodeGenerator(@NotNull Grammar g) {
this(g.tool, g, g.getOptionString("language", "Java"));
this(g.tool, g, g.getOptionString("language"));
}
public CodeGenerator(@NotNull Tool tool, @NotNull Grammar g, String language) {
@ -109,7 +118,7 @@ public class CodeGenerator {
}
catch (IllegalArgumentException iae) {
tool.errMgr.toolError(ErrorType.CANNOT_CREATE_TARGET_GENERATOR,
iae,
null,
language);
}
}
@ -132,8 +141,6 @@ public class CodeGenerator {
public ST generateParser() { return walk(createController().buildParserOutputModel()); }
public ST generateListener() { return walk(createController().buildListenerOutputModel()); }
public ST generateBaseListener() { return walk(createController().buildBaseListenerOutputModel()); }
public ST generateParseListener() { return walk(createController().buildParseListenerOutputModel()); }
public ST generateBaseParseListener() { return walk(createController().buildBaseParseListenerOutputModel()); }
public ST generateVisitor() { return walk(createController().buildVisitorOutputModel()); }
public ST generateBaseVisitor() { return walk(createController().buildBaseVisitorOutputModel()); }
@ -181,14 +188,6 @@ public class CodeGenerator {
target.genFile(g,outputFileST, getBaseListenerFileName());
}
public void writeParseListener(ST outputFileST) {
target.genFile(g,outputFileST, getParseListenerFileName());
}
public void writeBaseParseListener(ST outputFileST) {
target.genFile(g,outputFileST, getBaseParseListenerFileName());
}
public void writeVisitor(ST outputFileST) {
target.genFile(g,outputFileST, getVisitorFileName());
}
@ -296,20 +295,6 @@ public class CodeGenerator {
return listenerName+extST.render();
}
public String getParseListenerFileName() {
assert g.name != null;
ST extST = templates.getInstanceOf("codeFileExtension");
String listenerName = g.name + "ParseListener";
return listenerName+extST.render();
}
public String getBaseParseListenerFileName() {
assert g.name != null;
ST extST = templates.getInstanceOf("codeFileExtension");
String listenerName = g.name + "BaseParseListener";
return listenerName+extST.render();
}
/** A given grammar T, return a blank listener implementation
* such as TBaseListener.java, if we're using the Java target.
*/

53
tool/src/org/antlr/v4/codegen/OutputModelController.java
View File

@ -31,16 +31,45 @@ package org.antlr.v4.codegen;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.v4.analysis.LeftRecursiveRuleAltInfo;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.codegen.model.Action;
import org.antlr.v4.codegen.model.AltBlock;
import org.antlr.v4.codegen.model.BaseListenerFile;
import org.antlr.v4.codegen.model.BaseVisitorFile;
import org.antlr.v4.codegen.model.Choice;
import org.antlr.v4.codegen.model.CodeBlockForAlt;
import org.antlr.v4.codegen.model.CodeBlockForOuterMostAlt;
import org.antlr.v4.codegen.model.LabeledOp;
import org.antlr.v4.codegen.model.LeftRecursiveRuleFunction;
import org.antlr.v4.codegen.model.Lexer;
import org.antlr.v4.codegen.model.LexerFile;
import org.antlr.v4.codegen.model.ListenerFile;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.codegen.model.Parser;
import org.antlr.v4.codegen.model.ParserFile;
import org.antlr.v4.codegen.model.RuleActionFunction;
import org.antlr.v4.codegen.model.RuleFunction;
import org.antlr.v4.codegen.model.RuleSempredFunction;
import org.antlr.v4.codegen.model.SrcOp;
import org.antlr.v4.codegen.model.StarBlock;
import org.antlr.v4.codegen.model.VisitorFile;
import org.antlr.v4.codegen.model.decl.CodeBlock;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import org.stringtemplate.v4.*;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.tool.Alternative;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.PredAST;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import java.util.*;
import org.antlr.runtime.RecognitionException;
import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
/** This receives events from SourceGenTriggers.g and asks factory to do work.
* Then runs extensions in order on resulting SrcOps to get final list.
@ -115,16 +144,6 @@ public class OutputModelController {
return new BaseListenerFile(delegate, gen.getBaseListenerFileName());
}
public OutputModelObject buildParseListenerOutputModel() {
CodeGenerator gen = delegate.getGenerator();
return new ParseListenerFile(delegate, gen.getParseListenerFileName());
}
public OutputModelObject buildBaseParseListenerOutputModel() {
CodeGenerator gen = delegate.getGenerator();
return new BaseParseListenerFile(delegate, gen.getBaseParseListenerFileName());
}
public OutputModelObject buildVisitorOutputModel() {
CodeGenerator gen = delegate.getGenerator();
return new VisitorFile(delegate, gen.getVisitorFileName());

12
tool/src/org/antlr/v4/codegen/OutputModelWalker.java
View File

@ -33,12 +33,18 @@ import org.antlr.v4.Tool;
import org.antlr.v4.codegen.model.ModelElement;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.tool.ErrorType;
import org.stringtemplate.v4.*;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.compiler.FormalArgument;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.util.*;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
/** Convert an output model tree to template hierarchy by walking
* the output model. Each output model object has a corresponding template

9
tool/src/org/antlr/v4/codegen/model/BaseParseListenerFile.java
View File

@ -1,9 +0,0 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
public class BaseParseListenerFile extends ParseListenerFile {
public BaseParseListenerFile(OutputModelFactory factory, String fileName) {
super(factory, fileName);
}
}

3
tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java
View File

@ -29,7 +29,8 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.*;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.PlusBlockStartState;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.Grammar;

13
tool/src/org/antlr/v4/codegen/model/Lexer.java
View File

@ -31,6 +31,9 @@ package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.chunk.ActionChunk;
import org.antlr.v4.codegen.model.chunk.ActionText;
import org.antlr.v4.codegen.model.chunk.DefaultLexerSuperClass;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
@ -49,7 +52,7 @@ public class Lexer extends OutputModelObject {
public String[] tokenNames;
public Set<String> ruleNames;
public Collection<String> modes;
public boolean abstractRecognizer;
@ModelElement public ActionChunk superClass;
@ModelElement public SerializedATN atn;
@ModelElement public LinkedHashMap<Rule, RuleActionFunction> actionFuncs =
@ -90,7 +93,13 @@ public class Lexer extends OutputModelObject {
}
}
ruleNames = g.rules.keySet();
abstractRecognizer = g.isAbstract();
if (g.getOptionString("superClass") != null) {
superClass = new ActionText(null, g.getOptionString("superClass"));
}
else {
superClass = new DefaultLexerSuperClass();
}
}
}

13
tool/src/org/antlr/v4/codegen/model/ParseListenerDispatchMethod.java
View File

@ -1,13 +0,0 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
public class ParseListenerDispatchMethod extends DispatchMethod {
public boolean isEnter;
public ParseListenerDispatchMethod(OutputModelFactory factory, boolean isEnter) {
super(factory);
this.isEnter = isEnter;
}
}

51
tool/src/org/antlr/v4/codegen/model/ParseListenerFile.java
View File

@ -1,51 +0,0 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.misc.Triple;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class ParseListenerFile extends OutputFile {
public String grammarName;
public String parserName;
public Set<String> listenerEnterNames = new HashSet<String>();
public Set<String> listenerExitNames = new HashSet<String>();
@ModelElement public Action header;
public ParseListenerFile(OutputModelFactory factory, String fileName) {
super(factory, fileName);
Grammar g = factory.getGrammar();
parserName = g.getRecognizerName();
grammarName = g.name;
for (Rule r : g.rules.values()) {
List<Triple<Integer,AltAST,String>> labels = r.getAltLabels();
// EXIT RULES
if ( labels!=null ) {
// add exit rules for alt labels
for (Triple<Integer,AltAST,String> pair : labels) {
listenerExitNames.add(pair.c);
if ( !(r instanceof LeftRecursiveRule) ) {
listenerEnterNames.add(pair.c);
}
}
}
else {
// add exit rule if no labels
listenerExitNames.add(r.name);
if ( !(r instanceof LeftRecursiveRule) ) {
listenerEnterNames.add(r.name);
}
}
}
ActionAST ast = g.namedActions.get("header");
if ( ast!=null ) header = new Action(factory, ast);
}
}

29
tool/src/org/antlr/v4/codegen/model/Parser.java
View File

@ -29,25 +29,33 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.*;
import org.antlr.v4.codegen.model.chunk.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.chunk.ActionChunk;
import org.antlr.v4.codegen.model.chunk.ActionText;
import org.antlr.v4.codegen.model.chunk.DefaultParserSuperClass;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.io.File;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** */
public class Parser extends OutputModelObject {
public String name;
public String grammarFileName;
public String grammarName;
@ModelElement public ActionChunk superclass;
@ModelElement public ActionChunk superClass;
public Map<String,Integer> tokens;
public String[] tokenNames;
public Set<String> ruleNames;
public Collection<Rule> rules;
public ParserFile file;
public boolean abstractRecognizer;
@ModelElement public List<RuleFunction> funcs = new ArrayList<RuleFunction>();
@ModelElement public SerializedATN atn;
@ -86,11 +94,10 @@ public class Parser extends OutputModelObject {
rules = g.rules.values();
atn = new SerializedATN(factory, g.atn);
if (g.getOptionString("superClass") != null) {
superclass = new ActionText(null, g.getOptionString("superClass"));
} else {
superclass = new DefaultParserSuperClass();
superClass = new ActionText(null, g.getOptionString("superClass"));
}
else {
superClass = new DefaultParserSuperClass();
}
abstractRecognizer = g.isAbstract();
}
}

14
tool/src/org/antlr/v4/codegen/model/RuleFunction.java
View File

@ -170,7 +170,19 @@ public class RuleFunction extends OutputModelObject {
FrequencySet<String> altFreq = getElementFrequenciesForAlt(ast);
for (GrammarAST t : refs) {
String refLabelName = t.getText();
if ( altFreq.count(t.getText())>1 ) needsList.add(refLabelName);
if (needsList.contains(refLabelName)) {
continue;
}
if ( altFreq.count(t.getText())>1 ) {
needsList.add(refLabelName);
}
else {
boolean inLoop = t.hasAncestor(CLOSURE) || t.hasAncestor(POSITIVE_CLOSURE);
if (inLoop) {
needsList.add(refLabelName);
}
}
}
}
Set<Decl> decls = new HashSet<Decl>();

2
tool/src/org/antlr/v4/codegen/model/SemPred.java
View File

@ -47,7 +47,7 @@ public class SemPred extends Action {
public SemPred(OutputModelFactory factory, GrammarAST ast) {
super(factory,ast);
GrammarAST failNode = ((PredAST)ast).getOption("fail");
GrammarAST failNode = ((PredAST)ast).getOptionAST("fail");
CodeGenerator gen = factory.getGenerator();
if ( failNode==null ) {
msg = ast.getText();

3
tool/src/org/antlr/v4/codegen/model/SerializedATN.java
View File

@ -34,7 +34,8 @@ import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.misc.IntegerList;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
public class SerializedATN extends OutputModelObject {
// TODO: make this into a kind of decl or multiple?

5
tool/src/org/antlr/v4/codegen/model/chunk/DefaultLexerSuperClass.java Normal file
View File

@ -0,0 +1,5 @@
package org.antlr.v4.codegen.model.chunk;
public class DefaultLexerSuperClass extends ActionChunk {
public DefaultLexerSuperClass() { super(null); }
}

13
tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java
View File

@ -30,8 +30,10 @@
package org.antlr.v4.codegen.model.decl;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.codegen.model.DispatchMethod;
import org.antlr.v4.codegen.model.ListenerDispatchMethod;
import org.antlr.v4.codegen.model.VisitorDispatchMethod;
import org.antlr.v4.tool.Rule;
import java.util.ArrayList;
@ -58,13 +60,6 @@ public class AltLabelStructDecl extends StructDecl {
if ( factory.getGrammar().tool.gen_visitor ) {
dispatchMethods.add(new VisitorDispatchMethod(factory));
}
if ( factory.getGrammar().tool.gen_parse_listener ) {
if ( !(r instanceof LeftRecursiveRule) ) {
dispatchMethods.add(new ParseListenerDispatchMethod(factory, true));
}
dispatchMethods.add(new ParseListenerDispatchMethod(factory, false));
}
}
@Override

8
tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java
View File

@ -34,11 +34,9 @@ import org.antlr.v4.codegen.model.DispatchMethod;
import org.antlr.v4.codegen.model.ListenerDispatchMethod;
import org.antlr.v4.codegen.model.ModelElement;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.codegen.model.ParseListenerDispatchMethod;
import org.antlr.v4.codegen.model.VisitorDispatchMethod;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import org.antlr.v4.tool.Attribute;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import java.util.ArrayList;
@ -76,12 +74,6 @@ public class StructDecl extends Decl {
if ( factory.getGrammar().tool.gen_visitor ) {
dispatchMethods.add(new VisitorDispatchMethod(factory));
}
if ( factory.getGrammar().tool.gen_parse_listener ) {
if ( !(r instanceof LeftRecursiveRule) ) {
dispatchMethods.add(new ParseListenerDispatchMethod(factory, true));
}
dispatchMethods.add(new ParseListenerDispatchMethod(factory, false));
}
}
}

4
tool/src/org/antlr/v4/misc/Utils.java
View File

@ -32,7 +32,9 @@ package org.antlr.v4.misc;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.tool.ast.GrammarAST;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/** */
public class Utils {

1
tool/src/org/antlr/v4/parse/ANTLRLexer.g
View File

@ -446,7 +446,6 @@ OR : '|' ;
DOLLAR : '$' ;
DOT : '.' ; // can be WILDCARD or DOT in qid or imported rule ref
RANGE : '..' ;
ETC : '...' ;
AT : '@' ;
POUND : '#' ;
NOT : '~' ;

11
tool/src/org/antlr/v4/parse/ANTLRParser.g
View File

@ -283,16 +283,7 @@ delegateGrammar
* {tree} parser.
*/
tokensSpec
: TOKENS_SPEC tokenSpec+ RBRACE -> ^(TOKENS_SPEC tokenSpec+)
;
tokenSpec
: id
( ASSIGN STRING_LITERAL -> ^(ASSIGN id STRING_LITERAL<TerminalAST>)
| -> id
)
SEMI
| RULE_REF // INVALID! (an error alt)
: TOKENS_SPEC id (COMMA id)* RBRACE -> ^(TOKENS_SPEC id+)
;
// A declaration of a language target specifc section,

11
tool/src/org/antlr/v4/parse/BlockSetTransformer.g
View File

@ -23,7 +23,6 @@ import org.antlr.v4.runtime.misc.IntervalSet;
@members {
public String currentRuleName;
public GrammarAST currentAlt;
public Set<String> rewriteElems = new HashSet<String>();
public Grammar g;
public BlockSetTransformer(TreeNodeStream input, Grammar g) {
this(input, new RecognizerSharedState());
@ -40,7 +39,7 @@ topdown
setAlt
: {inContext("RULE BLOCK")}?
ALT {currentAlt = $start; rewriteElems.clear();}
ALT {currentAlt = $start;}
;
// (BLOCK (ALT (+ (BLOCK (ALT INT) (ALT ID)))))
@ -65,7 +64,10 @@ boolean inLexer = Character.isUpperCase(currentRuleName.charAt(0));
@after {
GrammarTransformPipeline.setGrammarPtr(g, $tree);
}
: {!inContext("RULE")}? // if not rule block and > 1 alt
: {inContext("RULE")}? // top-level: rule block and > 1 alt
^(BLOCK ^(ALT setElement[inLexer]) ( ^(ALT setElement[inLexer]) )+)
-> ^(BLOCK<BlockAST>[$BLOCK.token] ^(ALT<AltAST>[$BLOCK.token,"ALT"] ^(SET[$BLOCK.token, "SET"] setElement+)))
| {!inContext("RULE")}? // if not rule block and > 1 alt
^(BLOCK ^(ALT setElement[inLexer]) ( ^(ALT setElement[inLexer]) )+)
-> ^(SET[$BLOCK.token, "SET"] setElement+)
;
@ -74,8 +76,7 @@ setElement[boolean inLexer]
@after {
GrammarTransformPipeline.setGrammarPtr(g, $tree);
}
: {!rewriteElems.contains($start.getText())}?
( a=STRING_LITERAL {!inLexer || CharSupport.getCharValueFromGrammarCharLiteral($a.getText())!=-1}?
: ( a=STRING_LITERAL {!inLexer || CharSupport.getCharValueFromGrammarCharLiteral($a.getText())!=-1}?
| {!inLexer}?=> TOKEN_REF
| {inLexer}?=> ^(RANGE a=STRING_LITERAL b=STRING_LITERAL)
{CharSupport.getCharValueFromGrammarCharLiteral($a.getText())!=-1 &&

5
tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g
View File

@ -121,7 +121,7 @@ public void finishGrammar(GrammarRootAST root, GrammarAST ID) { }
public void grammarOption(GrammarAST ID, GrammarAST valueAST) { }
public void ruleOption(GrammarAST ID, GrammarAST valueAST) { }
public void blockOption(GrammarAST ID, GrammarAST valueAST) { }
public void tokenAlias(GrammarAST ID, GrammarAST literal) { }
public void defineToken(GrammarAST ID) { }
public void globalNamedAction(GrammarAST scope, GrammarAST ID, ActionAST action) { }
public void importGrammar(GrammarAST label, GrammarAST ID) { }
@ -225,8 +225,7 @@ tokensSpec
;
tokenSpec
: ^(ASSIGN ID STRING_LITERAL) {tokenAlias($ID, $STRING_LITERAL);}
| ID {tokenAlias($ID, null);}
: ID {defineToken($ID);}
;
action

5
tool/src/org/antlr/v4/parse/ToolANTLRParser.java
View File

@ -29,7 +29,10 @@
package org.antlr.v4.parse;
import org.antlr.runtime.*;
import org.antlr.runtime.NoViableAltException;
import org.antlr.runtime.Parser;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.TokenStream;
import org.antlr.v4.Tool;
import org.antlr.v4.tool.ErrorType;

3
tool/src/org/antlr/v4/parse/v4ParserException.java
View File

@ -29,7 +29,8 @@
package org.antlr.v4.parse;
import org.antlr.runtime.*;
import org.antlr.runtime.IntStream;
import org.antlr.runtime.RecognitionException;
/** */
public class v4ParserException extends RecognitionException {

8
tool/src/org/antlr/v4/semantics/AttributeChecks.java
View File

@ -33,7 +33,13 @@ import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.Token;
import org.antlr.v4.parse.ActionSplitter;
import org.antlr.v4.parse.ActionSplitterListener;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Alternative;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.LabelType;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import java.util.List;

94
tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java
View File

@ -31,13 +31,25 @@ package org.antlr.v4.semantics;
import org.antlr.runtime.Token;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTWithOptions;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.antlr.v4.tool.ast.RuleAST;
import org.antlr.v4.tool.ast.TerminalAST;
import org.stringtemplate.v4.misc.MultiMap;
import java.io.File;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
/** No side-effects except for setting options into the appropriate node.
* TODO: make the side effects into a separate pass this
@ -67,60 +79,7 @@ import java.util.*;
* TODO: 1 action per lex rule
*/
public class BasicSemanticChecks extends GrammarTreeVisitor {
public static final Set<String> legalLexerOptions =
new HashSet<String>() {
{
add("language"); add("tokenVocab");
add("TokenLabelType");
add("superClass");
add("filter");
add("abstract");
}
};
public static final Set<String> legalParserOptions =
new HashSet<String>() {
{
add("tokenVocab");
add("TokenLabelType");
add("superClass");
add("abstract");
}
};
public static final Set<String> legalRuleOptions =
new HashSet<String>() {
{
add("context");
add("simrecursion_");
}
};
public static final Set<String> legalBlockOptions =
new HashSet<String>() {
{
add("greedy");
add("simrecursion_");
}
};
/** Legal options for terminal refs like ID<node=MyVarNode> */
public static final Set<String> legalTokenOptions =
new HashSet<String>() {
{
add("assoc");
}
};
public static final Set<String> legalSemPredOptions =
new HashSet<String>() {
{
add("fail");
}
};
/** Set of valid imports. E.g., can only import a tree parser into
* another tree parser. Maps delegate to set of delegator grammar types.
/** Set of valid imports. Maps delegate to set of delegator grammar types.
* validDelegations.get(LEXER) gives list of the kinds of delegators
* that can import lexers.
*/
@ -166,11 +125,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
checkNumPrequels(options, imports, tokens);
}
@Override
public void tokenAlias(GrammarAST ID, GrammarAST literal) {
if ( literal!=null ) checkTokenAlias(ID.token);
}
@Override
public void importGrammar(GrammarAST label, GrammarAST ID) {
checkImport(ID.token);
@ -380,7 +334,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
{
boolean ok = true;
if ( parent.getType()==ANTLRParser.BLOCK ) {
if ( !legalBlockOptions.contains(optionID.getText()) ) { // block
if ( !Grammar.subruleOptions.contains(optionID.getText()) ) { // block
g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
g.fileName,
optionID,
@ -389,7 +343,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
}
}
else if ( parent.getType()==ANTLRParser.RULE ) {
if ( !legalRuleOptions.contains(optionID.getText()) ) { // rule
if ( !Grammar.ruleOptions.contains(optionID.getText()) ) { // rule
g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
g.fileName,
optionID,
@ -423,7 +377,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
if ( elem.getType()==ANTLRParser.SEMPRED ) {
Token optionID = ID.token;
String fileName = optionID.getInputStream().getSourceName();
if ( valueAST!=null && !legalSemPredOptions.contains(optionID.getText()) ) {
if ( valueAST!=null && !Grammar.semPredOptions.contains(optionID.getText()) ) {
g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
fileName,
optionID,
@ -439,7 +393,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
Token optionID = ID.token;
String fileName = optionID.getInputStream().getSourceName();
// don't care about ID<ASTNodeName> options
if ( valueAST!=null && !legalTokenOptions.contains(optionID.getText()) ) {
if ( valueAST!=null && !Grammar.tokenOptions.contains(optionID.getText()) ) {
g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
fileName,
optionID,
@ -453,11 +407,11 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
boolean legalGrammarOption(String key) {
switch ( g.getType() ) {
case ANTLRParser.LEXER :
return legalLexerOptions.contains(key);
return Grammar.lexerOptions.contains(key);
case ANTLRParser.PARSER :
return legalParserOptions.contains(key);
return Grammar.parserOptions.contains(key);
default :
return legalParserOptions.contains(key);
return Grammar.parserOptions.contains(key);
}
}

20
tool/src/org/antlr/v4/semantics/RuleCollector.java
View File

@ -30,13 +30,23 @@
package org.antlr.v4.semantics;
import org.antlr.v4.analysis.LeftRecursiveRuleAnalyzer;
import org.antlr.v4.misc.*;
import org.antlr.v4.parse.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.misc.OrderedHashMap;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.parse.ScopeParser;
import org.antlr.v4.tool.AttributeDict;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.RuleAST;
import org.stringtemplate.v4.misc.MultiMap;
import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class RuleCollector extends GrammarTreeVisitor {
/** which grammar are we checking */

27
tool/src/org/antlr/v4/semantics/SemanticPipeline.java
View File

@ -33,13 +33,15 @@ import org.antlr.v4.analysis.LeftRecursiveRuleTransformer;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Pair;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.GrammarAST;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** Do as much semantic checking as we can and fill in grammar
* with rules, actions, and token definitions.
@ -87,7 +89,7 @@ public class SemanticPipeline {
// TRANSFORM LEFT-RECURSIVE RULES
LeftRecursiveRuleTransformer lrtrans =
new LeftRecursiveRuleTransformer(g.ast, ruleCollector.rules.values(), g.tool);
new LeftRecursiveRuleTransformer(g.ast, ruleCollector.rules.values(), g);
lrtrans.translateLeftRecursiveRules();
// STORE RULES IN GRAMMAR
@ -162,10 +164,25 @@ public class SemanticPipeline {
}
// FOR ALL X : 'xxx'; RULES, DEFINE 'xxx' AS TYPE X
Map<String,String> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(g.ast);
List<Pair<GrammarAST,GrammarAST>> litAliases =
Grammar.getStringLiteralAliasesFromLexerRules(g.ast);
Set<String> conflictingLiterals = new HashSet<String>();
if ( litAliases!=null ) {
for (String lit : litAliases.keySet()) {
G.defineTokenAlias(litAliases.get(lit), lit);
for (Pair<GrammarAST,GrammarAST> pair : litAliases) {
GrammarAST nameAST = pair.a;
GrammarAST litAST = pair.b;
if ( !G.stringLiteralToTypeMap.containsKey(litAST.getText()) ) {
G.defineTokenAlias(nameAST.getText(), litAST.getText());
}
else {
// oops two literal defs in two rules (within or across modes).
conflictingLiterals.add(litAST.getText());
}
}
for (String lit : conflictingLiterals) {
// Remove literal if repeated across rules so it's not
// found by parser grammar.
G.stringLiteralToTypeMap.remove(lit);
}
}

67
tool/src/org/antlr/v4/semantics/SymbolChecks.java
View File

@ -30,10 +30,21 @@
package org.antlr.v4.semantics;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Alternative;
import org.antlr.v4.tool.ErrorManager;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.GrammarAST;
import java.util.*;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** Check for symbol problems; no side-effects. Inefficient to walk rules
* and such multiple times, but I like isolating all error checking outside
@ -79,7 +90,6 @@ public class SymbolChecks {
// done in sem pipe for now
checkForRuleConflicts(g.rules.values()); // sets nameToRuleMap
checkActionRedefinitions(collector.namedActions);
checkTokenAliasRedefinitions(collector.tokensDefs);
//checkRuleArgs(collector.rulerefs);
checkForTokenConflicts(collector.tokenIDRefs); // sets tokenIDs
checkForLabelConflicts(g.rules.values());
@ -129,57 +139,6 @@ public class SymbolChecks {
}
}
/** Catch:
tokens { A='a'; A; } can't redefine token type if has alias
tokens { A; A='a'; } can't redefine token type if has alias
tokens { A='a'; A='b'; } can't have two aliases for single token type
tokens { A='a'; B='a'; } can't have to token types for same string alias
*/
public void checkTokenAliasRedefinitions(List<GrammarAST> aliases) {
if ( aliases==null ) return;
// map names, strings to root of A or (= A 'a')
Map<String, GrammarAST> aliasTokenNames = new HashMap<String, GrammarAST>();
Map<String, GrammarAST> aliasStringValues = new HashMap<String, GrammarAST>();
for (int i=0; i<aliases.size(); i++) {
GrammarAST a = aliases.get(i);
GrammarAST idNode = a;
if ( a.getChildCount()>0 ) idNode = (GrammarAST)a.getChild(0);
GrammarAST prevToken = aliasTokenNames.get(idNode.getText());
GrammarAST stringNode = null;
if ( a.getChildCount()>0 ) stringNode = (GrammarAST)a.getChild(1);
GrammarAST prevString = null;
if ( stringNode!=null ) prevString = aliasStringValues.get(stringNode.getText());
if ( a.getType() == ANTLRParser.ASSIGN ) { // A='a'
idNode = (GrammarAST)a.getChild(0);
if ( prevString==null ) { // not seen string before
if ( stringNode!=null ) aliasStringValues.put(stringNode.getText(), a);
}
}
if ( prevToken==null ) { // not seen before, define it
aliasTokenNames.put(idNode.getText(), a);
}
// we've defined token names and strings at this point if not seen previously.
// now, look for trouble.
if ( prevToken!=null ) {
if ( !(prevToken.getChildCount()==0 && a.getChildCount()==0) ) {
// one or both have strings; disallow
errMgr.grammarError(ErrorType.TOKEN_NAME_REASSIGNMENT,
a.g.fileName, idNode.token, idNode.getText());
}
}
if ( prevString!=null ) {
// A='a' and A='a' are ok but not B='a' and A='a' are ok
if ( !prevString.getChild(0).getText().equals(idNode.getText()) ) {
errMgr.grammarError(ErrorType.TOKEN_STRING_REASSIGNMENT,
a.g.fileName, idNode.token, idNode.getText()+"="+stringNode.getText(),
prevString.getChild(0).getText());
}
}
}
}
public void checkForTokenConflicts(List<GrammarAST> tokenIDRefs) {
// for (GrammarAST a : tokenIDRefs) {
// Token t = a.token;

26
tool/src/org/antlr/v4/semantics/SymbolCollector.java
View File

@ -33,7 +33,13 @@ import org.antlr.v4.parse.GrammarTreeVisitor;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LabelElementPair;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTWithOptions;
import org.antlr.v4.tool.ast.PredAST;
import org.antlr.v4.tool.ast.RuleAST;
import org.antlr.v4.tool.ast.TerminalAST;
import java.util.ArrayList;
import java.util.HashSet;
@ -75,18 +81,10 @@ public class SymbolCollector extends GrammarTreeVisitor {
}
@Override
public void tokenAlias(GrammarAST ID, GrammarAST literal) {
if ( literal==null ) {
terminals.add(ID);
tokenIDRefs.add(ID);
tokensDefs.add(ID);
}
else {
terminals.add(ID);
tokenIDRefs.add(ID);
tokensDefs.add((GrammarAST)ID.getParent());
strings.add(literal.getText());
}
public void defineToken(GrammarAST ID) {
terminals.add(ID);
tokenIDRefs.add(ID);
tokensDefs.add(ID);
}
@Override
@ -162,7 +160,7 @@ public class SymbolCollector extends GrammarTreeVisitor {
@Override
public void ruleRef(GrammarAST ref, ActionAST arg) {
if ( inContext("DOT ...") ) qualifiedRulerefs.add((GrammarAST)ref.getParent());
// if ( inContext("DOT ...") ) qualifiedRulerefs.add((GrammarAST)ref.getParent());
rulerefs.add(ref);
if ( currentRule!=null ) {
currentRule.alt[currentOuterAltNumber].ruleRefs.map(ref.getText(), ref);

32
tool/src/org/antlr/v4/tool/DOTGenerator.java
View File

@ -30,7 +30,25 @@
package org.antlr.v4.tool;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ActionTransition;
import org.antlr.v4.runtime.atn.AtomTransition;
import org.antlr.v4.runtime.atn.BlockEndState;
import org.antlr.v4.runtime.atn.BlockStartState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.NotSetTransition;
import org.antlr.v4.runtime.atn.PlusBlockStartState;
import org.antlr.v4.runtime.atn.PlusLoopbackState;
import org.antlr.v4.runtime.atn.PredicateTransition;
import org.antlr.v4.runtime.atn.RangeTransition;
import org.antlr.v4.runtime.atn.RuleStopState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.SetTransition;
import org.antlr.v4.runtime.atn.StarBlockStartState;
import org.antlr.v4.runtime.atn.StarLoopEntryState;
import org.antlr.v4.runtime.atn.StarLoopbackState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.IntegerList;
@ -38,7 +56,12 @@ import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupDir;
import java.util.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
/** The DOT (part of graphviz) generation aspect. */
public class DOTGenerator {
@ -116,6 +139,9 @@ public class DOTGenerator {
if ( s.isAcceptState ) {
buf.append("=>").append(s.prediction);
}
if ( s.isCtxSensitive ) {
buf.append("^");
}
if ( grammar!=null && grammar.tool.verbose_dfa ) {
Set<Integer> alts = s.getAltSet();
if ( alts!=null ) {
@ -124,7 +150,7 @@ public class DOTGenerator {
IntegerList altList = new IntegerList();
altList.addAll(alts);
altList.sort();
Set<ATNConfig> configurations = s.configset;
Set<ATNConfig> configurations = s.configs;
for (int altIndex = 0; altIndex < altList.size(); altIndex++) {
int alt = altList.get(altIndex);
if ( altIndex>0 ) {

5
tool/src/org/antlr/v4/tool/ErrorManager.java
View File

@ -82,11 +82,6 @@ public class ErrorManager {
public ErrorManager(Tool tool) {
this.tool = tool;
// try to load the message format group
// the user might have specified one on the command line
// if not, or if the user has given an illegal value, we will fall back to "antlr"
setFormat("antlr");
//org.stringtemplate.v4.misc.ErrorManager.setErrorListener(theDefaultSTListener);
}
public void resetErrorState() {

36
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -28,6 +28,8 @@
*/
package org.antlr.v4.tool;
import org.antlr.v4.Tool;
/**
* A complex enumeration of all the error messages that the tool can issue.
*
@ -52,7 +54,7 @@ public enum ErrorType {
OUTPUT_DIR_IS_FILE(6, "output directory is a file: <arg>", ErrorSeverity.ERROR),
CANNOT_OPEN_FILE(7, "cannot find or open file: <arg><if(exception)>; reason: <exception><endif>", ErrorSeverity.ERROR),
FILE_AND_GRAMMAR_NAME_DIFFER(8, "grammar name <arg> and file name <arg2> differ", ErrorSeverity.ERROR),
// FILENAME_EXTENSION_ERROR("", ErrorSeverity.ERROR),
BAD_OPTION_SET_SYNTAX(9, "invalid -Dname=value syntax: <arg>", ErrorSeverity.ERROR),
INTERNAL_ERROR(20, "internal error: <arg> <arg2><if(exception)>: <exception><endif>\n" +
"<stackTrace; separator=\"\\n\">", ErrorSeverity.ERROR),
@ -61,7 +63,7 @@ public enum ErrorType {
// Code generation errors
MISSING_CODE_GEN_TEMPLATES(30, "can't find code generation templates: <arg>", ErrorSeverity.ERROR),
CANNOT_CREATE_TARGET_GENERATOR(31, "cannot create target <arg> code generator: <exception>", ErrorSeverity.ERROR),
CANNOT_CREATE_TARGET_GENERATOR(31, "ANTLR cannot generate <arg> code as of version "+ Tool.getVersion(), ErrorSeverity.ERROR),
CODE_TEMPLATE_ARG_ISSUE(32, "code generation template <arg> has missing, misnamed, or incomplete arg list; missing <arg2>", ErrorSeverity.ERROR),
CODE_GEN_TEMPLATES_INCOMPLETE(33, "missing code generation template <arg>", ErrorSeverity.ERROR),
NO_MODEL_TO_TEMPLATE_MAPPING(34, "no mapping to template name for output model class <arg>", ErrorSeverity.ERROR),
@ -85,7 +87,7 @@ public enum ErrorType {
UNKNOWN_RULE_ATTRIBUTE(65, "unknown attribute <arg> for rule <arg2> in <arg3>", ErrorSeverity.ERROR),
UNKNOWN_ATTRIBUTE_IN_SCOPE(66, "attribute <arg> isn't a valid property in <arg2>", ErrorSeverity.ERROR),
ISOLATED_RULE_REF(67, "missing attribute access on rule reference <arg> in <arg2>", ErrorSeverity.ERROR),
SYMBOL_CONFLICTS_WITH_GLOBAL_SCOPE(68, "symbol <arg> conflicts with global dynamic scope with same name", ErrorSeverity.ERROR),
SYMBOL_CONFLICTS_WITH_GLOBAL_SCOPE(68, "", ErrorSeverity.ERROR), // not used
LABEL_CONFLICTS_WITH_RULE(69, "label <arg> conflicts with rule with same name", ErrorSeverity.ERROR),
LABEL_CONFLICTS_WITH_TOKEN(70, "label <arg> conflicts with token with same name", ErrorSeverity.ERROR),
LABEL_CONFLICTS_WITH_RULE_ARG_RETVAL(72, "label <arg> conflicts with rule <arg2>'s return value or parameter with same name", ErrorSeverity.ERROR),
@ -93,7 +95,7 @@ public enum ErrorType {
ATTRIBUTE_CONFLICTS_WITH_RULE_ARG_RETVAL(74, "rule <arg2>'s dynamically-scoped attribute <arg> conflicts with <arg2>'s return value or parameter", ErrorSeverity.ERROR),
LABEL_TYPE_CONFLICT(75, "label <arg> type mismatch with previous definition: <arg2>", ErrorSeverity.ERROR),
ARG_RETVAL_CONFLICT(76, "rule <arg2>'s argument <arg> conflicts a return value with same name", ErrorSeverity.ERROR),
NONUNIQUE_REF(77, "arg> is a non-unique reference", ErrorSeverity.ERROR),
NONUNIQUE_REF(77, "", ErrorSeverity.ERROR), // not used
FORWARD_ELEMENT_REF(78, "", ErrorSeverity.ERROR),
MISSING_RULE_ARGS(79, "missing parameter(s) on rule reference: <arg>", ErrorSeverity.ERROR),
RULE_HAS_NO_ARGS(80, "rule <arg> has no defined parameters", ErrorSeverity.ERROR),
@ -102,9 +104,8 @@ public enum ErrorType {
ILLEGAL_OPTION(83, "illegal option <arg>", ErrorSeverity.WARNING),
ILLEGAL_OPTION_VALUE(84, "illegal option value <arg>=<arg2>", ErrorSeverity.WARNING),
LIST_LABEL_INVALID_UNLESS_RETVAL_STRUCT(85, "", ErrorSeverity.ERROR),
REWRITE_ELEMENT_NOT_PRESENT_ON_LHS(86, "reference to rewrite element <arg> not found to left of ->", ErrorSeverity.ERROR),
UNDEFINED_TOKEN_REF_IN_REWRITE(87, "token <arg> in rewrite is undefined", ErrorSeverity.ERROR),
///UNDEFINED_LABEL_REF_IN_REWRITE(, "", ErrorSeverity.ERROR), use previous
REWRITE_ELEMENT_NOT_PRESENT_ON_LHS(86, "", ErrorSeverity.ERROR),
UNDEFINED_TOKEN_REF_IN_REWRITE(87, "", ErrorSeverity.ERROR),
NO_GRAMMAR_START_RULE(88, "", ErrorSeverity.ERROR),
EMPTY_COMPLEMENT(89, "empty complement", ErrorSeverity.ERROR),
UNKNOWN_DYNAMIC_SCOPE(90, "unknown dynamic scope: <arg> in <arg2>", ErrorSeverity.ERROR),
@ -113,19 +114,19 @@ public enum ErrorType {
INVALID_ACTION_SCOPE(93, "", ErrorSeverity.ERROR),
ACTION_REDEFINITION(94, "redefinition of <arg> action", ErrorSeverity.ERROR),
SCOPE_REDEFINITION(95, "scope <arg> redefinition", ErrorSeverity.ERROR),
INVALID_TEMPLATE_ACTION(96, "invalid StringTemplate % shorthand syntax: '<arg>'", ErrorSeverity.ERROR),
INVALID_TEMPLATE_ACTION(96, "", ErrorSeverity.ERROR),
ARG_INIT_VALUES_ILLEGAL(97, "", ErrorSeverity.ERROR),
REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION(98, "<if(arg)>rule <arg> uses <endif>rewrite syntax or operator with no output option", ErrorSeverity.ERROR),
REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION(98, "", ErrorSeverity.ERROR),
NO_RULES(99, "<if(arg2.implicitLexerOwner)>implicitly generated <endif>grammar <arg> has no rules", ErrorSeverity.ERROR),
WRITE_TO_READONLY_ATTR(100, "", ErrorSeverity.ERROR),
MISSING_AST_TYPE_IN_TREE_GRAMMAR(101, "", ErrorSeverity.ERROR),
REWRITE_FOR_MULTI_ELEMENT_ALT(102, "with rewrite=true, alt <arg> not simple node or obvious tree element; text attribute for rule not guaranteed to be correct", ErrorSeverity.ERROR),
REWRITE_FOR_MULTI_ELEMENT_ALT(102, "", ErrorSeverity.ERROR),
RULE_INVALID_SET(103, "", ErrorSeverity.ERROR),
HETERO_ILLEGAL_IN_REWRITE_ALT(104, "alts with rewrites can't use heterogeneous types left of ->", ErrorSeverity.ERROR),
HETERO_ILLEGAL_IN_REWRITE_ALT(104, "", ErrorSeverity.ERROR),
NO_SUCH_GRAMMAR_SCOPE(105, "reference to undefined grammar in rule reference: <arg>.<arg2>", ErrorSeverity.ERROR),
NO_SUCH_RULE_IN_SCOPE(106, "rule <arg2> is not defined in grammar <arg>", ErrorSeverity.ERROR),
TOKEN_STRING_REASSIGNMENT(107, "cannot alias <arg>; string already assigned to <arg2>", ErrorSeverity.ERROR),
TOKEN_NAME_REASSIGNMENT(108, "cannot redefine <arg>; token name already <if(arg2)>assigned to <arg2><else>defined<endif>", ErrorSeverity.ERROR),
// TOKEN_STRING_REASSIGNMENT(107, "cannot alias <arg> in tokens {}; string already assigned to <arg2>", ErrorSeverity.ERROR),
// TOKEN_NAME_REASSIGNMENT(108, "cannot redefine <arg>; token name already <if(arg2)>assigned to <arg2><else>defined<endif>", ErrorSeverity.ERROR),
//TOKEN_VOCAB_IN_DELEGATE(, "tokenVocab option ignored in imported grammar <arg>", ErrorSeverity.ERROR),
OPTIONS_IN_DELEGATE(109, "options ignored in imported grammar <arg>", ErrorSeverity.WARNING),
// TOKEN_ALIAS_IN_DELEGATE(, "can't assign string to token name <arg> to string in imported grammar <arg2>", ErrorSeverity.ERROR),
@ -133,10 +134,10 @@ public enum ErrorType {
INVALID_IMPORT(111, "<arg.typeString> grammar <arg.name> cannot import <arg2.typeString> grammar <arg2.name>", ErrorSeverity.ERROR),
IMPORTED_TOKENS_RULE_EMPTY(112, "", ErrorSeverity.ERROR),
IMPORT_NAME_CLASH(113, "<arg.typeString> grammar <arg.name> and imported <arg2.typeString> grammar <arg2.name> both generate <arg2.recognizerName>", ErrorSeverity.ERROR),
AST_OP_WITH_NON_AST_OUTPUT_OPTION(114, "AST operator with non-AST output option: <arg>", ErrorSeverity.ERROR),
AST_OP_IN_ALT_WITH_REWRITE(115, "rule <arg> alt <arg2> uses rewrite syntax and also an AST operator", ErrorSeverity.ERROR),
WILDCARD_AS_ROOT(116, "Wildcard invalid as root; wildcard can itself be a tree", ErrorSeverity.ERROR),
CONFLICTING_OPTION_IN_TREE_FILTER(117, "option <arg>=<arg2> conflicts with tree grammar filter mode", ErrorSeverity.ERROR),
AST_OP_WITH_NON_AST_OUTPUT_OPTION(114, " <arg>", ErrorSeverity.ERROR),
AST_OP_IN_ALT_WITH_REWRITE(115, "", ErrorSeverity.ERROR),
WILDCARD_AS_ROOT(116, "", ErrorSeverity.ERROR),
CONFLICTING_OPTION_IN_TREE_FILTER(117, "", ErrorSeverity.ERROR),
ALL_OPS_NEED_SAME_ASSOC(118, "all operators of alt <arg> of left-recursive rule must have same associativity", ErrorSeverity.WARNING),
LEFT_RECURSION_CYCLES(119, "The following sets of rules are mutually left-recursive <arg:{c| [<c:{r|<r.name>}; separator=\", \">]}; separator=\" and \">", ErrorSeverity.ERROR),
MODE_NOT_IN_LEXER(120, "lexical modes are only allowed in lexer grammars", ErrorSeverity.ERROR),
@ -146,6 +147,7 @@ public enum ErrorType {
ALT_LABEL_CONFLICTS_WITH_RULE(124, "rule alt label <arg> conflicts with rule <arg2>", ErrorSeverity.ERROR),
IMPLICIT_TOKEN_DEFINITION(125, "implicit definition of token <arg> in parser", ErrorSeverity.WARNING),
IMPLICIT_STRING_DEFINITION(126, "cannot create implicit token for string literal <arg> in non-combined grammar", ErrorSeverity.ERROR),
// ALIAS_REASSIGNMENT(127, "token literal <arg> aliased to new token name <arg2>", ErrorSeverity.WARNING),
/** Documentation comment is unterminated */
//UNTERMINATED_DOC_COMMENT(, "", ErrorSeverity.ERROR),

89
tool/src/org/antlr/v4/tool/Grammar.java
View File

@ -50,6 +50,7 @@ import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.Pair;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.GrammarASTWithOptions;
@ -69,12 +70,40 @@ import java.util.Set;
public class Grammar implements AttributeResolver {
public static final String GRAMMAR_FROM_STRING_NAME = "<string>";
public static final Set<String> parserOptions = new HashSet<String>() {{
add("superClass");
add("TokenLabelType");
add("tokenVocab");
add("language");
}};
public static final Set<String> lexerOptions = parserOptions;
public static final Set<String> ruleOptions = new HashSet<String>() {{
}};
public static final Set<String> subruleOptions = new HashSet<String>() {{
add("greedy");
}};
/** Legal options for terminal refs like ID<assoc=right> */
public static final Set<String> tokenOptions = new HashSet<String>() {{
add("assoc");
}};
public static final Set<String> actionOptions = new HashSet<String>() {{
}};
public static final Set<String> semPredOptions = new HashSet<String>() {{
add("fail");
}};
public static final Set doNotCopyOptionsToLexer =
new HashSet() {
{
add("TokenLabelType"); add("superClass");
}
};
new HashSet() {{
add("superClass");
add("TokenLabelType");
add("tokenVocab");
}};
public static Map<String, AttributeDict> grammarAndLabelRefTypeToScope =
new HashMap<String, AttributeDict>() {{
@ -140,6 +169,7 @@ public class Grammar implements AttributeResolver {
* field will have entries both mapped to 35.
*/
public Map<String, Integer> stringLiteralToTypeMap = new LinkedHashMap<String, Integer>();
/** Reverse index for stringLiteralToTypeMap. Indexed with raw token type.
* 0 is invalid. */
public List<String> typeToStringLiteralList = new ArrayList<String>();
@ -370,11 +400,6 @@ public class Grammar implements AttributeResolver {
return parent.getOutermostGrammar();
}
public boolean isAbstract() {
return Boolean.parseBoolean(getOptionString("abstract"))
|| (tool != null && tool.abstract_recognizer);
}
/** Get the name of the generated recognizer; may or may not be same
* as grammar name.
* Recognizer is TParser and TLexer from T if combined, else
@ -390,15 +415,9 @@ public class Grammar implements AttributeResolver {
buf.append(g.name);
buf.append('_');
}
if (isAbstract()) {
buf.append("Abstract");
}
buf.append(name);
qualifiedName = buf.toString();
}
else if (isAbstract()) {
qualifiedName = "Abstract" + name;
}
if ( isCombined() || (isLexer() && implicitLexer!=null) )
{
@ -697,32 +716,6 @@ public class Grammar implements AttributeResolver {
}
public String getOptionString(String key) { return ast.getOptionString(key); }
public GrammarAST getOption(String key) { return ast.getOption(key); }
public String getOptionString(String key, String defaultValue) {
String v = ast.getOptionString(key);
if ( v!=null ) return v;
return defaultValue;
}
/** Manually get language option from tree */
// TODO: move to general tree visitor/parser class?
// TODO: don't need anymore as i set optins in parser?
public static String getLanguageOption(GrammarRootAST ast) {
GrammarAST options = (GrammarAST)ast.getFirstChildWithType(ANTLRParser.OPTIONS);
String language = "Java";
if ( options!=null ) {
for (Object o : options.getChildren()) {
GrammarAST c = (GrammarAST)o;
if ( c.getType() == ANTLRParser.ASSIGN &&
c.getChild(0).getText().equals("language") )
{
language = c.getChild(1).getText();
}
}
}
return language;
}
/** Given ^(TOKEN_REF ^(OPTIONS ^(ELEMENT_OPTIONS (= assoc right))))
* set option assoc=right in TOKEN_REF.
@ -741,7 +734,8 @@ public class Grammar implements AttributeResolver {
}
}
public static Map<String,String> getStringLiteralAliasesFromLexerRules(GrammarRootAST ast) {
/** Return list of (TOKEN_NAME node, 'literal' node) pairs */
public static List<Pair<GrammarAST,GrammarAST>> getStringLiteralAliasesFromLexerRules(GrammarRootAST ast) {
String[] patterns = {
"(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL)))",
"(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL ACTION)))",
@ -755,7 +749,8 @@ public class Grammar implements AttributeResolver {
};
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(ast.token.getInputStream());
TreeWizard wiz = new TreeWizard(adaptor,ANTLRParser.tokenNames);
Map<String,String> lexerRuleToStringLiteral = new HashMap<String,String>();
List<Pair<GrammarAST,GrammarAST>> lexerRuleToStringLiteral =
new ArrayList<Pair<GrammarAST,GrammarAST>>();
List<GrammarAST> ruleNodes = ast.getNodesWithType(ANTLRParser.RULE);
if ( ruleNodes==null || ruleNodes.isEmpty() ) return null;
@ -780,13 +775,15 @@ public class Grammar implements AttributeResolver {
protected static boolean defAlias(GrammarAST r, String pattern,
TreeWizard wiz,
Map<String, String> lexerRuleToStringLiteral)
List<Pair<GrammarAST,GrammarAST>> lexerRuleToStringLiteral)
{
HashMap<String, Object> nodes = new HashMap<String, Object>();
if ( wiz.parse(r, pattern, nodes) ) {
GrammarAST litNode = (GrammarAST)nodes.get("lit");
GrammarAST nameNode = (GrammarAST)nodes.get("name");
lexerRuleToStringLiteral.put(litNode.getText(), nameNode.getText());
Pair<GrammarAST, GrammarAST> pair =
new Pair<GrammarAST, GrammarAST>(nameNode, litNode);
lexerRuleToStringLiteral.add(pair);
return true;
}
return false;

16
tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java
View File

@ -34,10 +34,11 @@ import org.antlr.runtime.tree.Tree;
import org.antlr.runtime.tree.TreeVisitor;
import org.antlr.runtime.tree.TreeVisitorAction;
import org.antlr.v4.Tool;
import org.antlr.v4.misc.DoubleKeyMap;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.BlockSetTransformer;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import org.antlr.v4.runtime.misc.Pair;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
@ -50,7 +51,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** Handle left-recursion and block-set transforms */
@ -114,6 +114,7 @@ public class GrammarTransformPipeline {
/** Utility visitor that sets grammar ptr in each node */
public static void setGrammarPtr(final Grammar g, GrammarAST tree) {
if ( tree==null ) return;
// ensure each node has pointer to surrounding grammar
TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor());
v.visit(tree, new TreeVisitorAction() {
@ -351,15 +352,22 @@ public class GrammarTransformPipeline {
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
Map<String,String> litAliases =
List<Pair<GrammarAST,GrammarAST>> litAliases =
Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
Set<String> stringLiterals = combinedGrammar.getStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
nextLit:
for (String lit : stringLiterals) {
if ( litAliases!=null && litAliases.containsKey(lit) ) continue; // already has rule
// if lexer already has a rule for literal, continue
if ( litAliases!=null ) {
for (Pair<GrammarAST,GrammarAST> pair : litAliases) {
GrammarAST litAST = pair.b;
if ( lit.equals(litAST.getText()) ) continue nextLit;
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types

1
tool/src/org/antlr/v4/tool/ast/AltAST.java
View File

@ -56,6 +56,7 @@ public class AltAST extends GrammarAST {
public AltAST(Token t) { super(t); }
public AltAST(int type) { super(type); }
public AltAST(int type, Token t) { super(type, t); }
public AltAST(int type, Token t, String text) { super(type,t,text); }
@Override
public Tree dupNode() { return new AltAST(this); }

10
tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java
View File

@ -32,7 +32,8 @@ package org.antlr.v4.tool.ast;
import org.antlr.runtime.Token;
import org.antlr.v4.misc.CharSupport;
import java.util.*;
import java.util.HashMap;
import java.util.Map;
public abstract class GrammarASTWithOptions extends GrammarAST {
protected Map<String, GrammarAST> options;
@ -53,7 +54,7 @@ public abstract class GrammarASTWithOptions extends GrammarAST {
}
public String getOptionString(String key) {
GrammarAST value = getOption(key);
GrammarAST value = getOptionAST(key);
if ( value == null ) return null;
if ( value instanceof ActionAST ) {
return value.getText();
@ -67,7 +68,10 @@ public abstract class GrammarASTWithOptions extends GrammarAST {
}
}
public GrammarAST getOption(String key) {
/** Gets AST node holding value for option key; ignores default options
* and command-line forced options.
*/
public GrammarAST getOptionAST(String key) {
if ( options==null ) return null;
return options.get(key);
}

38
tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java
View File

@ -37,16 +37,16 @@ import java.util.HashMap;
import java.util.Map;
public class GrammarRootAST extends GrammarASTWithOptions {
public static final Map<String, String> defaultOptions =
new HashMap<String, String>() {
{
put("language","Java");
}
};
public int grammarType; // LEXER, PARSER, TREE, GRAMMAR (combined)
public static final Map<String, String> defaultOptions =
new HashMap<String, String>() {{
put("language","Java");
}};
public int grammarType; // LEXER, PARSER, GRAMMAR (combined)
public boolean hasErrors;
/** Track stream used to create this tree */
public TokenStream tokens;
public Map<String, String> cmdLineOptions; // -DsuperClass=T on command line
public String fileName;
public GrammarRootAST(GrammarAST node) {
super(node);
@ -54,9 +54,6 @@ public class GrammarRootAST extends GrammarASTWithOptions {
this.hasErrors = ((GrammarRootAST)node).hasErrors;
}
@Override
public Tree dupNode() { return new GrammarRootAST(this); }
public GrammarRootAST(int type) { super(type); }
public GrammarRootAST(Token t) { super(t); }
public GrammarRootAST(int type, Token t) { super(type, t); }
@ -64,6 +61,27 @@ public class GrammarRootAST extends GrammarASTWithOptions {
super(type,t,text);
}
public String getGrammarName() {
Tree t = getChild(0);
if ( t!=null ) return t.getText();
return null;
}
@Override
public String getOptionString(String key) {
if ( cmdLineOptions!=null && cmdLineOptions.containsKey(key) ) {
return cmdLineOptions.get(key);
}
String value = super.getOptionString(key);
if ( value==null ) {
value = defaultOptions.get(key);
}
return value;
}
@Override
public Object visit(GrammarASTVisitor v) { return v.visit(this); }
@Override
public Tree dupNode() { return new GrammarRootAST(this); }
}

Some files were not shown because too many files have changed in this diff Show More