forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

pull in Sam's ATN alt collapsing optimizations with optimizeStates off as it causes a class cast exception.

This commit is contained in:
Terence Parr 2012-08-04 13:32:07 -07:00
parent b7b2a45c8b c7d1ea7e23
commit c2b49bd94e
21 changed files with 487 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java
View File

@ -31,7 +31,12 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.IntervalSet;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ATNState {
public static final int INITIAL_NUM_TRANSITIONS = 4;
@ -137,6 +142,10 @@ public class ATNState {
transitions.set(i, e);
}
public Transition removeTransition(int index) {
return transitions.remove(index);
}
public int getStateType() {
return serializationTypes.get(this.getClass());
}

7
runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java
View File

@ -31,7 +31,7 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
public class ActionTransition extends Transition {
public final class ActionTransition extends Transition {
public final int ruleIndex;
public final int actionIndex;
public final boolean isCtxDependent; // e.g., $i ref in action
@ -47,6 +47,11 @@ public class ActionTransition extends Transition {
this.isCtxDependent = isCtxDependent;
}
@Override
public int getSerializationType() {
return ACTION;
}
@Override
public boolean isEpsilon() {
return true; // we are to be ignored by analysis 'cept for predicates

9
runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java
View File

@ -29,11 +29,11 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
/** TODO: make all transitions sets? no, should remove set edges */
public class AtomTransition extends Transition {
public final class AtomTransition extends Transition {
/** The token type or character value; or, signifies special label. */
public final int label;
@ -42,6 +42,11 @@ public class AtomTransition extends Transition {
this.label = label;
}
@Override
public int getSerializationType() {
return ATOM;
}
@Override
@NotNull
public IntervalSet label() { return IntervalSet.of(label); }

7
runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java
View File

@ -31,9 +31,14 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
public class EpsilonTransition extends Transition {
public final class EpsilonTransition extends Transition {
public EpsilonTransition(@NotNull ATNState target) { super(target); }
@Override
public int getSerializationType() {
return EPSILON;
}
@Override
public boolean isEpsilon() { return true; }

135
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -462,7 +462,8 @@ public class LexerATNSimulator extends ATNSimulator {
@Nullable
public ATNState getReachableTarget(Transition trans, int t) {
if ( trans instanceof AtomTransition ) {
switch (trans.getSerializationType()) {
case Transition.ATOM:
AtomTransition at = (AtomTransition)trans;
if ( at.label == t ) {
if ( debug ) {
@ -471,8 +472,10 @@ public class LexerATNSimulator extends ATNSimulator {
return at.target;
}
}
else if ( trans.getClass() == RangeTransition.class ) {
return null;
case Transition.RANGE:
RangeTransition rt = (RangeTransition)trans;
if ( t>=rt.from && t<=rt.to ) {
if ( debug ) {
@ -481,24 +484,44 @@ public class LexerATNSimulator extends ATNSimulator {
return rt.target;
}
}
else if ( trans instanceof SetTransition ) {
return null;
case Transition.SET:
SetTransition st = (SetTransition)trans;
boolean not = trans instanceof NotSetTransition;
if ( (!not && st.set.contains(t)) ||
(not && !st.set.contains(t) && t!=CharStream.EOF) ) // ~set doesn't not match EOF
{
if ( st.set.contains(t) ) {
if ( debug ) {
System.out.format("match %sset %s\n", not ? "~" : "", st.set.toString(true));
System.out.format("match set %s\n", st.set.toString(true));
}
return st.target;
}
return null;
case Transition.NOT_SET:
NotSetTransition nst = (NotSetTransition)trans;
if (!nst.set.contains(t) && t!=CharStream.EOF) // ~set doesn't not match EOF
{
if ( debug ) {
System.out.format("match ~set %s\n", nst.set.toString(true));
}
return nst.target;
}
return null;
case Transition.WILDCARD:
if (t != CharStream.EOF) {
return trans.target;
}
return null;
default:
return null;
}
else if ( trans instanceof WildcardTransition && t!=CharStream.EOF ) {
return trans.target;
}
return null;
}
/** Delete configs for alt following ci. Closure is unmodified; copy returned. */
@ -591,51 +614,55 @@ public class LexerATNSimulator extends ATNSimulator {
@NotNull ATNConfigSet configs)
{
ATNState p = config.state;
LexerATNConfig c = null;
if ( t.getClass() == RuleTransition.class ) {
PredictionContext newContext =
new SingletonPredictionContext(config.context, p.stateNumber);
c = new LexerATNConfig(config, t.target, newContext);
}
else if ( t.getClass() == PredicateTransition.class ) {
if (recog == null) {
System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");
}
switch (t.getSerializationType()) {
case Transition.RULE:
PredictionContext newContext =
new SingletonPredictionContext(config.context, p.stateNumber);
c = new LexerATNConfig(config, t.target, newContext);
break;
case Transition.PREDICATE:
// if (recog == null) {
// System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");
// }
/* Track traversing semantic predicates. If we traverse,
we cannot add a DFA state for this "reach" computation
because the DFA would not test the predicate again in the
future. Rather than creating collections of semantic predicates
like v3 and testing them on prediction, v4 will test them on the
fly all the time using the ATN not the DFA. This is slower but
semantically it's not used that often. One of the key elements to
this predicate mechanism is not adding DFA states that see
predicates immediately afterwards in the ATN. For example,
/* Track traversing semantic predicates. If we traverse,
we cannot add a DFA state for this "reach" computation
because the DFA would not test the predicate again in the
future. Rather than creating collections of semantic predicates
like v3 and testing them on prediction, v4 will test them on the
fly all the time using the ATN not the DFA. This is slower but
semantically it's not used that often. One of the key elements to
this predicate mechanism is not adding DFA states that see
predicates immediately afterwards in the ATN. For example,
a : ID {p1}? | ID {p2}? ;
a : ID {p1}? | ID {p2}? ;
should create the start state for rule 'a' (to save start state
competition), but should not create target of ID state. The
collection of ATN states the following ID references includes
states reached by traversing predicates. Since this is when we
test them, we cannot cash the DFA state target of ID.
*/
PredicateTransition pt = (PredicateTransition)t;
if ( debug ) {
System.out.println("EVAL rule "+pt.ruleIndex+":"+pt.predIndex);
}
configs.hasSemanticContext = true;
if ( recog == null || recog.sempred(null, pt.ruleIndex, pt.predIndex) ) {
c = new LexerATNConfig(config, t.target, pt.getPredicate());
}
}
// ignore actions; just exec one per rule upon accept
else if ( t.getClass() == ActionTransition.class ) {
c = new LexerATNConfig(config, t.target, ((ActionTransition)t).actionIndex);
}
else if ( t.isEpsilon() ) {
c = new LexerATNConfig(config, t.target);
should create the start state for rule 'a' (to save start state
competition), but should not create target of ID state. The
collection of ATN states the following ID references includes
states reached by traversing predicates. Since this is when we
test them, we cannot cash the DFA state target of ID.
*/
PredicateTransition pt = (PredicateTransition)t;
if ( debug ) {
System.out.println("EVAL rule "+pt.ruleIndex+":"+pt.predIndex);
}
configs.hasSemanticContext = true;
if ( recog == null || recog.sempred(null, pt.ruleIndex, pt.predIndex) ) {
c = new LexerATNConfig(config, t.target, pt.getPredicate());
}
break;
// ignore actions; just exec one per rule upon accept
case Transition.ACTION:
c = new LexerATNConfig(config, t.target, ((ActionTransition)t).actionIndex);
break;
case Transition.EPSILON:
c = new LexerATNConfig(config, t.target);
break;
}
return c;
}

7
runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java
View File

@ -33,11 +33,16 @@ import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
public class NotSetTransition extends SetTransition {
public final class NotSetTransition extends SetTransition {
public NotSetTransition(@NotNull ATNState target, @Nullable IntervalSet set) {
super(target, set);
}
@Override
public int getSerializationType() {
return NOT_SET;
}
@Override
public String toString() {
return '~'+super.toString();

60
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -911,27 +911,44 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
@Nullable
public ATNState getReachableTarget(@NotNull Transition trans, int ttype) {
if ( trans instanceof AtomTransition ) {
switch (trans.getSerializationType()) {
case Transition.ATOM:
AtomTransition at = (AtomTransition)trans;
if ( at.label == ttype ) {
return at.target;
}
}
else if ( trans instanceof SetTransition ) {
return null;
case Transition.SET:
SetTransition st = (SetTransition)trans;
boolean not = trans instanceof NotSetTransition;
if ( !not && st.set.contains(ttype) || not && !st.set.contains(ttype) ) {
if ( st.set.contains(ttype) ) {
return st.target;
}
}
else if ( trans instanceof RangeTransition ) { // TODO: can't happen in parser, right? remove
return null;
case Transition.NOT_SET:
NotSetTransition nst = (NotSetTransition)trans;
if ( !nst.set.contains(ttype) ) {
return nst.target;
}
return null;
case Transition.RANGE:
RangeTransition rt = (RangeTransition)trans;
if ( ttype>=rt.from && ttype<=rt.to ) return rt.target;
if ( ttype>=rt.from && ttype<=rt.to ) {
return rt.target;
}
return null;
case Transition.WILDCARD:
if (ttype != Token.EOF) {
return trans.target;
}
return null;
default:
return null;
}
else if ( trans instanceof WildcardTransition && ttype!=Token.EOF ) {
return trans.target;
}
return null;
}
public SemanticContext[] getPredsForAmbigAlts(@NotNull IntervalSet ambigAlts,
@ -1214,22 +1231,25 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
boolean inContext,
boolean fullCtx)
{
if ( t instanceof RuleTransition ) {
switch (t.getSerializationType()) {
case Transition.RULE:
return ruleTransition(config, t);
}
else if ( t instanceof PredicateTransition ) {
case Transition.PREDICATE:
return predTransition(config, (PredicateTransition)t,
collectPredicates,
inContext,
fullCtx);
}
else if ( t instanceof ActionTransition ) {
case Transition.ACTION:
return actionTransition(config, (ActionTransition)t);
}
else if ( t.isEpsilon() ) {
case Transition.EPSILON:
return new ATNConfig(config, t.target);
default:
return null;
}
return null;
}
@NotNull

7
runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java
View File

@ -37,7 +37,7 @@ import org.antlr.v4.runtime.misc.NotNull;
* may have to combine a bunch of them as it collects predicates from
* multiple ATN configurations into a single DFA state.
*/
public class PredicateTransition extends Transition {
public final class PredicateTransition extends Transition {
public final int ruleIndex;
public final int predIndex;
public final boolean isCtxDependent; // e.g., $i ref in pred
@ -49,6 +49,11 @@ public class PredicateTransition extends Transition {
this.isCtxDependent = isCtxDependent;
}
@Override
public int getSerializationType() {
return PREDICATE;
}
@Override
public boolean isEpsilon() { return true; }

9
runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java
View File

@ -29,10 +29,10 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
public class RangeTransition extends Transition {
public final class RangeTransition extends Transition {
public final int from;
public final int to;
@ -42,6 +42,11 @@ public class RangeTransition extends Transition {
this.to = to;
}
@Override
public int getSerializationType() {
return RANGE;
}
@Override
@NotNull
public IntervalSet label() { return IntervalSet.of(from, to); }

7
runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java
View File

@ -32,7 +32,7 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
/** */
public class RuleTransition extends Transition {
public final class RuleTransition extends Transition {
/** Ptr to the rule definition object for this rule ref */
public final int ruleIndex; // no Rule object at runtime
@ -49,6 +49,11 @@ public class RuleTransition extends Transition {
this.followState = followState;
}
@Override
public int getSerializationType() {
return RULE;
}
@Override
public boolean isEpsilon() { return true; }
}

9
runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java
View File

@ -29,10 +29,10 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
/** A transition containing a set of values */
public class SetTransition extends Transition {
@ -46,6 +46,11 @@ public class SetTransition extends Transition {
this.set = set;
}
@Override
public int getSerializationType() {
return SET;
}
@Override
@NotNull
public IntervalSet label() { return set; }

8
runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java
View File

@ -33,7 +33,11 @@ import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/** An ATN transition between any two ATN states. Subclasses define
* atom, set, epsilon, action, predicate, rule transitions.
@ -99,7 +103,7 @@ public abstract class Transition {
this.target = target;
}
public int getSerializationType() { return 0; }
public abstract int getSerializationType();
/** Are we epsilon, action, sempred? */
public boolean isEpsilon() { return false; }

7
runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java
View File

@ -31,9 +31,14 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
public class WildcardTransition extends Transition {
public final class WildcardTransition extends Transition {
public WildcardTransition(@NotNull ATNState target) { super(target); }
@Override
public int getSerializationType() {
return WILDCARD;
}
@Override
@NotNull
public String toString() {

4
tool/playground/A.g4
View File

@ -2,8 +2,8 @@ grammar A;
s : e ;
e : e '*' e -> Mult
| INT -> primary
e : e '*' e # Mult
| INT # primary
;
INT : [0-9]+ ;

14
tool/playground/LL1.g
View File

@ -1,14 +0,0 @@
grammar LL1;
b : B | C ;
c1 : A? B ;
c2 : (B|C)? D ;
d1 : A* B ;
d2 : ({true}? B | {true}? A)* D {System.out.println("works!");} ;
e1 : A+ B ;
e2 : (B|A)+ D ;

9
tool/src/org/antlr/v4/automata/ATNFactory.java
View File

@ -31,7 +31,11 @@ package org.antlr.v4.automata;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.tool.ast.*;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.BlockAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.PredAST;
import org.antlr.v4.tool.ast.TerminalAST;
import java.util.List;
@ -74,9 +78,6 @@ public interface ATNFactory {
Handle charSetLiteral(GrammarAST charSetAST);
Handle tree(GrammarAST node, List<Handle> els);
Handle range(GrammarAST a, GrammarAST b);
/** For a non-lexer, just build a simple token reference atom.

191
tool/src/org/antlr/v4/automata/ATNOptimizer.java Normal file
View File

@ -0,0 +1,191 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
Copyright (c) 2012 Sam Harwell
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.automata;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.AtomTransition;
import org.antlr.v4.runtime.atn.BlockEndState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.EpsilonTransition;
import org.antlr.v4.runtime.atn.NotSetTransition;
import org.antlr.v4.runtime.atn.RangeTransition;
import org.antlr.v4.runtime.atn.SetTransition;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.util.ArrayList;
import java.util.List;
/**
*
* @author Sam Harwell
*/
public class ATNOptimizer {
public static void optimize(@NotNull Grammar g, @NotNull ATN atn) {
optimizeSets(g, atn);
// optimizeStates(atn); // TODO leads to class cast exception
}
private static void optimizeSets(Grammar g, ATN atn) {
if (g.isParser()) {
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
List<DecisionState> decisions = atn.decisionToState;
for (DecisionState decision : decisions) {
if (decision.ruleIndex >= 0) {
Rule rule = g.getRule(decision.ruleIndex);
if (Character.isLowerCase(rule.name.charAt(0))) {
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.getNumberOfTransitions(); i++) {
Transition epsTransition = decision.transition(i);
if (!(epsTransition instanceof EpsilonTransition)) {
continue;
}
if (epsTransition.target.getNumberOfTransitions() != 1) {
continue;
}
Transition transition = epsTransition.target.transition(0);
if (!(transition.target instanceof BlockEndState)) {
continue;
}
if (transition instanceof NotSetTransition) {
// TODO: not yet implemented
continue;
}
if (transition instanceof AtomTransition
|| transition instanceof RangeTransition
|| transition instanceof SetTransition)
{
setTransitions.add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.getIntervals().size() - 1; i >= 0; i--) {
Interval interval = setTransitions.getIntervals().get(i);
if (interval.length() <= 1) {
continue;
}
ATNState blockEndState = decision.transition(interval.a).target.transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++) {
Transition matchTransition = decision.transition(j).target.transition(0);
if (matchTransition instanceof NotSetTransition) {
throw new UnsupportedOperationException("Not yet implemented.");
} else {
matchSet.addAll(matchTransition.label());
}
}
Transition newTransition;
if (matchSet.getIntervals().size() == 1) {
if (matchSet.size() == 1) {
newTransition = new AtomTransition(blockEndState, matchSet.getMinElement());
} else {
Interval matchInterval = matchSet.getIntervals().get(0);
newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
}
} else {
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.transition(interval.a).target.setTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++) {
Transition removed = decision.removeTransition(interval.a + 1);
atn.removeState(removed.target);
removedStates++;
}
}
}
// System.out.println("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
private static void optimizeStates(ATN atn) {
System.out.println(atn.states);
List<ATNState> compressed = new ArrayList<ATNState>();
int i = 0; // new state number
for (ATNState s : atn.states) {
if ( s!=null ) {
compressed.add(s);
s.stateNumber = i; // reset state number as we shift to new position
i++;
}
}
System.out.println(compressed);
// System.out.println("ATN optimizer removed " + (atn.states.size() - compressed.size()) + " null states.");
atn.states.clear();
atn.states.addAll(compressed);
// List<ATNState> states = atn.states;
// int current = 0;
// for (int i = 0; i < states.size(); i++) {
// ATNState state = states.get(i);
// if (state == null) {
// continue;
// }
//
// if (i != current) {
// state.stateNumber = current;
// states.set(current, state);
// states.set(i, null);
// }
//
// current++;
// }
//
// System.out.println("ATN optimizer removed " + (states.size() - current) + " null states.");
// states.subList(current, states.size()).clear();
}
private ATNOptimizer() {
}
}

2
tool/src/org/antlr/v4/automata/ATNVisitor.java
View File

@ -57,5 +57,5 @@ public class ATNVisitor {
}
}
public void visitState(ATNState s) { }
public void visitState(@NotNull ATNState s) { }
}

1
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -104,6 +104,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
}
ATNOptimizer.optimize(g, atn);
return atn;
}

56
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -83,41 +83,6 @@ import java.util.List;
* No side-effects. It builds an ATN object and returns it.
*/
public class ParserATNFactory implements ATNFactory {
class TailEpsilonRemover extends ATNVisitor {
@Override
public void visitState(ATNState p) {
if ( p.getClass() == ATNState.class && p.getNumberOfTransitions()==1 ) {
ATNState q = p.transition(0).target;
if ( p.transition(0) instanceof RuleTransition ) {
q = ((RuleTransition)p.transition(0)).followState;
}
if ( q.getClass() == ATNState.class ) {
// we have p-x->q for x in {rule, action, pred, token, ...}
// if edge out of q is single epsilon to block end
// we can strip epsilon p-x->q-eps->r
Transition trans = q.transition(0);
if ( q.getNumberOfTransitions()==1 && trans.isEpsilon() &&
!(trans instanceof ActionTransition) ) {
ATNState r = trans.target;
if ( r instanceof BlockEndState ||
r instanceof PlusLoopbackState ||
r instanceof StarLoopbackState )
{
// skip over q
if ( p.transition(0) instanceof RuleTransition ) {
((RuleTransition)p.transition(0)).followState = r;
}
else {
p.transition(0).target = r;
}
atn.removeState(q);
}
}
}
}
}
}
@NotNull
public final Grammar g;
@ -128,7 +93,14 @@ public class ParserATNFactory implements ATNFactory {
public int currentOuterAlt;
public ParserATNFactory(@NotNull Grammar g) { this.g = g; atn = new ATN(); }
public ParserATNFactory(@NotNull Grammar g) {
if (g == null) {
throw new NullPointerException("g");
}
this.g = g;
this.atn = new ATN();
}
@Override
public ATN createATN() {
@ -136,6 +108,7 @@ public class ParserATNFactory implements ATNFactory {
atn.maxTokenType = g.getMaxTokenType();
addRuleFollowLinks();
addEOFTransitionToStartRules();
ATNOptimizer.optimize(g, atn);
return atn;
}
@ -218,11 +191,6 @@ public class ParserATNFactory implements ATNFactory {
return new Handle(left, right);
}
@Override
public Handle tree(GrammarAST node, List<Handle> els) {
throw new UnsupportedOperationException("^(...) not allowed in non-tree grammar");
}
/** Not valid for non-lexers */
@Override
public Handle range(GrammarAST a, GrammarAST b) {
@ -316,7 +284,7 @@ public class ParserATNFactory implements ATNFactory {
/** Build what amounts to an epsilon transition with an action.
* The action goes into ATN though it is ignored during prediction
* if actionIndex < 0. Only forced are executed during prediction.
* if actionIndex &lt; 0. Only forced are executed during prediction.
*/
@Override
public Handle action(ActionAST action) {
@ -351,7 +319,7 @@ public class ParserATNFactory implements ATNFactory {
* begin/end.
*
* Special case: if just a list of tokens/chars/sets, then collapse
* to a single edge'd o-set->o graph.
* to a single edged o-set->o graph.
*
* TODO: Set alt number (1..n) in the states?
*/
@ -396,7 +364,7 @@ public class ParserATNFactory implements ATNFactory {
epsilon(alt.right, end);
// no back link in ATN so must walk entire alt to see if we can
// strip out the epsilon to 'end' state
TailEpsilonRemover opt = new TailEpsilonRemover();
TailEpsilonRemover opt = new TailEpsilonRemover(atn);
opt.visit(alt.left);
}
Handle h = new Handle(start, end);

81
tool/src/org/antlr/v4/automata/TailEpsilonRemover.java Normal file
View File

@ -0,0 +1,81 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.automata;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ActionTransition;
import org.antlr.v4.runtime.atn.BlockEndState;
import org.antlr.v4.runtime.atn.PlusLoopbackState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.StarLoopbackState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.NotNull;
/**
*
* @author Terence Parr
*/
public class TailEpsilonRemover extends ATNVisitor {
@NotNull
private final ATN _atn;
public TailEpsilonRemover(@NotNull ATN atn) {
this._atn = atn;
}
@Override
public void visitState(@NotNull ATNState p) {
if (p.getClass() == ATNState.class && p.getNumberOfTransitions() == 1) {
ATNState q = p.transition(0).target;
if (p.transition(0) instanceof RuleTransition) {
q = ((RuleTransition) p.transition(0)).followState;
}
if (q.getClass() == ATNState.class) {
// we have p-x->q for x in {rule, action, pred, token, ...}
// if edge out of q is single epsilon to block end
// we can strip epsilon p-x->q-eps->r
Transition trans = q.transition(0);
if (q.getNumberOfTransitions() == 1 && trans.isEpsilon() && !(trans instanceof ActionTransition)) {
ATNState r = trans.target;
if (r instanceof BlockEndState || r instanceof PlusLoopbackState || r instanceof StarLoopbackState) {
// skip over q
if (p.transition(0) instanceof RuleTransition) {
((RuleTransition) p.transition(0)).followState = r;
} else {
p.transition(0).target = r;
}
_atn.removeState(q);
}
}
}
}
}
}