jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge remote-tracking branch 'sharwell/atn-optimization'

This commit is contained in:
Terence Parr 2012-08-04 12:08:43 -07:00
parent 20d96b0a86 5019278204
commit c7d1ea7e23
5 changed files with 264 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java
View File

@ -137,6 +137,10 @@ public class ATNState {
transitions.set(i, e);
}
public Transition removeTransition(int index) {
return transitions.remove(index);
}
public int getStateType() {
return serializationTypes.get(this.getClass());
}

176
tool/src/org/antlr/v4/automata/ATNOptimizer.java Normal file
View File

@ -0,0 +1,176 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
Copyright (c) 2012 Sam Harwell
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.automata;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.AtomTransition;
import org.antlr.v4.runtime.atn.BlockEndState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.EpsilonTransition;
import org.antlr.v4.runtime.atn.NotSetTransition;
import org.antlr.v4.runtime.atn.RangeTransition;
import org.antlr.v4.runtime.atn.SetTransition;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.util.List;
/**
*
* @author Sam Harwell
*/
public class ATNOptimizer {
public static void optimize(@NotNull Grammar g, @NotNull ATN atn) {
optimizeSets(g, atn);
optimizeStates(atn);
}
private static void optimizeSets(Grammar g, ATN atn) {
if (g.isParser()) {
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
List<DecisionState> decisions = atn.decisionToState;
for (DecisionState decision : decisions) {
if (decision.ruleIndex >= 0) {
Rule rule = g.getRule(decision.ruleIndex);
if (Character.isLowerCase(rule.name.charAt(0))) {
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.getNumberOfTransitions(); i++) {
Transition epsTransition = decision.transition(i);
if (!(epsTransition instanceof EpsilonTransition)) {
continue;
}
if (epsTransition.target.getNumberOfTransitions() != 1) {
continue;
}
Transition transition = epsTransition.target.transition(0);
if (!(transition.target instanceof BlockEndState)) {
continue;
}
if (transition instanceof NotSetTransition) {
// TODO: not yet implemented
continue;
}
if (transition instanceof AtomTransition
|| transition instanceof RangeTransition
|| transition instanceof SetTransition)
{
setTransitions.add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.getIntervals().size() - 1; i >= 0; i--) {
Interval interval = setTransitions.getIntervals().get(i);
if (interval.length() <= 1) {
continue;
}
ATNState blockEndState = decision.transition(interval.a).target.transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++) {
Transition matchTransition = decision.transition(j).target.transition(0);
if (matchTransition instanceof NotSetTransition) {
throw new UnsupportedOperationException("Not yet implemented.");
} else {
matchSet.addAll(matchTransition.label());
}
}
Transition newTransition;
if (matchSet.getIntervals().size() == 1) {
if (matchSet.size() == 1) {
newTransition = new AtomTransition(blockEndState, matchSet.getMinElement());
} else {
Interval matchInterval = matchSet.getIntervals().get(0);
newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
}
} else {
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.transition(interval.a).target.setTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++) {
Transition removed = decision.removeTransition(interval.a + 1);
atn.removeState(removed.target);
removedStates++;
}
}
}
System.out.println("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
private static void optimizeStates(ATN atn) {
List<ATNState> states = atn.states;
int current = 0;
for (int i = 0; i < states.size(); i++) {
ATNState state = states.get(i);
if (state == null) {
continue;
}
if (i != current) {
state.stateNumber = current;
states.set(current, state);
states.set(i, null);
}
current++;
}
System.out.println("ATN optimizer removed " + (states.size() - current) + " null states.");
states.subList(current, states.size()).clear();
}
private ATNOptimizer() {
}
}

1
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -104,6 +104,7 @@ public class LexerATNFactory extends ParserATNFactory {
}
}
ATNOptimizer.optimize(g, atn);
return atn;
}

38
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -58,41 +58,6 @@ import java.util.List;
* No side-effects. It builds an ATN object and returns it.
*/
public class ParserATNFactory implements ATNFactory {
class TailEpsilonRemover extends ATNVisitor {
@Override
public void visitState(ATNState p) {
if ( p.getClass() == ATNState.class && p.getNumberOfTransitions()==1 ) {
ATNState q = p.transition(0).target;
if ( p.transition(0) instanceof RuleTransition ) {
q = ((RuleTransition)p.transition(0)).followState;
}
if ( q.getClass() == ATNState.class ) {
// we have p-x->q for x in {rule, action, pred, token, ...}
// if edge out of q is single epsilon to block end
// we can strip epsilon p-x->q-eps->r
Transition trans = q.transition(0);
if ( q.getNumberOfTransitions()==1 && trans.isEpsilon() &&
!(trans instanceof ActionTransition) ) {
ATNState r = trans.target;
if ( r instanceof BlockEndState ||
r instanceof PlusLoopbackState ||
r instanceof StarLoopbackState )
{
// skip over q
if ( p.transition(0) instanceof RuleTransition ) {
((RuleTransition)p.transition(0)).followState = r;
}
else {
p.transition(0).target = r;
}
atn.removeState(q);
}
}
}
}
}
}
@NotNull
public final Grammar g;
@ -118,6 +83,7 @@ public class ParserATNFactory implements ATNFactory {
atn.maxTokenType = g.getMaxTokenType();
addRuleFollowLinks();
addEOFTransitionToStartRules();
ATNOptimizer.optimize(g, atn);
return atn;
}
@ -375,7 +341,7 @@ public class ParserATNFactory implements ATNFactory {
epsilon(alt.right, end);
// no back link in ATN so must walk entire alt to see if we can
// strip out the epsilon to 'end' state
TailEpsilonRemover opt = new TailEpsilonRemover();
TailEpsilonRemover opt = new TailEpsilonRemover(atn);
opt.visit(alt.left);
}
Handle h = new Handle(start, end);

81
tool/src/org/antlr/v4/automata/TailEpsilonRemover.java Normal file
View File

@ -0,0 +1,81 @@
/*
[The "BSD license"]
Copyright (c) 2012 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.automata;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ActionTransition;
import org.antlr.v4.runtime.atn.BlockEndState;
import org.antlr.v4.runtime.atn.PlusLoopbackState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.StarLoopbackState;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.NotNull;
/**
*
* @author Terence Parr
*/
public class TailEpsilonRemover extends ATNVisitor {
@NotNull
private final ATN _atn;
public TailEpsilonRemover(@NotNull ATN atn) {
this._atn = atn;
}
@Override
public void visitState(@NotNull ATNState p) {
if (p.getClass() == ATNState.class && p.getNumberOfTransitions() == 1) {
ATNState q = p.transition(0).target;
if (p.transition(0) instanceof RuleTransition) {
q = ((RuleTransition) p.transition(0)).followState;
}
if (q.getClass() == ATNState.class) {
// we have p-x->q for x in {rule, action, pred, token, ...}
// if edge out of q is single epsilon to block end
// we can strip epsilon p-x->q-eps->r
Transition trans = q.transition(0);
if (q.getNumberOfTransitions() == 1 && trans.isEpsilon() && !(trans instanceof ActionTransition)) {
ATNState r = trans.target;
if (r instanceof BlockEndState || r instanceof PlusLoopbackState || r instanceof StarLoopbackState) {
// skip over q
if (p.transition(0) instanceof RuleTransition) {
((RuleTransition) p.transition(0)).followState = r;
} else {
p.transition(0).target = r;
}
_atn.removeState(q);
}
}
}
}
}
}