forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Improve ParserInterpreter code simplicity

This commit is contained in:
parrt 2015-06-05 16:14:20 -07:00
parent b58eb66774
commit dc445af663
14 changed files with 444 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java
View File

@ -43,7 +43,7 @@ package org.antlr.v4.runtime;
*/
public class InterpreterRuleContext extends ParserRuleContext {
/** This is the backing field for {@link #getRuleIndex}. */
private int ruleIndex = -1;
protected int ruleIndex = -1;
public InterpreterRuleContext() { }
@ -67,17 +67,4 @@ public class InterpreterRuleContext extends ParserRuleContext {
public int getRuleIndex() {
return ruleIndex;
}
/** Copy a {@link ParserRuleContext} or {@link InterpreterRuleContext}
* stack to a {@link InterpreterRuleContext} tree.
* Return {@link null} if {@code ctx} is null.
*/
public static InterpreterRuleContext fromParserRuleContext(ParserRuleContext ctx) {
if ( ctx==null ) return null;
InterpreterRuleContext dup = new InterpreterRuleContext();
dup.copyFrom(ctx);
dup.ruleIndex = ctx.getRuleIndex();
dup.parent = fromParserRuleContext(ctx.getParent());
return dup;
}
}

2
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -823,7 +823,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
// Create a new parser interpreter to parse the ambiguous subphrase
ParserInterpreter parser;
if ( originalParser instanceof ParserInterpreter ) {
parser = new ParserInterpreter((ParserInterpreter) originalParser);
parser = ((ParserInterpreter)originalParser).copyFrom((ParserInterpreter)originalParser);
}
else {
char[] serializedAtn = ATNSerializer.getSerializedAsChars(originalParser.getATN());

152
runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java
View File

@ -48,7 +48,7 @@ import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.Pair;
import java.util.ArrayDeque;
import java.util.BitSet;
import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
@ -68,14 +68,9 @@ import java.util.Deque;
public class ParserInterpreter extends Parser {
protected final String grammarFileName;
protected final ATN atn;
/** This identifies StarLoopEntryState's that begin the (...)*
* precedence loops of left recursive rules.
*/
protected final BitSet statesNeedingLeftRecursionContext;
protected final DFA[] decisionToDFA; // not shared like it is for generated parsers
protected final PredictionContextCache sharedContextCache =
new PredictionContextCache();
protected final PredictionContextCache sharedContextCache = new PredictionContextCache();
@Deprecated
protected final String[] tokenNames;
@ -83,7 +78,19 @@ public class ParserInterpreter extends Parser {
private final Vocabulary vocabulary;
/** Tracks LR rules for adjusting the contexts */
/** This stack corresponds to the _parentctx, _parentState pair of locals
* that would exist on call stack frames with a recursive descent parser;
* in the generated function for a left-recursive rule you'd see:
*
* private EContext e(int _p) throws RecognitionException {
* ParserRuleContext _parentctx = _ctx; // Pair.a
* int _parentState = getState(); // Pair.b
* ...
* }
*
* Those values are used to create new recursive rule invocation contexts
* associated with left operand of an alt like "expr '*' expr".
*/
protected final Deque<Pair<ParserRuleContext, Integer>> _parentContextStack =
new ArrayDeque<Pair<ParserRuleContext, Integer>>();
@ -94,27 +101,6 @@ public class ParserInterpreter extends Parser {
protected int overrideDecisionInputIndex = -1;
protected int overrideDecisionAlt = -1;
/** A copy constructor that creates a new parser interpreter by reusing
* the fields of a previous interpreter.
*
* @since 4.5.1
*
* @param old The interpreter to copy
*/
public ParserInterpreter(ParserInterpreter old) {
super(old.getTokenStream());
this.atn = old.atn;
this.grammarFileName = old.grammarFileName;
this.statesNeedingLeftRecursionContext = old.statesNeedingLeftRecursionContext;
this.decisionToDFA = old.decisionToDFA;
this.tokenNames = old.tokenNames;
this.ruleNames = old.ruleNames;
this.vocabulary = old.vocabulary;
setInterpreter(new ParserATNSimulator(this, atn,
decisionToDFA,
sharedContextCache));
}
/**
* @deprecated Use {@link #ParserInterpreter(String, Vocabulary, Collection, ATN, TokenStream)} instead.
*/
@ -137,21 +123,13 @@ public class ParserInterpreter extends Parser {
this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]);
this.vocabulary = vocabulary;
this.decisionToDFA = new DFA[atn.getNumberOfDecisions()];
for (int i = 0; i < decisionToDFA.length; i++) {
decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
}
// identify the ATN states where pushNewRecursionContext() must be called
this.statesNeedingLeftRecursionContext = new BitSet(atn.states.size());
for (ATNState state : atn.states) {
if (!(state instanceof StarLoopEntryState)) {
continue;
}
if (((StarLoopEntryState)state).precedenceRuleDecision) {
this.statesNeedingLeftRecursionContext.set(state.stateNumber);
}
// init decision DFA
int numberOfDecisions = atn.getNumberOfDecisions();
this.decisionToDFA = new DFA[numberOfDecisions];
for (int i = 0; i < numberOfDecisions; i++) {
DecisionState decisionState = atn.getDecisionState(i);
decisionToDFA[i] = new DFA(decisionState, i);
}
// get atn simulator that knows how to do predictions
@ -160,6 +138,19 @@ public class ParserInterpreter extends Parser {
sharedContextCache));
}
/** A factory-like copy constructor that creates a new parser interpreter by reusing
* the fields of a previous interpreter.
*
* @since 4.5.1
*
* @param old The interpreter to copy
*/
public ParserInterpreter copyFrom(ParserInterpreter old) {
return new ParserInterpreter(old.grammarFileName, old.vocabulary,
Arrays.asList(old.ruleNames),
old.atn, old.getTokenStream());
}
@Override
public ATN getATN() {
return atn;
@ -190,8 +181,8 @@ public class ParserInterpreter extends Parser {
public ParserRuleContext parse(int startRuleIndex) {
RuleStartState startRuleStartState = atn.ruleToStartState[startRuleIndex];
InterpreterRuleContext rootContext = new InterpreterRuleContext(null, ATNState.INVALID_STATE_NUMBER, startRuleIndex);
if (startRuleStartState.isPrecedenceRule) {
InterpreterRuleContext rootContext = createInterpreterRuleContext(null, ATNState.INVALID_STATE_NUMBER, startRuleIndex);
if (startRuleStartState.isLeftRecursiveRule) {
enterRecursionRule(rootContext, startRuleStartState.stateNumber, startRuleIndex, 0);
}
else {
@ -204,7 +195,7 @@ public class ParserInterpreter extends Parser {
case ATNState.RULE_STOP :
// pop; return from rule
if ( _ctx.isEmpty() ) {
if (startRuleStartState.isPrecedenceRule) {
if (startRuleStartState.isLeftRecursiveRule) {
ParserRuleContext result = _ctx;
Pair<ParserRuleContext, Integer> parentContext = _parentContextStack.pop();
unrollRecursionContexts(parentContext.a);
@ -247,31 +238,28 @@ public class ParserInterpreter extends Parser {
}
protected void visitState(ATNState p) {
int altNum;
if (p.getNumberOfTransitions() > 1) {
getErrorHandler().sync(this);
int decision = ((DecisionState) p).decision;
if ( decision == overrideDecision && _input.index() == overrideDecisionInputIndex ) {
altNum = overrideDecisionAlt;
}
else {
altNum = getInterpreter().adaptivePredict(_input, decision, _ctx);
}
}
else {
altNum = 1;
// System.out.println("visitState "+p.stateNumber);
int edge = 1;
if ( p instanceof DecisionState ) {
edge = visitDecisionsState((DecisionState) p);
}
Transition transition = p.transition(altNum - 1);
Transition transition = p.transition(edge - 1);
switch (transition.getSerializationType()) {
case Transition.EPSILON:
if ( statesNeedingLeftRecursionContext.get(p.stateNumber) &&
if ( p.getStateType()==ATNState.STAR_LOOP_ENTRY &&
((StarLoopEntryState)p).isPrecedenceDecision &&
!(transition.target instanceof LoopEndState))
{
// We are at the start of a left recursive rule's (...)* loop
// but it's not the exit branch of loop.
InterpreterRuleContext ctx = new InterpreterRuleContext(_parentContextStack.peek().a, _parentContextStack.peek().b, _ctx.getRuleIndex());
pushNewRecursionContext(ctx, atn.ruleToStartState[p.ruleIndex].stateNumber, _ctx.getRuleIndex());
// and we're not taking the exit branch of loop.
InterpreterRuleContext localctx =
createInterpreterRuleContext(_parentContextStack.peek().a,
_parentContextStack.peek().b,
_ctx.getRuleIndex());
pushNewRecursionContext(localctx,
atn.ruleToStartState[p.ruleIndex].stateNumber,
_ctx.getRuleIndex());
}
break;
@ -295,12 +283,12 @@ public class ParserInterpreter extends Parser {
case Transition.RULE:
RuleStartState ruleStartState = (RuleStartState)transition.target;
int ruleIndex = ruleStartState.ruleIndex;
InterpreterRuleContext ctx = new InterpreterRuleContext(_ctx, p.stateNumber, ruleIndex);
if (ruleStartState.isPrecedenceRule) {
enterRecursionRule(ctx, ruleStartState.stateNumber, ruleIndex, ((RuleTransition)transition).precedence);
InterpreterRuleContext newctx = createInterpreterRuleContext(_ctx, p.stateNumber, ruleIndex);
if (ruleStartState.isLeftRecursiveRule) {
enterRecursionRule(newctx, ruleStartState.stateNumber, ruleIndex, ((RuleTransition)transition).precedence);
}
else {
enterRule(ctx, transition.target.stateNumber, ruleIndex);
enterRule(newctx, transition.target.stateNumber, ruleIndex);
}
break;
@ -330,9 +318,35 @@ public class ParserInterpreter extends Parser {
setState(transition.target.stateNumber);
}
protected int visitDecisionsState(DecisionState p) {
int edge = 1;
if ( p.getNumberOfTransitions()>1 ) {
int predictedAlt;
getErrorHandler().sync(this);
int decision = p.decision;
if (decision == overrideDecision && _input.index() == overrideDecisionInputIndex) {
predictedAlt = overrideDecisionAlt;
}
else {
predictedAlt = getInterpreter().adaptivePredict(_input, decision, _ctx);
}
edge = predictedAlt;
}
return edge;
}
/** Provide simple "factory" for InterpreterRuleContext's. */
protected InterpreterRuleContext createInterpreterRuleContext(
ParserRuleContext parent,
int invokingStateNumber,
int ruleIndex)
{
return new InterpreterRuleContext(parent, invokingStateNumber, ruleIndex);
}
protected void visitRuleStopState(ATNState p) {
RuleStartState ruleStartState = atn.ruleToStartState[p.ruleIndex];
if (ruleStartState.isPrecedenceRule) {
if (ruleStartState.isLeftRecursiveRule) {
Pair<ParserRuleContext, Integer> parentContext = _parentContextStack.pop();
unrollRecursionContexts(parentContext.a);
setState(parentContext.b);

15
runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java
View File

@ -210,7 +210,7 @@ public class ATNDeserializer {
int numPrecedenceStates = toInt(data[p++]);
for (int i = 0; i < numPrecedenceStates; i++) {
int stateNumber = toInt(data[p++]);
((RuleStartState)atn.states.get(stateNumber)).isPrecedenceRule = true;
((RuleStartState)atn.states.get(stateNumber)).isLeftRecursiveRule = true;
}
}
@ -239,9 +239,6 @@ public class ATNDeserializer {
// this piece of unused metadata was serialized prior to the
// addition of LexerAction
int actionIndexIgnored = toInt(data[p++]);
if (actionIndexIgnored == 0xFFFF) {
actionIndexIgnored = -1;
}
}
}
}
@ -319,7 +316,7 @@ public class ATNDeserializer {
RuleTransition ruleTransition = (RuleTransition)t;
int outermostPrecedenceReturn = -1;
if (atn.ruleToStartState[ruleTransition.target.ruleIndex].isPrecedenceRule) {
if (atn.ruleToStartState[ruleTransition.target.ruleIndex].isLeftRecursiveRule) {
if (ruleTransition.precedence == 0) {
outermostPrecedenceReturn = ruleTransition.target.ruleIndex;
}
@ -451,7 +448,7 @@ public class ATNDeserializer {
ATNState endState;
Transition excludeTransition = null;
if (atn.ruleToStartState[i].isPrecedenceRule) {
if (atn.ruleToStartState[i].isLeftRecursiveRule) {
// wrap from the beginning of the rule to the StarLoopEntryState
endState = null;
for (ATNState state : atn.states) {
@ -524,7 +521,7 @@ public class ATNDeserializer {
/**
* Analyze the {@link StarLoopEntryState} states in the specified ATN to set
* the {@link StarLoopEntryState#precedenceRuleDecision} field to the
* the {@link StarLoopEntryState#isPrecedenceDecision} field to the
* correct value.
*
* @param atn The ATN.
@ -539,11 +536,11 @@ public class ATNDeserializer {
* decision for the closure block that determines whether a
* precedence rule should continue or complete.
*/
if (atn.ruleToStartState[state.ruleIndex].isPrecedenceRule) {
if (atn.ruleToStartState[state.ruleIndex].isLeftRecursiveRule) {
ATNState maybeLoopEndState = state.transition(state.getNumberOfTransitions() - 1).target;
if (maybeLoopEndState instanceof LoopEndState) {
if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) {
((StarLoopEntryState)state).precedenceRuleDecision = true;
((StarLoopEntryState)state).isPrecedenceDecision = true;
}
}
}

2
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java
View File

@ -108,7 +108,7 @@ public class ATNSerializer {
nonGreedyStates.add(s.stateNumber);
}
if (s instanceof RuleStartState && ((RuleStartState)s).isPrecedenceRule) {
if (s instanceof RuleStartState && ((RuleStartState)s).isLeftRecursiveRule) {
precedenceStates.add(s.stateNumber);
}

2
runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java
View File

@ -32,7 +32,7 @@ package org.antlr.v4.runtime.atn;
public final class RuleStartState extends ATNState {
public RuleStopState stopState;
public boolean isPrecedenceRule; //Synonymous with rule being left recursive; consider renaming.
public boolean isLeftRecursiveRule;
@Override
public int getStateType() {

2
runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java
View File

@ -46,7 +46,7 @@ public final class StarLoopEntryState extends DecisionState {
*
* @see DFA#isPrecedenceDfa()
*/
public boolean precedenceRuleDecision;
public boolean isPrecedenceDecision;
@Override
public int getStateType() {

2
runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java
View File

@ -76,7 +76,7 @@ public class DFA {
boolean precedenceDfa = false;
if (atnStartState instanceof StarLoopEntryState) {
if (((StarLoopEntryState)atnStartState).precedenceRuleDecision) {
if (((StarLoopEntryState)atnStartState).isPrecedenceDecision) {
precedenceDfa = true;
DFAState precedenceState = new DFAState(new ATNConfigSet());
precedenceState.edges = new DFAState[0];

28
runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java
View File

@ -35,6 +35,7 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.runtime.tree.gui.TreePostScriptGenerator;
import org.antlr.v4.runtime.tree.gui.TreeTextProvider;
import java.io.BufferedWriter;
import java.io.FileWriter;
@ -100,26 +101,39 @@ public class Trees {
return toStringTree(t, ruleNamesList);
}
/** Print out a whole tree in LISP form. {@link #getNodeText} is used on the
* node payloads to get the text for the nodes. Detect
* parse trees and extract data appropriately.
/** Print out a whole tree in LISP form. Arg nodeTextProvider is used on the
* node payloads to get the text for the nodes.
*
* @since 4.5.1
*/
public static String toStringTree(Tree t, List<String> ruleNames) {
String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false);
public static String toStringTree(Tree t, TreeTextProvider nodeTextProvider) {
String s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
if ( t.getChildCount()==0 ) return s;
StringBuilder buf = new StringBuilder();
buf.append("(");
s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false);
s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
buf.append(s);
buf.append(' ');
for (int i = 0; i<t.getChildCount(); i++) {
if ( i>0 ) buf.append(' ');
buf.append(toStringTree(t.getChild(i), ruleNames));
buf.append(toStringTree(t.getChild(i), nodeTextProvider));
}
buf.append(")");
return buf.toString();
}
/** Print out a whole tree in LISP form. {@link #getNodeText} is used on the
* node payloads to get the text for the nodes.
*/
public static String toStringTree(final Tree t, final List<String> ruleNames) {
return toStringTree(t, new TreeTextProvider() {
@Override
public String getText(Tree node) {
return getNodeText(node, ruleNames);
}
});
}
public static String getNodeText(Tree t, Parser recog) {
String[] ruleNames = recog != null ? recog.getRuleNames() : null;
List<String> ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null;

2
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -659,7 +659,7 @@ public class ParserATNFactory implements ATNFactory {
RuleStartState start = newState(RuleStartState.class, r.ast);
RuleStopState stop = newState(RuleStopState.class, r.ast);
start.stopState = stop;
start.isPrecedenceRule = r instanceof LeftRecursiveRule;
start.isLeftRecursiveRule = r instanceof LeftRecursiveRule;
start.setRuleIndex(r.index);
stop.setRuleIndex(r.index);
atn.ruleToStartState[r.index] = start;

2
tool/src/org/antlr/v4/tool/DOTGenerator.java
View File

@ -246,7 +246,7 @@ public class DOTGenerator {
edgeST = stlib.getInstanceOf("edge");
String label = "<" + ruleNames[rr.ruleIndex];
if (((RuleStartState)rr.target).isPrecedenceRule) {
if (((RuleStartState)rr.target).isLeftRecursiveRule) {
label += "[" + rr.precedence + "]";
}
label += ">";

47
tool/src/org/antlr/v4/tool/LeftRecursiveRule.java
View File

@ -89,6 +89,53 @@ public class LeftRecursiveRule extends Rule {
return alts;
}
/** Return an array that maps predicted alt from primary decision
* to original alt of rule. For following rule, return [0, 2, 4]
*
e : e '*' e
| INT
| e '+' e
| ID
;
* That maps predicted alt 1 to original alt 2 and predicted 2 to alt 4.
*
* @since 4.5.1
*/
public int[] getPrimaryAlts() {
if ( recPrimaryAlts.size()==0 ) return null;
int[] alts = new int[recPrimaryAlts.size()+1];
for (int i = 0; i < recPrimaryAlts.size(); i++) { // recPrimaryAlts is a List not Map like recOpAlts
LeftRecursiveRuleAltInfo altInfo = recPrimaryAlts.get(i);
alts[i+1] = altInfo.altNum;
}
return alts;
}
/** Return an array that maps predicted alt from recursive op decision
* to original alt of rule. For following rule, return [0, 1, 3]
*
e : e '*' e
| INT
| e '+' e
| ID
;
* That maps predicted alt 1 to original alt 1 and predicted 2 to alt 3.
*
* @since 4.5.1
*/
public int[] getRecursiveOpAlts() {
if ( recOpAlts.size()==0 ) return null;
int[] alts = new int[recOpAlts.size()+1];
int alt = 1;
for (LeftRecursiveRuleAltInfo altInfo : recOpAlts.values()) {
alts[alt] = altInfo.altNum;
alt++; // recOpAlts has alts possibly with gaps
}
return alts;
}
/** Get -> labels from those alts we deleted for left-recursive rules. */
@Override
public Map<String, List<Pair<Integer, AltAST>>> getAltLabels() {

497
tool/test/org/antlr/v4/test/tool/TestATNParserPrediction.java
View File

@ -43,153 +43,157 @@ import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import java.util.Arrays;
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
public class TestATNParserPrediction extends BaseTest {
@Test public void testAorB() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"a : A{;} | B ;");
"parser grammar T;\n"+
"a : A{;} | B ;");
int decision = 0;
checkPredictedAlt(lg, g, decision, "a", 1);
checkPredictedAlt(lg, g, decision, "b", 2);
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"a",
"b",
"a"
"a",
"b",
"a"
};
String[] dfa = {
"s0-'a'->:s1=>1\n",
"s0-'a'->:s1=>1\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" + // don't change after it works
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" + // don't change after it works
"s0-'b'->:s2=>2\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testEmptyInput() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"a : A | ;");
"parser grammar T;\n"+
"a : A | ;");
int decision = 0;
checkPredictedAlt(lg, g, decision, "a", 1);
checkPredictedAlt(lg, g, decision, "", 2);
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"a",
"",
"a",
"",
};
String[] dfa = {
"s0-'a'->:s1=>1\n",
"s0-'a'->:s1=>1\n",
"s0-EOF->:s2=>2\n" +
"s0-'a'->:s1=>1\n",
"s0-EOF->:s2=>2\n" +
"s0-'a'->:s1=>1\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testPEGAchillesHeel() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"a : A | A B ;");
"parser grammar T;\n"+
"a : A | A B ;");
checkPredictedAlt(lg, g, 0, "a", 1);
checkPredictedAlt(lg, g, 0, "ab", 2);
checkPredictedAlt(lg, g, 0, "abc", 2);
String[] inputs = {
"a",
"ab",
"abc"
"a",
"ab",
"abc"
};
String[] dfa = {
"s0-'a'->s1\n" +
"s1-EOF->:s2=>1\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2=>1\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2=>1\n" +
"s1-'b'->:s3=>2\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2=>1\n" +
"s1-'b'->:s3=>2\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2=>1\n" +
"s1-'b'->:s3=>2\n"
"s0-'a'->s1\n" +
"s1-EOF->:s2=>1\n" +
"s1-'b'->:s3=>2\n"
};
checkDFAConstruction(lg, g, 0, inputs, dfa);
}
@Test public void testRuleRefxory() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"a : x | y ;\n" +
"x : A ;\n" +
"y : B ;\n");
"parser grammar T;\n"+
"a : x | y ;\n" +
"x : A ;\n" +
"y : B ;\n");
int decision = 0;
checkPredictedAlt(lg, g, decision, "a", 1);
checkPredictedAlt(lg, g, decision, "b", 2);
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"a",
"b",
"a"
"a",
"b",
"a"
};
String[] dfa = {
"s0-'a'->:s1=>1\n",
"s0-'a'->:s1=>1\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" + // don't change after it works
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" + // don't change after it works
"s0-'b'->:s2=>2\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testOptionalRuleChasesGlobalFollow() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A,B,C}\n" +
"a : x B ;\n" +
"b : x C ;\n" +
"x : A | ;\n");
"parser grammar T;\n"+
"tokens {A,B,C}\n" +
"a : x B ;\n" +
"b : x C ;\n" +
"x : A | ;\n");
int decision = 0;
checkPredictedAlt(lg, g, decision, "a", 1);
checkPredictedAlt(lg, g, decision, "b", 2);
@ -197,116 +201,116 @@ public class TestATNParserPrediction extends BaseTest {
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"a",
"b",
"c",
"c",
"a",
"b",
"c",
"c",
};
String[] dfa = {
"s0-'a'->:s1=>1\n",
"s0-'a'->:s1=>1\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n" +
"s0-'c'->:s3=>2\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n" +
"s0-'c'->:s3=>2\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n" +
"s0-'c'->:s3=>2\n",
"s0-'a'->:s1=>1\n" +
"s0-'b'->:s2=>2\n" +
"s0-'c'->:s3=>2\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testLL1Ambig() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"a : A | A | A B ;");
"parser grammar T;\n"+
"a : A | A | A B ;");
int decision = 0;
checkPredictedAlt(lg, g, decision, "a", 1);
checkPredictedAlt(lg, g, decision, "ab", 3);
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"a",
"ab",
"ab"
"a",
"ab",
"ab"
};
String[] dfa = {
"s0-'a'->s1\n" +
"s1-EOF->:s2^=>1\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2^=>1\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2^=>1\n" +
"s1-'b'->:s3=>3\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2^=>1\n" +
"s1-'b'->:s3=>3\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2^=>1\n" +
"s1-'b'->:s3=>3\n",
"s0-'a'->s1\n" +
"s1-EOF->:s2^=>1\n" +
"s1-'b'->:s3=>3\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testLL2Ambig() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"a : A B | A B | A B C ;");
"parser grammar T;\n"+
"a : A B | A B | A B C ;");
int decision = 0;
checkPredictedAlt(lg, g, decision, "ab", 1);
checkPredictedAlt(lg, g, decision, "abc", 3);
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"ab",
"abc",
"ab"
"ab",
"abc",
"ab"
};
String[] dfa = {
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3^=>1\n",
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3^=>1\n",
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3^=>1\n" +
"s2-'c'->:s4=>3\n",
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3^=>1\n" +
"s2-'c'->:s4=>3\n",
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3^=>1\n" +
"s2-'c'->:s4=>3\n",
"s0-'a'->s1\n" +
"s1-'b'->s2\n" +
"s2-EOF->:s3^=>1\n" +
"s2-'c'->:s4=>3\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testRecursiveLeftPrefix() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n" +
"LP : '(' ;\n" +
"RP : ')' ;\n" +
"INT : '0'..'9'+ ;\n"
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n" +
"LP : '(' ;\n" +
"RP : ')' ;\n" +
"INT : '0'..'9'+ ;\n"
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A,B,C,LP,RP,INT}\n" +
"a : e B | e C ;\n" +
"e : LP e RP\n" +
" | INT\n" +
" ;");
"parser grammar T;\n"+
"tokens {A,B,C,LP,RP,INT}\n" +
"a : e B | e C ;\n" +
"e : LP e RP\n" +
" | INT\n" +
" ;");
int decision = 0;
checkPredictedAlt(lg, g, decision, "34b", 1);
checkPredictedAlt(lg, g, decision, "34c", 2);
@ -315,57 +319,57 @@ public class TestATNParserPrediction extends BaseTest {
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"34b",
"34c",
"((34))b",
"((34))c"
"34b",
"34c",
"((34))b",
"((34))c"
};
String[] dfa = {
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n",
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n",
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n" +
"s1-'c'->:s3=>2\n",
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n" +
"s1-'c'->:s3=>2\n",
"s0-'('->s4\n" +
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n" +
"s1-'c'->:s3=>2\n" +
"s4-'('->s5\n" +
"s5-INT->s6\n" +
"s6-')'->s7\n" +
"s7-')'->s1\n",
"s0-'('->s4\n" +
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n" +
"s1-'c'->:s3=>2\n" +
"s4-'('->s5\n" +
"s5-INT->s6\n" +
"s6-')'->s7\n" +
"s7-')'->s1\n",
"s0-'('->s4\n" +
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n" +
"s1-'c'->:s3=>2\n" +
"s4-'('->s5\n" +
"s5-INT->s6\n" +
"s6-')'->s7\n" +
"s7-')'->s1\n",
"s0-'('->s4\n" +
"s0-INT->s1\n" +
"s1-'b'->:s2=>1\n" +
"s1-'c'->:s3=>2\n" +
"s4-'('->s5\n" +
"s5-INT->s6\n" +
"s6-')'->s7\n" +
"s7-')'->s1\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@Test public void testRecursiveLeftPrefixWithAorABIssue() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n" +
"LP : '(' ;\n" +
"RP : ')' ;\n" +
"INT : '0'..'9'+ ;\n"
"lexer grammar L;\n" +
"A : 'a' ;\n" +
"B : 'b' ;\n" +
"C : 'c' ;\n" +
"LP : '(' ;\n" +
"RP : ')' ;\n" +
"INT : '0'..'9'+ ;\n"
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A,B,C,LP,RP,INT}\n" +
"a : e A | e A B ;\n" +
"e : LP e RP\n" +
" | INT\n" +
" ;");
"parser grammar T;\n"+
"tokens {A,B,C,LP,RP,INT}\n" +
"a : e A | e A B ;\n" +
"e : LP e RP\n" +
" | INT\n" +
" ;");
int decision = 0;
checkPredictedAlt(lg, g, decision, "34a", 1);
checkPredictedAlt(lg, g, decision, "34ab", 2); // PEG would miss this one!
@ -374,40 +378,40 @@ public class TestATNParserPrediction extends BaseTest {
// After matching these inputs for decision, what is DFA after each prediction?
String[] inputs = {
"34a",
"34ab",
"((34))a",
"((34))ab",
"34a",
"34ab",
"((34))a",
"((34))ab",
};
String[] dfa = {
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n",
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n",
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n" +
"s2-'b'->:s4=>2\n",
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n" +
"s2-'b'->:s4=>2\n",
"s0-'('->s5\n" +
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n" +
"s2-'b'->:s4=>2\n" +
"s5-'('->s6\n" +
"s6-INT->s7\n" +
"s7-')'->s8\n" +
"s8-')'->s1\n",
"s0-'('->s5\n" +
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n" +
"s2-'b'->:s4=>2\n" +
"s5-'('->s6\n" +
"s6-INT->s7\n" +
"s7-')'->s8\n" +
"s8-')'->s1\n",
"s0-'('->s5\n" +
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n" +
"s2-'b'->:s4=>2\n" +
"s5-'('->s6\n" +
"s6-INT->s7\n" +
"s7-')'->s8\n" +
"s8-')'->s1\n",
"s0-'('->s5\n" +
"s0-INT->s1\n" +
"s1-'a'->s2\n" +
"s2-EOF->:s3=>1\n" +
"s2-'b'->:s4=>2\n" +
"s5-'('->s6\n" +
"s6-INT->s7\n" +
"s7-')'->s8\n" +
"s8-')'->s1\n",
};
checkDFAConstruction(lg, g, decision, inputs, dfa);
}
@ -418,15 +422,15 @@ public class TestATNParserPrediction extends BaseTest {
// be declared ambig since (12, 2, []) can take us to
// unambig state maybe. keep going.
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"ID : 'a'..'z' ;\n" + // one char
"SEMI : ';' ;\n"+
"INT : '0'..'9'+ ;\n"
"lexer grammar L;\n" +
"ID : 'a'..'z' ;\n" + // one char
"SEMI : ';' ;\n"+
"INT : '0'..'9'+ ;\n"
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {ID,SEMI,INT}\n" +
"a : (ID | ID ID?) SEMI ;");
"parser grammar T;\n"+
"tokens {ID,SEMI,INT}\n" +
"a : (ID | ID ID?) SEMI ;");
int decision = 1;
checkPredictedAlt(lg, g, decision, "a;", 1);
checkPredictedAlt(lg, g, decision, "ab;", 2);
@ -435,30 +439,85 @@ public class TestATNParserPrediction extends BaseTest {
@Test public void testContinuePrediction2() throws Exception {
// ID is ambig for first two alts, but ID SEMI lets us move forward with alt 3
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n" +
"ID : 'a'..'z' ;\n" + // one char
"SEMI : ';' ;\n"+
"INT : '0'..'9'+ ;\n"
"lexer grammar L;\n" +
"ID : 'a'..'z' ;\n" + // one char
"SEMI : ';' ;\n"+
"INT : '0'..'9'+ ;\n"
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {ID,SEMI,INT}\n" +
"a : ID | ID | ID SEMI ;\n");
"parser grammar T;\n"+
"tokens {ID,SEMI,INT}\n" +
"a : ID | ID | ID SEMI ;\n");
int decision = 0;
checkPredictedAlt(lg, g, decision, "a", 1);
checkPredictedAlt(lg, g, decision, "a;", 3);
}
@Test public void testAltsForLRRuleComputation() throws Exception {
Grammar g = new Grammar(
"grammar T;\n" +
"e : e '*' e\n" +
" | INT\n" +
" | e '+' e\n" +
" | ID\n" +
" ;\n" +
"ID : [a-z]+ ;\n" +
"INT : [0-9]+ ;\n" +
"WS : [ \\r\\t\\n]+ ;");
Rule e = g.getRule("e");
assertTrue(e instanceof LeftRecursiveRule);
LeftRecursiveRule lr = (LeftRecursiveRule)e;
assertEquals("[0, 2, 4]", Arrays.toString(lr.getPrimaryAlts()));
assertEquals("[0, 1, 3]", Arrays.toString(lr.getRecursiveOpAlts()));
}
@Test public void testAltsForLRRuleComputation2() throws Exception {
Grammar g = new Grammar(
"grammar T;\n" +
"e : INT\n" +
" | e '*' e\n" +
" | ID\n" +
" ;\n" +
"ID : [a-z]+ ;\n" +
"INT : [0-9]+ ;\n" +
"WS : [ \\r\\t\\n]+ ;");
Rule e = g.getRule("e");
assertTrue(e instanceof LeftRecursiveRule);
LeftRecursiveRule lr = (LeftRecursiveRule)e;
assertEquals("[0, 1, 3]", Arrays.toString(lr.getPrimaryAlts()));
assertEquals("[0, 2]", Arrays.toString(lr.getRecursiveOpAlts()));
}
@Test public void testAltsForLRRuleComputation3() throws Exception {
Grammar g = new Grammar(
"grammar T;\n" +
"random : 'blort';\n" + // should have no effect
"e : '--' e\n" +
" | e '*' e\n" +
" | e '+' e\n" +
" | e '--'\n" +
" | ID\n" +
" ;\n" +
"ID : [a-z]+ ;\n" +
"INT : [0-9]+ ;\n" +
"WS : [ \\r\\t\\n]+ ;");
Rule e = g.getRule("e");
assertTrue(e instanceof LeftRecursiveRule);
LeftRecursiveRule lr = (LeftRecursiveRule)e;
assertEquals("[0, 1, 5]", Arrays.toString(lr.getPrimaryAlts()));
assertEquals("[0, 2, 3, 4]", Arrays.toString(lr.getRecursiveOpAlts()));
}
/** first check that the ATN predicts right alt.
* Then check adaptive prediction.
*/
public void checkPredictedAlt(LexerGrammar lg, Grammar g, int decision,
String inputString, int expectedAlt)
String inputString, int expectedAlt)
{
Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg, true);
LexerATNSimulator lexInterp =
new LexerATNSimulator(lexatn,new DFA[] { new DFA(lexatn.modeToStartState.get(Lexer.DEFAULT_MODE)) },new PredictionContextCache());
new LexerATNSimulator(lexatn,new DFA[] { new DFA(lexatn.modeToStartState.get(Lexer.DEFAULT_MODE)) },new PredictionContextCache());
IntegerList types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);
@ -501,12 +560,12 @@ public class TestATNParserPrediction extends BaseTest {
}
public void checkDFAConstruction(LexerGrammar lg, Grammar g, int decision,
String[] inputString, String[] dfaString)
String[] inputString, String[] dfaString)
{
// Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg, true);
LexerATNSimulator lexInterp =
new LexerATNSimulator(lexatn,new DFA[] { new DFA(lexatn.getDecisionState(Lexer.DEFAULT_MODE)) }, new PredictionContextCache());
new LexerATNSimulator(lexatn,new DFA[] { new DFA(lexatn.getDecisionState(Lexer.DEFAULT_MODE)) }, new PredictionContextCache());
semanticProcess(lg);
g.importVocab(lg);

1
tool/tool.iml
View File

@ -43,6 +43,7 @@
<orderEntry type="library" name="maven-plugin-annotations-3.4" level="application" />
<orderEntry type="library" name="plexus-compiler-api-1.5.3" level="application" />
<orderEntry type="library" name="plexus-build-api-0.0.7" level="application" />
<orderEntry type="library" name="Python 2.7.8 (/usr/local/Cellar/python/2.7.8_1/Frameworks/Python.framework/Versions/2.7/bin/python2.7) interpreter library" level="application" />
</component>
<component name="PluginUploader.PluginUploadConfigurable">
<option name="ARCHIVE_PATH" />