forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

pull from master

This commit is contained in:
Terence Parr 2012-03-27 16:22:01 -07:00
parent 2b5896e83b 9c1e58db7c
commit a363e70582
23 changed files with 245 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -45,18 +45,18 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
public class TraceListener implements ParseListener<Token> {
@Override
public void enterNonLRRule(ParserRuleContext<Token> ctx) {
System.out.println("enter " + getRuleNames()[ctx.ruleIndex] + ", LT(1)=" + _input.LT(1).getText());
System.out.println("enter " + getRuleNames()[ctx.getRuleIndex()] + ", LT(1)=" + _input.LT(1).getText());
}
@Override
public void exitEveryRule(ParserRuleContext<Token> ctx) {
System.out.println("exit "+getRuleNames()[ctx.ruleIndex]+", LT(1)="+_input.LT(1).getText());
System.out.println("exit "+getRuleNames()[ctx.getRuleIndex()]+", LT(1)="+_input.LT(1).getText());
}
@Override
public void visitTerminal(ParserRuleContext<Token> parent, Token token) {
System.out.println("consume "+token+" rule "+
getRuleNames()[parent.ruleIndex]+
getRuleNames()[parent.getRuleIndex()]+
" alt="+parent.altNum);
}
}
@ -380,12 +380,12 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
public void enterRule(ParserRuleContext<Token> localctx, int ruleIndex) {
_ctx = localctx;
_ctx.start = _input.LT(1);
_ctx.ruleIndex = ruleIndex;
if (_buildParseTrees) addContextToParseTree();
if ( _parseListeners != null) triggerEnterRuleEvent();
}
public void exitRule() {
_ctx.stop = _input.LT(-1);
// trigger event on _ctx, before it reverts to parent
if ( _parseListeners != null) triggerExitRuleEvent();
_ctx = (ParserRuleContext<Token>)_ctx.parent;
@ -407,10 +407,10 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
public void pushNewRecursionContext(ParserRuleContext<Token> localctx, int ruleIndex) {
_ctx = localctx;
_ctx.start = _input.LT(1);
_ctx.ruleIndex = ruleIndex;
}
public void unrollRecursionContexts(ParserRuleContext<Token> _parentctx) {
_ctx.stop = _input.LT(-1);
ParserRuleContext<Token> retctx = _ctx; // save current ctx (return value)
// unroll so _ctx is as it was before call to recursive method

17
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
View File

@ -101,12 +101,15 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
public Symbol start, stop;
/** Set during parsing to identify which rule parser is in. */
public int ruleIndex;
/** Set during parsing to identify which alt of rule parser is in. */
public int altNum;
/**
* The exception which forced this rule to return. If the rule successfully
* completed, this is {@code null}.
*/
public RecognitionException exception;
public ParserRuleContext() { }
/** COPY a ctx (I'm deliberately not using copy constructor) */
@ -118,7 +121,6 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
this.start = ctx.start;
this.stop = ctx.stop;
this.ruleIndex = ctx.ruleIndex;
}
public ParserRuleContext(@Nullable ParserRuleContext<Symbol> parent, int invokingStateNumber, int stateNumber) {
@ -287,12 +289,9 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
@Override
public int getChildCount() { return children!=null ? children.size() : 0; }
@Override
public int getRuleIndex() { return ruleIndex; }
@Override
public Interval getSourceInterval() {
if ( start==null || stop==null ) return Interval.EMPTY;
if ( start==null || stop==null ) return Interval.INVALID;
return Interval.of(start.getTokenIndex(), stop.getTokenIndex());
}
@ -303,7 +302,7 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
*/
public String getText(TokenStream tokens) {
Interval range = getSourceInterval();
return range==Interval.EMPTY ? null : tokens.toString(range.a, range.b);
return range==Interval.INVALID ? null : tokens.toString(range.a, range.b);
}
public Symbol getStart() { return start; }

2
runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
View File

@ -214,7 +214,7 @@ public class RuleContext implements ParseTree.RuleNode {
@Override
public Interval getSourceInterval() {
return Interval.EMPTY;
return Interval.INVALID;
}
@Override

25
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -60,6 +60,10 @@ public abstract class ATNSimulator {
int p = 0;
atn.grammarType = toInt(data[p++]);
atn.maxTokenType = toInt(data[p++]);
//
// STATES
//
int nstates = toInt(data[p++]);
for (int i=1; i<=nstates; i++) {
int stype = toInt(data[p++]);
@ -75,6 +79,10 @@ public abstract class ATNSimulator {
}
atn.addState(s);
}
//
// RULES
//
int nrules = toInt(data[p++]);
if ( atn.grammarType == ATN.LEXER ) {
atn.ruleToTokenType = new int[nrules];
@ -92,11 +100,19 @@ public abstract class ATNSimulator {
atn.ruleToActionIndex[i] = actionIndex;
}
}
//
// MODES
//
int nmodes = toInt(data[p++]);
for (int i=0; i<nmodes; i++) {
int s = toInt(data[p++]);
atn.modeToStartState.add((TokensStartState)atn.states.get(s));
}
//
// SETS
//
int nsets = toInt(data[p++]);
for (int i=1; i<=nsets; i++) {
int nintervals = toInt(data[p]);
@ -108,6 +124,10 @@ public abstract class ATNSimulator {
p += 2;
}
}
//
// EDGES
//
int nedges = toInt(data[p++]);
for (int i=1; i<=nedges; i++) {
int src = toInt(data[p]);
@ -125,6 +145,10 @@ public abstract class ATNSimulator {
srcState.addTransition(trans);
p += 6;
}
//
// DECISIONS
//
int ndecisions = toInt(data[p++]);
for (int i=1; i<=ndecisions; i++) {
int s = toInt(data[p++]);
@ -134,6 +158,7 @@ public abstract class ATNSimulator {
decState.decision = i-1;
decState.isGreedy = isGreedy==1;
}
return atn;
}

3
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -243,7 +243,8 @@ public class LexerATNSimulator extends ATNSimulator {
t = input.LA(1);
}
return failOrAccept(prevAccept, input, prevAccept.state.configset, t);
ATNConfigSet reach = prevAccept.state != null ? prevAccept.state.configset : null;
return failOrAccept(prevAccept, input, reach, t);
}
protected int execATN(@NotNull CharStream input, @NotNull ATNConfigSet s0, @Nullable DFAState ds0) {

8
runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java
View File

@ -91,7 +91,13 @@ public abstract class Transition {
@NotNull
public ATNState target;
protected Transition(@NotNull ATNState target) { this.target = target; }
protected Transition(@NotNull ATNState target) {
if (target == null) {
throw new NullPointerException("target cannot be null.");
}
this.target = target;
}
public int getSerializationType() { return 0; }

1
runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java
View File

@ -33,7 +33,6 @@ public class Interval {
public static final int INTERVAL_POOL_MAX_VALUE = 1000;
public static final Interval INVALID = new Interval(-1,-2);
public static final Interval EMPTY = new Interval(0,-1); // len 0
static Interval[] cache = new Interval[INTERVAL_POOL_MAX_VALUE+1];

2
runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java
View File

@ -41,7 +41,7 @@ public interface SyntaxTree extends Tree {
* node is a leaf, then the interval represents a single token.
*
* If source interval is unknown, this does not return null.
* It returns an interval of length 0: Interval.EMPTY.
* It returns Interval.INVALID.
*/
Interval getSourceInterval();
}

9
tool/playground/T.g
View File

@ -1,3 +1,8 @@
grammar T;
s : 'a' 'b' ;
WS : [ \n]+ -> skip ;
s : e ';' ;
e : e '*' e
| ID
| INT
;
INT : '0'..'9'+;
WS : (' '|'\n') {skip();} ;

2
tool/resources/org/antlr/v4/tool/templates/LeftRecursiveRules.stg
View File

@ -37,7 +37,7 @@ recRule(ruleName, precArgDef, argName, primaryAlts, opAlts, setResultAction,
userRetvals, leftRecursiveRuleRefLabels) ::=
<<
<ruleName>[<precArgDef>]<if(userRetvals)> returns [<userRetvals>]<endif>
: ( <primaryAlts:{alt | <alt.altText> }; separator="\n | ">
: ( {} <primaryAlts:{alt | <alt.altText> }; separator="\n | ">
)
( <opAlts; separator="\n | ">
)*

9
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -117,7 +117,7 @@ Parser(parser, funcs, atn, sempredFuncs, superclass) ::= <<
Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public class <parser.name> extends <superclass> {
public <if(parser.abstractRecognizer)>abstract <endif>class <parser.name> extends <superclass> {
<if(parser.tokens)>
public static final int
<parser.tokens:{k | <k>=<parser.tokens.(k)>}; separator=", ", wrap, anchor>;
@ -229,11 +229,11 @@ RuleFunction(currentRule,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAc
int _alt;
<endif>
<code>
_localctx.stop = _input.LT(-1);
<postamble; separator="\n">
<namedActions.after>
}
catch (RecognitionException re) {
_localctx.exception = re;
_errHandler.reportError(this, re);
_errHandler.recover(this, re);
}
@ -265,11 +265,11 @@ LeftRecursiveRuleFunction(currentRule,code,locals,ruleCtx,altLabelCtxs,
int _alt;
<endif>
<code>
_localctx.stop = _input.LT(-1);
<postamble; separator="\n">
<namedActions.after>
}
catch (RecognitionException re) {
_localctx.exception = re;
_errHandler.reportError(this, re);
_errHandler.recover(this, re);
}
@ -608,6 +608,7 @@ public static class <struct.name> extends <superClass><if(interfaces)> implement
super(parent, state);
<struct.ctorAttrs:{a | this.<a.name> = <a.name>;}; separator="\n">
}
@Override public int getRuleIndex() { return RULE_<struct.derivedFromName>; }
<if(struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
public <struct.name>() { }
public void copyFrom(<struct.name> ctx) {
@ -713,7 +714,7 @@ import org.antlr.v4.runtime.misc.*;
Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public class <lexer.name> extends Lexer {
public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends Lexer {
public static final int
<lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>;
<rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n">

2
tool/src/org/antlr/v4/Tool.java
View File

@ -116,6 +116,7 @@ public class Tool {
public boolean gen_listener = true;
public boolean gen_parse_listener = false;
public boolean gen_visitor = false;
public boolean abstract_recognizer = false;
public static Option[] optionDefs = {
new Option("outputDirectory", "-o", OptionArgType.STRING, "specify output directory where all output is generated"),
@ -133,6 +134,7 @@ public class Tool {
new Option("gen_parse_listener", "-no-parse-listener", "don't generate parse listener (default)"),
new Option("gen_visitor", "-visitor", "generate parse tree visitor"),
new Option("gen_visitor", "-no-visitor", "don't generate parse tree visitor (default)"),
new Option("abstract_recognizer", "-abstract", "generate abstract recognizer classes"),
new Option("saveLexer", "-Xsave-lexer", "save temp lexer file created for combined grammars"),
new Option("launch_ST_inspector", "-XdbgST", "launch StringTemplate visualizer on generated code"),

5
tool/src/org/antlr/v4/automata/ATNSerializer.java
View File

@ -133,6 +133,11 @@ public class ATNSerializer {
if ( s==null ) continue; // might be optimized away
for (int i=0; i<s.getNumberOfTransitions(); i++) {
Transition t = s.transition(i);
if (atn.states.get(t.target.stateNumber) == null) {
throw new IllegalStateException("Cannot serialize a transition to a removed state.");
}
int src = s.stateNumber;
int trg = t.target.stateNumber;
int edgeType = Transition.serializationTypes.get(t.getClass());

2
tool/src/org/antlr/v4/codegen/model/Lexer.java
View File

@ -49,6 +49,7 @@ public class Lexer extends OutputModelObject {
public String[] tokenNames;
public Set<String> ruleNames;
public Collection<String> modes;
public boolean abstractRecognizer;
@ModelElement public SerializedATN atn;
@ModelElement public LinkedHashMap<Rule, RuleActionFunction> actionFuncs =
@ -89,6 +90,7 @@ public class Lexer extends OutputModelObject {
}
}
ruleNames = g.rules.keySet();
abstractRecognizer = g.isAbstract();
}
}

3
tool/src/org/antlr/v4/codegen/model/Parser.java
View File

@ -47,6 +47,7 @@ public class Parser extends OutputModelObject {
public Set<String> ruleNames;
public Collection<Rule> rules;
public ParserFile file;
public boolean abstractRecognizer;
@ModelElement public List<RuleFunction> funcs = new ArrayList<RuleFunction>();
@ModelElement public SerializedATN atn;
@ -89,5 +90,7 @@ public class Parser extends OutputModelObject {
} else {
superclass = new DefaultParserSuperClass();
}
abstractRecognizer = g.isAbstract();
}
}

2
tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java
View File

@ -74,6 +74,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
add("TokenLabelType");
add("superClass");
add("filter");
add("abstract");
}
};
@ -83,6 +84,7 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
add("tokenVocab");
add("TokenLabelType");
add("superClass");
add("abstract");
}
};

12
tool/src/org/antlr/v4/tool/Grammar.java
View File

@ -362,6 +362,11 @@ public class Grammar implements AttributeResolver {
return parent.getOutermostGrammar();
}
public boolean isAbstract() {
return Boolean.parseBoolean(getOptionString("abstract"))
|| (tool != null && tool.abstract_recognizer);
}
/** Get the name of the generated recognizer; may or may not be same
* as grammar name.
* Recognizer is TParser and TLexer from T if combined, else
@ -377,9 +382,16 @@ public class Grammar implements AttributeResolver {
buf.append(g.name);
buf.append('_');
}
if (isAbstract()) {
buf.append("Abstract");
}
buf.append(name);
qualifiedName = buf.toString();
}
else if (isAbstract()) {
qualifiedName = "Abstract" + name;
}
if ( isCombined() || (isLexer() && implicitLexer!=null) )
{
suffix = Grammar.getGrammarTypeToFileNameSuffix(getType());

84
tool/test/org/antlr/v4/test/TestFullContextParsing.java
View File

@ -55,7 +55,7 @@ public class TestFullContextParsing extends BaseTest {
"Decision 0:\n" +
"s0-ID->:s1=>1\n"; // not ctx sensitive
assertEquals(expecting, result);
assertEquals("line 1:0 reportAmbiguity d=0: ambigAlts={1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input='abc'\n",
assertEquals("line 1:0 reportAmbiguity d=0: ambigAlts={1..2}, input='abc'\n",
this.stderrDuringParse);
}
@ -77,8 +77,8 @@ public class TestFullContextParsing extends BaseTest {
"s0-INT->s1\n" +
"s1-ID->s2^\n";
assertEquals(expecting, result);
assertEquals("line 1:5 reportAttemptingFullContext d=1: [(28,1,[18 10]), (20,2,[10])], input='34abc'\n" +
"line 1:2 reportContextSensitivity d=1: [(20,1,[10])],uniqueAlt=1, input='34'\n",
assertEquals("line 1:5 reportAttemptingFullContext d=1, input='34abc'\n" +
"line 1:2 reportContextSensitivity d=1, input='34'\n",
this.stderrDuringParse);
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
@ -88,8 +88,8 @@ public class TestFullContextParsing extends BaseTest {
"s0-INT->s1\n" +
"s1-ID->s2^\n";
assertEquals(expecting, result);
assertEquals("line 1:5 reportAttemptingFullContext d=1: [(28,1,[22 14]), (24,2,[14])], input='34abc'\n" +
"line 1:5 reportContextSensitivity d=1: [(1,2,[])],uniqueAlt=2, input='34abc'\n",
assertEquals("line 1:5 reportAttemptingFullContext d=1, input='34abc'\n" +
"line 1:5 reportContextSensitivity d=1, input='34abc'\n",
this.stderrDuringParse);
}
@ -107,19 +107,14 @@ public class TestFullContextParsing extends BaseTest {
String result = execParser("T.g", grammar, "TParser", "TLexer", "s",
"$ 34 abc @ 34 abc", true);
String expecting =
"Decision 1:\n" +
"s0-EOF->:s3=>2\n" +
"s0-'@'->:s2=>1\n" +
"s0-'$'->:s1=>1\n" +
"\n" +
"Decision 2:\n" +
"s0-INT->s1\n" +
"s1-ID->s2^\n";
assertEquals(expecting, result);
assertEquals("line 1:5 reportAttemptingFullContext d=2: [(30,1,[20 10]), (22,2,[10])], input='34abc'\n" +
"line 1:2 reportContextSensitivity d=2: [(22,1,[10])],uniqueAlt=1, input='34'\n" +
"line 1:14 reportAttemptingFullContext d=2: [(30,1,[24 14]), (26,2,[14])], input='34abc'\n" +
"line 1:14 reportContextSensitivity d=2: [(8,2,[18]), (12,2,[18]), (1,2,[])],uniqueAlt=2, input='34abc'\n",
assertEquals("line 1:5 reportAttemptingFullContext d=2, input='34abc'\n" +
"line 1:2 reportContextSensitivity d=2, input='34'\n" +
"line 1:14 reportAttemptingFullContext d=2, input='34abc'\n" +
"line 1:14 reportContextSensitivity d=2, input='34abc'\n",
this.stderrDuringParse);
}
@ -139,10 +134,6 @@ public class TestFullContextParsing extends BaseTest {
String result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
String expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'}'->:s1=>2\n";
assertEquals(expecting, result);
@ -153,27 +144,20 @@ public class TestFullContextParsing extends BaseTest {
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1=>1\n" +
"s0-'else'->s1^\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:29 reportAmbiguity d=1: ambigAlts={1..2}:[(25,1,[]), (25,2,[],up=1)],conflictingAlts={1..2},dipsIntoOuterContext, input='else'\n",
assertEquals("line 1:29 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:38 reportAmbiguity d=1: ambigAlts={1..2}, input='elsefoo}'\n",
this.stderrDuringParse);
input = "{ if x then return else foo }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1=>1\n";
"s0-'else'->s1^\n";
assertEquals(expecting, result);
// Technically, this input sequence is not ambiguous because else
// uniquely predicts going into the optional subrule. else cannot
@ -181,21 +165,19 @@ public class TestFullContextParsing extends BaseTest {
// the start of a stat. But, we are using the theory that
// SLL(1)=LL(1) and so we are avoiding full context parsing
// by declaring all else clause parsing to be ambiguous.
assertEquals("line 1:19 reportAmbiguity d=1: ambigAlts={1..2}:[(25,1,[]), (25,2,[],up=1)],conflictingAlts={1..2},dipsIntoOuterContext, input='else'\n",
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n",
this.stderrDuringParse);
input = "{ if x then return else foo }";
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1=>1\n";
"s0-'else'->s1^\n";
assertEquals(expecting, result);
assertEquals("line 1:19 reportAmbiguity d=1: ambigAlts={1..2}:[(25,1,[]), (25,2,[],up=1)],conflictingAlts={1..2},dipsIntoOuterContext, input='else'\n",
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n",
this.stderrDuringParse);
input =
@ -204,15 +186,14 @@ public class TestFullContextParsing extends BaseTest {
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1=>1\n" +
"s0-'else'->s1^\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:19 reportAmbiguity d=1: ambigAlts={1..2}:[(25,1,[]), (25,2,[],up=1)],conflictingAlts={1..2},dipsIntoOuterContext, input='else'\n",
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n" +
"line 2:27 reportAttemptingFullContext d=1, input='else'\n" +
"line 2:36 reportAmbiguity d=1: ambigAlts={1..2}, input='elsefoo}'\n",
this.stderrDuringParse);
input =
@ -221,15 +202,14 @@ public class TestFullContextParsing extends BaseTest {
result = execParser("T.g", grammar, "TParser", "TLexer", "s",
input, true);
expecting =
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'else'->:s1=>1\n" +
"s0-'else'->s1^\n" +
"s0-'}'->:s2=>2\n";
assertEquals(expecting, result);
assertEquals("line 1:19 reportAmbiguity d=1: ambigAlts={1..2}:[(25,1,[]), (25,2,[],up=1)],conflictingAlts={1..2},dipsIntoOuterContext, input='else'\n",
assertEquals("line 1:19 reportAttemptingFullContext d=1, input='else'\n" +
"line 1:19 reportContextSensitivity d=1, input='else'\n" +
"line 2:27 reportAttemptingFullContext d=1, input='else'\n" +
"line 2:36 reportAmbiguity d=1: ambigAlts={1..2}, input='elsefoo}'\n",
this.stderrDuringParse);
}
@ -259,10 +239,10 @@ public class TestFullContextParsing extends BaseTest {
assertEquals("pass.\n", found);
String expecting =
"line 1:4 reportAttemptingFullContext d=1: [(35,1,[27 15 8]), (41,1,[27 15 8]), (49,1,[27 15 8]), (35,2,[27 21 8]), (41,2,[27 21 8]), (49,2,[27 21 8])], input='a(i)<-'\n" +
"line 1:7 reportContextSensitivity d=1: [(53,2,[])],uniqueAlt=2, input='a(i)<-x'\n" +
"line 1:3 reportAttemptingFullContext d=3: [(35,1,[27 21 8]), (41,2,[27 21 8]), (49,3,[27 21 8])], input='a(i)'\n" +
"line 1:7 reportAmbiguity d=3: ambigAlts={2..3}:[(53,2,[]), (53,3,[])],conflictingAlts={2..3}, input='a(i)<-x'\n";
"line 1:4 reportAttemptingFullContext d=1, input='a(i)<-'\n" +
"line 1:7 reportContextSensitivity d=1, input='a(i)<-x'\n" +
"line 1:3 reportAttemptingFullContext d=3, input='a(i)'\n" +
"line 1:7 reportAmbiguity d=3: ambigAlts={2..3}, input='a(i)<-x'\n";
assertEquals(expecting, this.stderrDuringParse);
}

53
tool/test/org/antlr/v4/test/TestLeftRecursion.java
View File

@ -276,6 +276,59 @@ public class TestLeftRecursion extends BaseTest {
runTests(grammar, tests, "s");
}
@Test
public void testAmbigLR() throws Exception {
String grammar =
"// START: g\n" +
"grammar Expr;\n" +
"// END: g\n" +
"\n" +
"// START:stat\n" +
"prog: stat ;\n" +
"\n" +
"stat: expr NEWLINE -> printExpr\n" +
" | ID '=' expr NEWLINE -> assign\n" +
" | NEWLINE -> blank\n" +
" ;\n" +
"// END:stat\n" +
"\n" +
"// START:expr\n" +
"expr: expr ('*'|'/') expr -> MulDiv\n" +
" | expr ('+'|'-') expr -> AddSub\n" +
" | INT -> int\n" +
" | ID -> id\n" +
" | '(' expr ')' -> parens\n" +
" ;\n" +
"// END:expr\n" +
"\n" +
"// show marginal cost of adding a clear/wipe command for memory\n" +
"\n" +
"// START:tokens\n" +
"MUL : '*' ; // assigns token name to '*' used above in grammar\n" +
"DIV : '/' ;\n" +
"ADD : '+' ;\n" +
"SUB : '-' ;\n" +
"ID : [a-zA-Z]+ ; // match identifiers\n" +
"INT : [0-9]+ ; // match integers\n" +
"NEWLINE:'\\r'? '\\n' ; // return newlines to parser (is end-statement signal)\n" +
"WS : [ \\t]+ -> skip ; // toss out whitespace\n" +
"// END:tokens\n";
String result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "1\n", true);
assertNull(stderrDuringParse);
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "a = 5\n", true);
assertNull(stderrDuringParse);
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "b = 6\n", true);
assertNull(stderrDuringParse);
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "a+b*2\n", true);
assertNull(stderrDuringParse);
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "(1+2)*3\n", true);
assertNull(stderrDuringParse);
}
public void runTests(String grammar, String[] tests, String startRule) {
rawGenerateAndBuildRecognizer("T.g", grammar, "TParser", "TLexer");
writeRecognizerAndCompile("TParser",

25
tool/test/org/antlr/v4/test/TestLexerErrors.java
View File

@ -142,4 +142,29 @@ public class TestLexerErrors extends BaseTest {
// TEST RECOVERY
/**
* This is a regression test for #45 "NullPointerException in LexerATNSimulator.execDFA".
* https://github.com/antlr/antlr4/issues/46
*/
@Test
public void testLexerExecDFA() throws Exception {
String grammar =
"grammar T;\n" +
"start : ID ':' expr;\n" +
"expr : primary expr? {} | expr '->' ID;\n" +
"primary : ID;\n" +
"ID : [a-z]+;\n" +
"\n";
String result = execLexer("T.g", grammar, "TLexer", "x : x", false);
String expecting =
"[@0,0:0='x',<5>,1:0]\n" +
"[@1,2:2=':',<4>,1:2]\n" +
"[@2,4:4='x',<5>,1:4]\n" +
"[@3,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, result);
assertEquals("line 1:1 token recognition error at: ' '\n" +
"line 1:3 token recognition error at: ' '\n",
this.stderrDuringParse);
}
}

105
tool/test/org/antlr/v4/test/TestNonGreedyLoops.java
View File

@ -180,23 +180,16 @@ public class TestNonGreedyLoops extends BaseTest {
"x", true);
assertEquals("alt 1\n" +
"Decision 0:\n" +
"s0-ID->:s1=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-ID->:s1=>2\n", found);
assertEquals("line 1:0 extraneous input 'x' expecting <EOF>\n", this.stderrDuringParse);
"s0-ID->:s1=>1\n", found);
assertNull(this.stderrDuringParse);
found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"34", true);
assertEquals("alt 1\n" +
"Decision 0:\n" +
"s0-INT->s1\n" +
"s1-EOF->:s2=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-INT->:s1=>2\n", found); // resolves INT EOF to alt 1 from s since ambig 'tween a and b
assertEquals("line 1:2 reportAmbiguity d=0: ambigAlts={1..2}:[(1,1,[]), (1,2,[])],conflictingAlts={1..2}, input='34'\n" +
"line 1:0 extraneous input '34' expecting <EOF>\n",
"s1-EOF->:s2=>1\n", found); // resolves INT EOF to alt 1 from s since ambig 'tween a and b
assertEquals("line 1:2 reportAmbiguity d=0: ambigAlts={1..2}, input='34'\n",
this.stderrDuringParse);
}
@ -360,10 +353,7 @@ public class TestNonGreedyLoops extends BaseTest {
"s2-INT->:s3=>1\n" +
"s2-ID->s4\n" +
"s4-';'->s5\n" +
"s5-EOF->:s6=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-ID->:s1=>3\n", found);
"s5-EOF->:s6=>2\n", found);
input =
"if ( 1 ) { x=3; { return 4; } } return 99; abc=def;";
found = execParser("T.g", grammar, "TParser", "TLexer", "s",
@ -376,19 +366,7 @@ public class TestNonGreedyLoops extends BaseTest {
"s3-'='->s4\n" +
"s4-ID->s5\n" +
"s5-';'->s6\n" +
"s6-EOF->:s7=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'{'->:s2=>4\n" +
"s0-'if'->:s1=>1\n" +
"s0-'return'->:s4=>2\n" +
"s0-ID->:s3=>3\n" +
"\n" +
"Decision 2:\n" +
"s0-'{'->:s2=>1\n" +
"s0-'return'->:s3=>1\n" +
"s0-'}'->:s4=>2\n" +
"s0-ID->:s1=>1\n", found);
"s6-EOF->:s7=>2\n", found);
input =
"x=1; a=3;"; // FAILS to match since it can't match last element
execParser("T.g", grammar, "TParser", "TLexer", "s",
@ -436,10 +414,7 @@ public class TestNonGreedyLoops extends BaseTest {
"s1-'='->s2\n" +
"s2-INT->:s3=>1\n" +
"s2-ID->s4\n" +
"s4-';'->:s5=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-ID->:s1=>3\n", found); // ignores x=1 that follows first a=b assignment
"s4-';'->:s5=>2\n", found); // ignores x=1 that follows first a=b assignment
input =
"if ( 1 ) { x=3; { return 4; } } return 99; abc=def;";
found = execParser("T.g", grammar, "TParser", "TLexer", "s",
@ -451,19 +426,7 @@ public class TestNonGreedyLoops extends BaseTest {
"s0-ID->s3\n" +
"s3-'='->s4\n" +
"s4-ID->s5\n" +
"s5-';'->:s6=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'{'->:s2=>4\n" +
"s0-'if'->:s1=>1\n" +
"s0-'return'->:s4=>2\n" +
"s0-ID->:s3=>3\n" +
"\n" +
"Decision 2:\n" +
"s0-'{'->:s2=>1\n" +
"s0-'return'->:s3=>1\n" +
"s0-'}'->:s4=>2\n" +
"s0-ID->:s1=>1\n", found);
"s5-';'->:s6=>2\n", found);
input =
"x=1; a=3;"; // FAILS to match since it can't match either stat
execParser("T.g", grammar, "TParser", "TLexer", "s",
@ -481,10 +444,7 @@ public class TestNonGreedyLoops extends BaseTest {
"s1-'='->s2\n" +
"s2-INT->:s3=>1\n" +
"s2-ID->s4\n" +
"s4-';'->:s5=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-ID->:s1=>3\n", found); // should not finish all input
"s4-';'->:s5=>2\n", found); // should not finish all input
}
@Test public void testHTMLTags() throws Exception {
@ -504,11 +464,6 @@ public class TestNonGreedyLoops extends BaseTest {
found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"<a>foo</a>", true);
assertEquals("<a>foo</a>\n" +
"Decision 0:\n" +
"s0-EOF->:s3=>2\n" +
"s0-'<'->:s1=>1\n" +
"s0-ID->:s2=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-'<'->s1\n" +
"s0-ID->:s5=>2\n" +
@ -527,20 +482,16 @@ public class TestNonGreedyLoops extends BaseTest {
"Decision 3:\n" +
"s0-'>'->:s2=>2\n" +
"s0-ID->:s1=>1\n", found);
assertEquals("line 1:6 reportAttemptingFullContext d=1: [(20,1,[14 6]), (16,2,[6])], input='<a>foo<'\n" +
"line 1:6 reportAmbiguity d=1: ambigAlts={1..2}:[(26,1,[32 32 32 32 14 6]), (33,1,[14 6]), (22,1,[14 6 10 10]), (26,1,[14 6 10 10]), (33,1,[14 6 10 10]), (20,1,[14 6 10 10 10]), (16,1,[6 10 10 10]), (1,1,[]), (22,2,[14 6 10 10 10 10]), (26,2,[14 6 10 10 10 10]), (33,2,[14 6 10 10 10 10]), (20,2,[14 6 10 10 10 10 10]), (16,2,[6 10 10 10 10 10]), (1,2,[])],conflictingAlts={1..2}, input='<a>foo<'\n" +
"line 1:10 reportAttemptingFullContext d=1: [(20,1,[14 6]), (16,2,[6])], input='</a>'\n" +
"line 1:10 reportAmbiguity d=1: ambigAlts={1..2}:[(35,1,[]), (35,2,[])],conflictingAlts={1..2}, input='</a>'\n" +
"line 1:7 reportAmbiguity d=2: ambigAlts={1..2}:[(26,1,[]), (33,1,[]), (26,2,[]), (33,2,[])],conflictingAlts={1..2}, input='/'\n",
assertEquals("line 1:6 reportAttemptingFullContext d=1, input='<a>foo<'\n" +
"line 1:6 reportAmbiguity d=1: ambigAlts={1..2}, input='<a>foo<'\n" +
"line 1:10 reportAttemptingFullContext d=1, input='</a>'\n" +
"line 1:10 reportAmbiguity d=1: ambigAlts={1..2}, input='</a>'\n" +
"line 1:7 reportAmbiguity d=2: ambigAlts={1..2}, input='/'\n",
this.stderrDuringParse);
found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"<a></a>", true);
assertEquals("<a></a>\n" +
"Decision 0:\n" +
"s0-EOF->:s2=>2\n" +
"s0-'<'->:s1=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-'<'->s1\n" +
"s1-'/'->s2\n" +
@ -560,10 +511,6 @@ public class TestNonGreedyLoops extends BaseTest {
found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"</b><a src=\"abc\", width=32>", true);
assertEquals("</b><asrc=\"abc\",width=32>\n" +
"Decision 0:\n" +
"s0-EOF->:s2=>2\n" +
"s0-'<'->:s1=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-'<'->s1\n" +
"s1-'/'->s2\n" +
@ -626,15 +573,6 @@ public class TestNonGreedyLoops extends BaseTest {
"s3-INT->s3\n" +
"s4-'='->s3\n" +
"\n" +
"Decision 1:\n" + // (tag|header)
"s0-'<'->:s1=>1\n" +
"\n" +
"Decision 2:\n" + // (...)*
"s0-EOF->:s3=>2\n" +
"s0-'<'->:s2=>1\n" +
"s0-','->:s1=>1\n" +
"s0-INT->:s1=>1\n" +
"\n" +
"Decision 3:\n" + // .+
"s0-'x'->:s1=>1\n" +
"s0-'>'->:s2=>2\n" +
@ -655,13 +593,6 @@ public class TestNonGreedyLoops extends BaseTest {
"s4-ID->s5\n" +
"s5-'>'->:s6=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'<'->:s1=>1\n" +
"\n" +
"Decision 2:\n" +
"s0-EOF->:s2=>2\n" +
"s0-'x'->:s1=>1\n" +
"\n" +
"Decision 3:\n" +
"s0-'>'->:s2=>2\n" +
"s0-ID->:s1=>1\n", found);
@ -682,14 +613,6 @@ public class TestNonGreedyLoops extends BaseTest {
"s4-'>'->:s7=>2\n" +
"s4-'<'->:s5=>2\n" +
"\n" +
"Decision 1:\n" +
"s0-'<'->:s1=>1\n" +
"\n" +
"Decision 2:\n" +
"s0-EOF->:s3=>2\n" +
"s0-'x'->:s1=>1\n" +
"s0-'>'->:s2=>1\n" +
"\n" +
"Decision 3:\n" +
"s0-'>'->:s1=>2\n" +
"s0-ID->:s2=>1\n", // doesn't match tag; null

24
tool/test/org/antlr/v4/test/TestParseErrors.java
View File

@ -28,6 +28,7 @@
package org.antlr.v4.test;
import org.antlr.v4.automata.ATNSerializer;
import org.junit.Test;
/** test runtime parse errors */
@ -253,4 +254,27 @@ public class TestParseErrors extends BaseTest {
assertEquals(expecting, result);
}
/**
* This is a regression test for #45 "NullPointerException in ATNConfig.hashCode".
* https://github.com/antlr/antlr4/issues/45
*
* The original cause of this issue was an error in the tool's ATN state optimization,
* which is now detected early in {@link ATNSerializer} by ensuring that all
* serialized transitions point to states which were not removed.
*/
@Test
public void testInvalidATNStateRemoval() throws Exception {
String grammar =
"grammar T;\n" +
"start : ID ':' expr;\n" +
"expr : primary expr? {} | expr '->' ID;\n" +
"primary : ID;\n" +
"ID : [a-z]+;\n" +
"\n";
String result = execParser("T.g", grammar, "TParser", "TLexer", "start", "x:x", true);
String expecting = "";
assertEquals(expecting, result);
assertNull(this.stderrDuringParse);
}
}

10
tool/test/org/antlr/v4/test/TestSemPredEvalParser.java
View File

@ -153,7 +153,10 @@ public class TestSemPredEvalParser extends BaseTest {
"alt 1\n" +
"alt 1\n";
assertEquals(expecting, found);
assertEquals("line 1:0 reportAmbiguity d=0: ambigAlts={1..2}:[(6,1,[],up=1), (1,1,[],up=1), (6,2,[],up=1), (1,2,[],up=1), (6,3,[],{1:0}?,up=1), (1,3,[],{1:0}?,up=1)],hasSemanticContext=true,conflictingAlts={1..3},dipsIntoOuterContext, input='x'\n",
assertEquals("line 1:0 reportAttemptingFullContext d=0, input='x'\n" +
"line 1:0 reportAmbiguity d=0: ambigAlts={1..2}, input='x'\n" +
"line 1:3 reportAttemptingFullContext d=0, input='y'\n" +
"line 1:3 reportAmbiguity d=0: ambigAlts={1..2}, input='y'\n",
this.stderrDuringParse);
}
@ -184,7 +187,10 @@ public class TestSemPredEvalParser extends BaseTest {
"alt 2\n" +
"alt 2\n";
assertEquals(expecting, found);
assertEquals("line 1:4 reportAmbiguity d=0: ambigAlts={2..3}:[(6,2,[],up=1), (10,2,[],up=1), (1,2,[],up=1), (6,3,[],up=1), (10,3,[],up=1), (1,3,[],up=1), (6,4,[],{1:0}?,up=1), (10,4,[],{1:0}?,up=1), (1,4,[],{1:0}?,up=1)],hasSemanticContext=true,conflictingAlts={2..4},dipsIntoOuterContext, input='x'\n",
assertEquals("line 1:4 reportAttemptingFullContext d=0, input='x'\n" +
"line 1:4 reportAmbiguity d=0: ambigAlts={2..3}, input='x'\n" +
"line 1:7 reportAttemptingFullContext d=0, input='y'\n" +
"line 1:7 reportAmbiguity d=0: ambigAlts={2..3}, input='y'\n",
this.stderrDuringParse);
}