jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

rm isGreedy from DecisionState, but allow ATN construction for lexer to be nongreedy. error if '.' in parser. rm unit tests for parser nongreedy

This commit is contained in:
Terence Parr 2012-09-29 12:33:00 -07:00
parent d3d5bebf9f
commit e78ecd418a
12 changed files with 107 additions and 776 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -250,11 +250,9 @@ public abstract class ATNSimulator {
int ndecisions = toInt(data[p++]);
for (int i=1; i<=ndecisions; i++) {
int s = toInt(data[p++]);
int isGreedy = toInt(data[p++]);
DecisionState decState = (DecisionState)atn.states.get(s);
atn.decisionToState.add(decState);
decState.decision = i-1;
decState.isGreedy = isGreedy==1;
}
verifyATN(atn);

2
runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java
View File

@ -31,6 +31,4 @@ package org.antlr.v4.runtime.atn;
public class DecisionState extends ATNState {
public int decision = -1;
public boolean isGreedy = true;
}

145
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -384,13 +384,12 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
", outerContext="+outerContext.toString(parser));
}
DecisionState decState = atn.getDecisionState(dfa.decision);
boolean greedy = decState.isGreedy;
boolean loopsSimulateTailRecursion = SLL_loopsSimulateTailRecursion;
boolean fullCtx = false;
ATNConfigSet s0_closure =
computeStartState(dfa.atnStartState,
ParserRuleContext.EMPTY,
greedy, loopsSimulateTailRecursion,
loopsSimulateTailRecursion,
fullCtx);
dfa.s0 = addDFAState(dfa, new DFAState(s0_closure));
@ -428,7 +427,6 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
DFAState s = s0;
DecisionState decState = atn.getDecisionState(dfa.decision);
boolean greedy = decState.isGreedy;
int t = input.LA(1);
loop:
@ -440,14 +438,13 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
boolean fullCtx = true;
ATNConfigSet s0_closure =
computeStartState(dfa.atnStartState, outerContext,
greedy, loopsSimulateTailRecursion,
loopsSimulateTailRecursion,
fullCtx);
retry_with_context_from_dfa++;
int alt = execATNWithFullContext(dfa, s, s0_closure,
input, startIndex,
outerContext,
ATN.INVALID_ALT_NUMBER,
greedy);
ATN.INVALID_ALT_NUMBER);
return alt;
}
if ( s.isAcceptState ) {
@ -598,13 +595,11 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
int t = input.LA(1);
DecisionState decState = atn.getDecisionState(dfa.decision);
boolean greedy = decState.isGreedy;
while (true) { // while more work
boolean loopsSimulateTailRecursion = SLL_loopsSimulateTailRecursion;
// System.out.println("REACH "+getLookaheadName(input));
ATNConfigSet reach = computeReachSet(previous, t,
greedy,
loopsSimulateTailRecursion,
false);
if ( reach==null ) {
@ -658,75 +653,44 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
// boolean cont = needMoreLookaheadLL(reach);
D.configs.conflictingAlts = getConflictingAlts(reach);
if ( D.configs.conflictingAlts!=null ) {
if ( greedy ) {
// CONFLICT, GREEDY (TYPICAL SITUATION)
if ( outerContext == ParserRuleContext.EMPTY || // in grammar start rule
!D.configs.dipsIntoOuterContext || // didn't fall out of rule
SLL ) // forcing SLL only
{
// SPECIAL CASE WHERE SLL KNOWS CONFLICT IS AMBIGUITY
if ( !D.configs.hasSemanticContext ) {
reportAmbiguity(dfa, D, startIndex, input.index(),
D.configs.conflictingAlts, D.configs);
}
D.isAcceptState = true;
D.prediction = D.configs.conflictingAlts.getMinElement();
if ( debug ) System.out.println("RESOLVED TO "+D.prediction+" for "+D);
predictedAlt = D.prediction;
// Falls through to check predicates below
}
else {
// SLL CONFLICT; RETRY WITH FULL LL CONTEXT
// (it's possible SLL with preds could resolve to single alt
// which would mean we could avoid full LL, but not worth
// code complexity.)
if ( debug ) System.out.println("RETRY with outerContext="+outerContext);
// don't look up context in cache now since we're just creating state D
loopsSimulateTailRecursion = true;
ATNConfigSet s0_closure =
computeStartState(dfa.atnStartState,
outerContext,
greedy,
loopsSimulateTailRecursion,
true);
predictedAlt = execATNWithFullContext(dfa, D, s0_closure,
input, startIndex,
outerContext,
D.configs.conflictingAlts.getMinElement(),
greedy);
// not accept state: isCtxSensitive
D.requiresFullContext = true; // always force DFA to ATN simulate
D.prediction = ATN.INVALID_ALT_NUMBER;
addDFAEdge(dfa, previousD, t, D);
return predictedAlt; // all done with preds, etc...
// CONFLICT, GREEDY (TYPICAL SITUATION)
if ( outerContext == ParserRuleContext.EMPTY || // in grammar start rule
!D.configs.dipsIntoOuterContext || // didn't fall out of rule
SLL ) // forcing SLL only
{
// SPECIAL CASE WHERE SLL KNOWS CONFLICT IS AMBIGUITY
if ( !D.configs.hasSemanticContext ) {
reportAmbiguity(dfa, D, startIndex, input.index(),
D.configs.conflictingAlts, D.configs);
}
D.isAcceptState = true;
D.prediction = D.configs.conflictingAlts.getMinElement();
if ( debug ) System.out.println("RESOLVED TO "+D.prediction+" for "+D);
predictedAlt = D.prediction;
// Falls through to check predicates below
}
else {
// CONFLICT, NONGREEDY (ATYPICAL SITUATION)
// upon ambiguity for nongreedy, default to exit branch to avoid inf loop
// this handles case where we find ambiguity that stops DFA construction
// before a config hits rule stop state. Was leaving prediction blank.
int exitAlt = 2;
// when ambig or ctx sens or nongreedy or .* loop hitting rule stop
D.isAcceptState = true;
D.prediction = predictedAlt = exitAlt;
}
}
}
if ( !greedy ) {
int exitAlt = 2;
if ( predictedAlt != ATN.INVALID_ALT_NUMBER && configWithAltAtStopState(reach, 1) ) {
if ( debug ) System.out.println("nongreedy loop but unique alt "+D.configs.uniqueAlt+" at "+reach);
// reaches end via .* means nothing after.
D.isAcceptState = true;
D.prediction = predictedAlt = exitAlt;
}
else {// if we reached end of rule via exit branch and decision nongreedy, we matched
if ( configWithAltAtStopState(reach, exitAlt) ) {
if ( debug ) System.out.println("nongreedy at stop state for exit branch");
D.isAcceptState = true;
D.prediction = predictedAlt = exitAlt;
// SLL CONFLICT; RETRY WITH FULL LL CONTEXT
// (it's possible SLL with preds could resolve to single alt
// which would mean we could avoid full LL, but not worth
// code complexity.)
if ( debug ) System.out.println("RETRY with outerContext="+outerContext);
// don't look up context in cache now since we're just creating state D
loopsSimulateTailRecursion = true;
ATNConfigSet s0_closure =
computeStartState(dfa.atnStartState,
outerContext,
loopsSimulateTailRecursion,
true);
predictedAlt = execATNWithFullContext(dfa, D, s0_closure,
input, startIndex,
outerContext,
D.configs.conflictingAlts.getMinElement());
// not accept state: isCtxSensitive
D.requiresFullContext = true; // always force DFA to ATN simulate
D.prediction = ATN.INVALID_ALT_NUMBER;
addDFAEdge(dfa, previousD, t, D);
return predictedAlt; // all done with preds, etc...
}
}
}
@ -789,15 +753,14 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
@NotNull ATNConfigSet s0,
@NotNull TokenStream input, int startIndex,
ParserRuleContext<?> outerContext,
int SLL_min_alt, // todo: is this in D as min ambig alts?
boolean greedy)
int SLL_min_alt) // todo: is this in D as min ambig alts?
{
// caller must have write lock on dfa
retry_with_context++;
reportAttemptingFullContext(dfa, s0, startIndex, input.index());
if ( debug || debug_list_atn_decisions ) {
System.out.println("execATNWithFullContext "+s0+", greedy="+greedy);
System.out.println("execATNWithFullContext "+s0);
}
boolean fullCtx = true;
ATNConfigSet reach = null;
@ -808,7 +771,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
// System.out.println("LL REACH "+getLookaheadName(input)+
// " from configs.size="+previous.size()+
// " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine());
reach = computeReachSet(previous, t, greedy, true, fullCtx);
reach = computeReachSet(previous, t, true, fullCtx);
if ( reach==null ) {
// if any configs in previous dipped into outer context, that
// means that input up to t actually finished entry rule
@ -854,7 +817,6 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
}
protected ATNConfigSet computeReachSet(ATNConfigSet closure, int t,
boolean greedy,
boolean loopsSimulateTailRecursion,
boolean fullCtx)
{
@ -890,7 +852,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
}
else {
for (ATNConfig c : intermediate) {
closure(c, reach, closureBusy, false, greedy,
closure(c, reach, closureBusy, false,
loopsSimulateTailRecursion, fullCtx);
}
}
@ -902,7 +864,6 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
@NotNull
public ATNConfigSet computeStartState(@NotNull ATNState p,
@Nullable RuleContext ctx,
boolean greedy,
boolean loopsSimulateTailRecursion,
boolean fullCtx)
{
@ -914,7 +875,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
ATNState target = p.transition(i).target;
ATNConfig c = new ATNConfig(target, i+1, initialContext);
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
closure(c, configs, closureBusy, true, greedy,
closure(c, configs, closureBusy, true,
loopsSimulateTailRecursion, fullCtx);
}
@ -1091,12 +1052,11 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
@NotNull ATNConfigSet configs,
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean greedy,
boolean loopsSimulateTailRecursion,
boolean fullCtx)
{
final int initialDepth = 0;
closureCheckingStopStateAndLoopRecursion(config, configs, closureBusy, collectPredicates, greedy,
closureCheckingStopStateAndLoopRecursion(config, configs, closureBusy, collectPredicates,
loopsSimulateTailRecursion,
fullCtx,
initialDepth);
@ -1106,7 +1066,6 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
@NotNull ATNConfigSet configs,
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean greedy,
boolean loopsSimulateTailRecursion,
boolean fullCtx,
int depth)
@ -1116,13 +1075,6 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
if ( !closureBusy.add(config) ) return; // avoid infinite recursion
if ( config.state instanceof RuleStopState ) {
if ( !greedy ) {
// don't see past end of a rule for any nongreedy decision
if ( debug ) System.out.println("NONGREEDY at stop state of "+
getRuleName(config.state.ruleIndex));
configs.add(config, mergeCache);
return;
}
// We hit rule end. If we have context info, use it
// run thru all possible stack tops in ctx
if ( config.context!=null && !config.context.isEmpty() ) {
@ -1131,7 +1083,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
// we have no context info, just chase follow links (if greedy)
if ( debug ) System.out.println("FALLING off rule "+
getRuleName(config.state.ruleIndex));
closure_(config, configs, closureBusy, collectPredicates, greedy,
closure_(config, configs, closureBusy, collectPredicates,
loopsSimulateTailRecursion, fullCtx, depth);
continue;
}
@ -1146,7 +1098,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
// Make sure we track that we are now out of context.
c.reachesIntoOuterContext = config.reachesIntoOuterContext;
assert depth > Integer.MIN_VALUE;
closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, collectPredicates, greedy,
closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, collectPredicates,
loopsSimulateTailRecursion,
fullCtx, depth - 1);
}
@ -1179,7 +1131,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
}
}
closure_(config, configs, closureBusy, collectPredicates, greedy,
closure_(config, configs, closureBusy, collectPredicates,
loopsSimulateTailRecursion, fullCtx, depth);
}
@ -1188,7 +1140,6 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
@NotNull ATNConfigSet configs,
@NotNull Set<ATNConfig> closureBusy,
boolean collectPredicates,
boolean greedy,
boolean loopsSimulateTailRecursion,
boolean fullCtx,
int depth)
@ -1233,7 +1184,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
}
}
closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, continueCollecting, greedy,
closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, continueCollecting,
loopsSimulateTailRecursion,
fullCtx, newDepth);
}

10
tool/playground/T.g
View File

@ -1,8 +1,4 @@
grammar T;
tokens {}
s : INT+ ;
ID : [a-z]+ ;
INT : [0-9]+ ;
WS : [ \t\n]+ -> skip ;
s : ID ;
ID : 'a'..'z'+ ;
WS : (' '|'\n') {skip();} ;

4
tool/src/org/antlr/v4/automata/ATNSerializer.java
View File

@ -226,7 +226,6 @@ public class ATNSerializer {
data.add(ndecisions);
for (DecisionState decStartState : atn.decisionToState) {
data.add(decStartState.stateNumber);
data.add(decStartState.isGreedy?1:0);
}
return data;
}
@ -300,8 +299,7 @@ public class ATNSerializer {
int ndecisions = ATNSimulator.toInt(data[p++]);
for (int i=1; i<=ndecisions; i++) {
int s = ATNSimulator.toInt(data[p++]);
int isGreedy = ATNSimulator.toInt(data[p++]);
buf.append(i-1).append(":").append(s).append(" ").append(isGreedy).append("\n");
buf.append(i-1).append(":").append(s).append("\n");
}
return buf.toString();
}

6
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -454,8 +454,7 @@ public class ParserATNFactory implements ATNFactory {
epsilon(blkEnd, loop); // blk can see loop back
BlockAST blkAST = (BlockAST)plusAST.getChild(0);
loop.isGreedy = isGreedy(blkAST);
if ( !g.isLexer() || loop.isGreedy ) {
if ( !g.isLexer() || isGreedy(blkAST) ) {
epsilon(loop, blkStart); // loop back to start
epsilon(loop, end); // or exit
}
@ -494,8 +493,7 @@ public class ParserATNFactory implements ATNFactory {
end.loopBackState = loop;
BlockAST blkAST = (BlockAST)starAST.getChild(0);
entry.isGreedy = isGreedy(blkAST);
if ( !g.isLexer() || entry.isGreedy ) {
if ( !g.isLexer() || isGreedy(blkAST) ) {
epsilon(entry, blkStart); // loop enter edge (alt 1)
epsilon(entry, end); // bypass loop edge (alt 2)
}

29
tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java
View File

@ -137,7 +137,18 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
@Override
public void modeDef(GrammarAST m, GrammarAST ID) {
checkMode(ID.token);
if ( !g.isLexer() ) {
g.tool.errMgr.grammarError(ErrorType.MODE_NOT_IN_LEXER, g.fileName,
ID.token, ID.token.getText(), g);
}
}
@Override
public void wildcardRef(GrammarAST ref) {
if ( !g.isLexer() ) {
g.tool.errMgr.grammarError(ErrorType.WILDCARD_IN_PARSER, g.fileName,
ref.getToken(), ref.getText(), g);
}
}
@Override
@ -258,13 +269,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
}
}
void checkMode(Token modeNameToken) {
if ( !g.isLexer() ) {
g.tool.errMgr.grammarError(ErrorType.MODE_NOT_IN_LEXER, g.fileName,
modeNameToken, modeNameToken.getText(), g);
}
}
void checkNumPrequels(List<GrammarAST> options,
List<GrammarAST> imports,
List<GrammarAST> tokens)
@ -334,7 +338,14 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
{
boolean ok = true;
if ( parent.getType()==ANTLRParser.BLOCK ) {
if ( !Grammar.subruleOptions.contains(optionID.getText()) ) { // block
if ( g.isLexer() && Grammar.LexerSubruleOptions.contains(optionID.getText()) ) { // block
g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
g.fileName,
optionID,
optionID.getText());
ok = false;
}
if ( !g.isLexer() && Grammar.ParserSubruleOptions.contains(optionID.getText()) ) { // block
g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION,
g.fileName,
optionID,

3
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -136,7 +136,7 @@ public enum ErrorType {
IMPORT_NAME_CLASH(113, "<arg.typeString> grammar <arg.name> and imported <arg2.typeString> grammar <arg2.name> both generate <arg2.recognizerName>", ErrorSeverity.ERROR),
AST_OP_WITH_NON_AST_OUTPUT_OPTION(114, " <arg>", ErrorSeverity.ERROR),
AST_OP_IN_ALT_WITH_REWRITE(115, "", ErrorSeverity.ERROR),
WILDCARD_AS_ROOT(116, "", ErrorSeverity.ERROR),
// WILDCARD_AS_ROOT(116, "", ErrorSeverity.ERROR),
CONFLICTING_OPTION_IN_TREE_FILTER(117, "", ErrorSeverity.ERROR),
ALL_OPS_NEED_SAME_ASSOC(118, "all operators of alt <arg> of left-recursive rule must have same associativity", ErrorSeverity.WARNING),
LEFT_RECURSION_CYCLES(119, "The following sets of rules are mutually left-recursive <arg:{c| [<c:{r|<r.name>}; separator=\", \">]}; separator=\" and \">", ErrorSeverity.ERROR),
@ -149,6 +149,7 @@ public enum ErrorType {
IMPLICIT_STRING_DEFINITION(126, "cannot create implicit token for string literal <arg> in non-combined grammar", ErrorSeverity.ERROR),
// ALIAS_REASSIGNMENT(127, "token literal <arg> aliased to new token name <arg2>", ErrorSeverity.WARNING),
ATTRIBUTE_IN_LEXER_ACTION(128, "attribute references not allowed in lexer actions: $<arg>", ErrorSeverity.ERROR),
WILDCARD_IN_PARSER(129, "wildcard '.' not allowed in parsers", ErrorSeverity.ERROR),
/** Documentation comment is unterminated */
//UNTERMINATED_DOC_COMMENT(, "", ErrorSeverity.ERROR),

5
tool/src/org/antlr/v4/tool/Grammar.java
View File

@ -82,7 +82,10 @@ public class Grammar implements AttributeResolver {
public static final Set<String> ruleOptions = new HashSet<String>() {{
}};
public static final Set<String> subruleOptions = new HashSet<String>() {{
public static final Set<String> ParserSubruleOptions = new HashSet<String>() {{
}};
public static final Set<String> LexerSubruleOptions = new HashSet<String>() {{
add("greedy");
}};

8
tool/test/org/antlr/v4/test/TestASTStructure.java
View File

@ -71,7 +71,7 @@ public class TestASTStructure {
// gunit test on line 18
RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n tokens { A, B }\n @header {foo}\n a : A;\n ", 18);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(PARSER_GRAMMAR P (tokens { A (= B '33')) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))";
Object expecting = "(PARSER_GRAMMAR P (tokens { A B) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))";
assertEquals("testing rule grammarSpec", expecting, actual);
}
@ -79,7 +79,7 @@ public class TestASTStructure {
// gunit test on line 30
RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n @header {foo}\n tokens { A,B }\n a : A;\n ", 30);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A (= B '33')) (RULES (RULE a (BLOCK (ALT A)))))";
Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A B) (RULES (RULE a (BLOCK (ALT A)))))";
assertEquals("testing rule grammarSpec", expecting, actual);
}
@ -113,9 +113,9 @@ public class TestASTStructure {
@Test public void test_rule3() throws Exception {
// gunit test on line 60
RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n public a[int i] returns [int y]\n options {backtrack=true;}\n @init {blort}\n : ID ;\n ", 60);
RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n a[int i] returns [int y]\n @init {blort}\n : ID ;\n ", 60);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(RULE a (RULEMODIFIERS public) int i (returns int y) (OPTIONS (= backtrack true)) (@ init {blort}) (BLOCK (ALT ID)))";
Object expecting = "(RULE a int i (returns int y) (@ init {blort}) (BLOCK (ALT ID)))";
assertEquals("testing rule rule", expecting, actual);
}

46
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -169,7 +169,7 @@ public class TestATNSerialization extends BaseTest {
"5->2 EPSILON 0,0,0\n" +
"5->3 EPSILON 0,0,0\n" +
"6->1 EPSILON 0,0,0\n" +
"0:5 1\n";
"0:5\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
@ -204,7 +204,7 @@ public class TestATNSerialization extends BaseTest {
"8->3 EPSILON 0,0,0\n" +
"8->5 EPSILON 0,0,0\n" +
"9->1 EPSILON 0,0,0\n" +
"0:8 1\n";
"0:8\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
@ -236,7 +236,7 @@ public class TestATNSerialization extends BaseTest {
"6->7 EPSILON 0,0,0\n" +
"7->8 ATOM 2,0,0\n" +
"8->1 EPSILON 0,0,0\n" +
"0:5 1\n";
"0:5\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
@ -298,7 +298,7 @@ public class TestATNSerialization extends BaseTest {
"6->2 EPSILON 0,0,0\n" +
"7->8 ATOM 98,0,0\n" +
"8->4 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -321,7 +321,7 @@ public class TestATNSerialization extends BaseTest {
"1->3 EPSILON 0,0,0\n" +
"3->4 RANGE 48,57,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -346,7 +346,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 ATOM 97,0,0\n" +
"4->5 ATOM -1,0,0\n" +
"5->2 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -374,8 +374,8 @@ public class TestATNSerialization extends BaseTest {
"4->6 SET 0,0,0\n" +
"5->4 EPSILON 0,0,0\n" +
"6->2 EPSILON 0,0,0\n" +
"0:0 1\n" +
"1:5 1\n";
"0:0\n" +
"1:5\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -405,8 +405,8 @@ public class TestATNSerialization extends BaseTest {
"6->4 EPSILON 0,0,0\n" +
"6->7 EPSILON 0,0,0\n" +
"7->2 EPSILON 0,0,0\n" +
"0:0 1\n" +
"1:6 1\n";
"0:0\n" +
"1:6\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -453,7 +453,7 @@ public class TestATNSerialization extends BaseTest {
"12->13 ATOM 99,0,0\n" +
"13->14 ACTION 2,1,0\n" +
"14->6 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -477,7 +477,7 @@ public class TestATNSerialization extends BaseTest {
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -501,7 +501,7 @@ public class TestATNSerialization extends BaseTest {
"1->3 EPSILON 0,0,0\n" +
"3->4 SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -525,7 +525,7 @@ public class TestATNSerialization extends BaseTest {
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -586,9 +586,9 @@ public class TestATNSerialization extends BaseTest {
"18->19 WILDCARD 0,0,0\n" +
"19->20 ACTION 2,1,0\n" +
"20->7 EPSILON 0,0,0\n" +
"0:0 1\n" +
"1:1 1\n" +
"2:11 1\n";
"0:0\n" +
"1:1\n" +
"2:11\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -615,7 +615,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 NOT_SET 0,0,0\n" +
"4->5 NOT_SET 1,0,0\n" +
"5->2 EPSILON 0,0,0\n" +
"0:0 1\n";
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -671,8 +671,8 @@ public class TestATNSerialization extends BaseTest {
"15->7 EPSILON 0,0,0\n" +
"16->17 ATOM 100,0,0\n" +
"17->9 EPSILON 0,0,0\n" +
"0:0 1\n" +
"1:1 1\n";
"0:0\n" +
"1:1\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
@ -721,9 +721,9 @@ public class TestATNSerialization extends BaseTest {
"12->6 EPSILON 0,0,0\n" +
"13->14 ATOM 99,0,0\n" +
"14->8 EPSILON 0,0,0\n" +
"0:0 1\n" +
"1:1 1\n" +
"2:2 1\n";
"0:0\n" +
"1:1\n" +
"2:2\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);

623
tool/test/org/antlr/v4/test/TestNonGreedyLoops.java
View File

@ -1,623 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test;
import org.junit.Test;
public class TestNonGreedyLoops extends BaseTest {
@Test public void testNongreedyLoopOnEndIsNop() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : any ID EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"any : .* ;\n"+
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"x", true);
assertEquals("x\n" +
"Decision 0:\n" +
"s0-ID->:s1=>2\n", found);
assertEquals(null, this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"34 x", true);
assertEquals("34x\n" +
"Decision 0:\n" +
"s0-INT->:s1=>2\n", found);
assertEquals("line 1:0 extraneous input '34' expecting ID\n", this.stderrDuringParse);
}
@Test public void testNongreedyPlusLoopOnEndIsNop() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : any ID EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"any : .+ ;\n"+ // .+ on end of rule always gives no viable alt. can't bypass but can't match
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"x", true);
assertEquals("x\n" +
"Decision 0:\n" +
"s0-ID->:s1=>2\n", found);
assertEquals("line 1:0 no viable alternative at input 'x'\n", this.stderrDuringParse);
}
@Test public void testNongreedyLoopInOtherRule() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : a {System.out.println(\"alt 1\");} | b {System.out.println(\"alt 2\");} ;\n" +
"a : .* ID ;\n"+
"b : .* INT ;\n"+
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"x", true);
assertEquals("alt 1\n" +
"Decision 0:\n" +
"s0-ID->s1\n" +
"s1-EOF->:s2=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-ID->:s1=>2\n", found);
assertEquals(null, this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"34", true);
assertEquals("alt 2\n" +
"Decision 0:\n" +
"s0-INT->s1\n" +
"s1-EOF->:s2=>2\n" +
"\n" +
"Decision 2:\n" +
"s0-INT->:s1=>2\n", found);
assertEquals(null, this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"34 x", true);
assertEquals("alt 1\n" +
"Decision 0:\n" +
"s0-INT->s1\n" +
"s1-ID->s2\n" +
"s2-EOF->:s3=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-INT->:s1=>1\n" +
"s0-ID->:s2=>2\n", found);
assertEquals(null, this.stderrDuringParse);
}
@Test public void testNongreedyPlusLoopInOtherRule() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : a {System.out.println(\"alt 1\");} | b {System.out.println(\"alt 2\");} ;\n" +
"a : .+ ID ;\n"+
"b : .+ INT ;\n"+
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"2 3 x", true);
assertEquals("alt 1\n" +
"Decision 0:\n" +
"s0-INT->s1\n" +
"s1-INT->s2\n" +
"s2-ID->s3\n" +
"s3-EOF->:s4=>1\n" +
"\n" +
"Decision 1:\n" +
"s0-INT->:s1=>1\n" +
"s0-ID->:s2=>2\n", found);
assertEquals(null, this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"2 3", true);
assertEquals("alt 2\n" +
"Decision 0:\n" +
"s0-INT->s1\n" +
"s1-INT->s2\n" +
"s2-EOF->:s3=>2\n" +
"\n" +
"Decision 2:\n" +
"s0-INT->:s1=>2\n", found);
assertEquals("line 1:0 no viable alternative at input '2'\n", this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"a b c 3", true);
assertEquals("alt 2\n" +
"Decision 0:\n" +
"s0-ID->s1\n" +
"s1-ID->s2\n" +
"s2-INT->s3\n" +
"s2-ID->s2\n" +
"s3-EOF->:s4=>2\n" +
"\n" +
"Decision 2:\n" +
"s0-INT->:s2=>2\n" +
"s0-ID->:s1=>1\n", found);
assertEquals(null, this.stderrDuringParse);
}
@Test public void testNongreedyLoopInOneAlt() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : a {System.out.println(\"alt 1\");} EOF | b {System.out.println(\"alt 2\");} EOF ;\n" +
"a : .* ;\n"+ // s comes here upon ID but then bypasses, error on EOF
"b : INT ;\n"+
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"x", true);
assertEquals("alt 1\n" +
"Decision 0:\n" +
"s0-ID->:s1=>1\n", found);
assertNull(this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"34", true);
assertEquals("alt 1\n" +
"Decision 0:\n" +
"s0-INT->s1\n" +
"s1-EOF->:s2=>1\n", found); // resolves INT EOF to alt 1 from s since ambig 'tween a and b
assertEquals("line 1:2 reportAmbiguity d=0: ambigAlts={1..2}, input='34'\n",
this.stderrDuringParse);
}
@Test public void testNongreedyLoopCantSeeEOF() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : block EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"block : '{' .* '}' ;\n"+
"EQ : '=' ;\n" +
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String input =
"{ }";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("{}\n" +
"Decision 0:\n" +
"s0-'}'->:s1=>2\n", found);
input =
"{a b { }";
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("{ab{}\n" +
"Decision 0:\n" +
"s0-'{'->:s1=>1\n" +
"s0-'}'->:s2=>2\n" +
"s0-ID->:s1=>1\n", found);
input =
"{ } a 2 { }"; // FAILS to match since it terminates loop at first { }
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("", found); // should not print output; resync kills rest of input til '}' then returns normally
}
@Test public void testNongreedyLoop() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"ifstat : 'if' '(' .* ')' block ;\n" +
"block : '{' '}' ;\n"+
"EQ : '=' ;\n" +
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String input =
"if ( x=34 ) { } ;";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("if(x=34){};\n" +
"Decision 0:\n" +
"s0-')'->s2\n" +
"s0-'='->:s1=>1\n" +
"s0-INT->:s1=>1\n" +
"s0-ID->:s1=>1\n" +
"s2-'{'->s3\n" +
"s3-'}'->:s4=>2\n", found);
input =
"if ( ))) ) { } ;";
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("if()))){};\n" +
"Decision 0:\n" +
"s0-')'->s1\n" +
"s1-'{'->s3\n" +
"s1-')'->:s2=>1\n" +
"s3-'}'->:s4=>2\n", found);
input =
"if (() { } a 2) { } ;"; // The first { } should match block so should stop
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("", found); // should not finish to print output
}
@Test public void testNongreedyLoopPassingThroughAnotherNongreedy() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"ifstat : 'if' '(' .* ')' block ;\n" +
"block : '{' (block|.)* '}' ;\n"+
"EQ : '=' ;\n" +
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String input =
"if ( x=34 ) { {return a} b 34 } ;";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("if(x=34){{returna}b34};\n" +
"Decision 0:\n" +
"s0-')'->s2\n" +
"s0-'='->:s1=>1\n" +
"s0-INT->:s1=>1\n" +
"s0-ID->:s1=>1\n" +
"s2-'{'->s3\n" +
"s3-'{'->s4\n" +
"s4-'}'->:s5=>2\n" +
"s4-ID->s4\n" +
"\n" +
"Decision 1:\n" +
"s0-'{'->:s1=>1\n" +
"s0-INT->:s2=>2\n" +
"s0-ID->:s2=>2\n" +
"\n" +
"Decision 2:\n" +
"s0-'{'->:s1=>1\n" +
"s0-'}'->:s3=>2\n" +
"s0-INT->:s2=>1\n" +
"s0-ID->:s2=>1\n", found);
input =
"if ( ()) ) { {return a} b 34 } ;";
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("if(())){{returna}b34};\n" +
"Decision 0:\n" +
"s0-')'->s2\n" +
"s0-'('->:s1=>1\n" +
"s2-'{'->s4\n" +
"s2-')'->:s3=>1\n" +
"s4-'{'->s5\n" +
"s5-'}'->:s6=>2\n" +
"s5-ID->s5\n" +
"\n" +
"Decision 1:\n" +
"s0-'{'->:s1=>1\n" +
"s0-INT->:s2=>2\n" +
"s0-ID->:s2=>2\n" +
"\n" +
"Decision 2:\n" +
"s0-'{'->:s1=>1\n" +
"s0-'}'->:s3=>2\n" +
"s0-INT->:s2=>1\n" +
"s0-ID->:s2=>1\n", found);
}
@Test public void testStatLoopNongreedyNotNecessary() throws Exception {
// EOF on end means LL(*) can identify when to stop the loop.
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : stat* ID '=' ID ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"stat : 'if' '(' INT ')' stat\n" +
" | 'return' INT ';'\n" +
" | ID '=' (INT|ID) ';'\n" +
" | block\n" +
" ;\n" +
"block : '{' stat* '}' ;\n"+
"EQ : '=' ;\n" +
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String input =
"x=1; a=b;";
String found = null;
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("x=1;a=b;\n" +
"Decision 0:\n" +
"s0-ID->s1\n" +
"s1-'='->s2\n" +
"s2-INT->:s3=>1\n" +
"s2-ID->s4\n" +
"s4-';'->s5\n" +
"s5-EOF->:s6=>2\n", found);
input =
"if ( 1 ) { x=3; { return 4; } } return 99; abc=def;";
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("if(1){x=3;{return4;}}return99;abc=def;\n" +
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'return'->:s2=>1\n" +
"s0-ID->s3\n" +
"s3-'='->s4\n" +
"s4-ID->s5\n" +
"s5-';'->s6\n" +
"s6-EOF->:s7=>2\n", found);
input =
"x=1; a=3;"; // FAILS to match since it can't match last element
execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
// can't match EOF to ID '=' '3' ';'
assertEquals("line 1:9 no viable alternative at input '<EOF>'\n",
this.stderrDuringParse);
input =
"x=1; a=b; z=3;"; // FAILS to match since it can't match last element
execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("line 1:14 no viable alternative at input '<EOF>'\n",
this.stderrDuringParse);
// should not finish to print output
}
@Test public void testStatLoopNongreedyNecessary() throws Exception {
// stops scanning ahead at end of rule s since decision is nongreedy.
// this says: "match statements until we see a=b; assignment; ignore any
// statements that follow."
String grammar =
"grammar T;\n" +
"random : s ;" + // call s so s isn't followed by EOF directly
"s @after {dumpDFA();} : (options {greedy=false;} : stat)* ID '=' ID ';'\n" +
" {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"stat : 'if' '(' INT ')' stat\n" +
" | 'return' INT ';'\n" +
" | ID '=' (INT|ID) ';'\n" +
" | block\n" +
" ;\n" +
"block : '{' stat* '}' ;\n"+
"EQ : '=' ;\n" +
"INT : '0'..'9'+ ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n')+ {skip();} ;\n";
String input =
"x=1; a=b; x=y;";
String found = null;
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("x=1;a=b;\n" +
"Decision 0:\n" +
"s0-ID->s1\n" +
"s1-'='->s2\n" +
"s2-INT->:s3=>1\n" +
"s2-ID->s4\n" +
"s4-';'->:s5=>2\n", found); // ignores x=1 that follows first a=b assignment
input =
"if ( 1 ) { x=3; { return 4; } } return 99; abc=def;";
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("if(1){x=3;{return4;}}return99;abc=def;\n" +
"Decision 0:\n" +
"s0-'if'->:s1=>1\n" +
"s0-'return'->:s2=>1\n" +
"s0-ID->s3\n" +
"s3-'='->s4\n" +
"s4-ID->s5\n" +
"s5-';'->:s6=>2\n", found);
input =
"x=1; a=3;"; // FAILS to match since it can't match either stat
execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
// can't match EOF to ID '=' '0' ';'
assertEquals("line 1:9 no viable alternative at input '<EOF>'\n",
this.stderrDuringParse);
input =
"x=1; a=b; z=3;"; // stops at a=b; ignores z=3;
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
input, true);
assertEquals("x=1;a=b;\n" +
"Decision 0:\n" +
"s0-ID->s1\n" +
"s1-'='->s2\n" +
"s2-INT->:s3=>1\n" +
"s2-ID->s4\n" +
"s4-';'->:s5=>2\n", found); // should not finish all input
}
@Test public void testHTMLTags() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : (item)+ {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
"item : tag | . ;\n" +
"tag : '<' '/'? .* '>' ;\n" +
"EQ : '=' ;\n" +
"COMMA : ',' ;\n" +
"ID : 'a'..'z'+ ;\n" +
"STR : '\"' .* '\"' ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = null;
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"<a>foo</a>", true);
assertEquals("<a>foo</a>\n" +
"Decision 1:\n" +
"s0-'<'->s1\n" +
"s0-ID->:s5=>2\n" +
"s1-'/'->s2\n" +
"s1-ID->s2\n" +
"s2-'>'->s3\n" +
"s2-ID->s2\n" +
"s3-EOF->s6^\n" +
"s3-'<'->s4^\n" +
"s3-ID->s3\n" +
"\n" +
"Decision 2:\n" +
"s0-'/'->:s2=>1\n" +
"s0-ID->:s1=>2\n" +
"\n" +
"Decision 3:\n" +
"s0-'>'->:s2=>2\n" +
"s0-ID->:s1=>1\n", found);
assertEquals("line 1:6 reportAttemptingFullContext d=1, input='<a>foo<'\n" +
"line 1:6 reportAmbiguity d=1: ambigAlts={1..2}, input='<a>foo<'\n" +
"line 1:10 reportAttemptingFullContext d=1, input='</a>'\n" +
"line 1:10 reportAmbiguity d=1: ambigAlts={1..2}, input='</a>'\n" +
"line 1:7 reportAmbiguity d=2: ambigAlts={1..2}, input='/'\n",
this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"<a></a>", true);
assertEquals("<a></a>\n" +
"Decision 1:\n" +
"s0-'<'->s1\n" +
"s1-'/'->s2\n" +
"s1-ID->s2\n" +
"s2-'>'->s3\n" +
"s2-ID->s2\n" +
"s3-EOF->s5^\n" +
"s3-'<'->s4^\n" +
"\n" +
"Decision 2:\n" +
"s0-'/'->:s2=>1\n" +
"s0-ID->:s1=>2\n" +
"\n" +
"Decision 3:\n" +
"s0-'>'->:s2=>2\n" +
"s0-ID->:s1=>1\n", found);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"</b><a src=\"abc\", width=32>", true);
assertEquals("</b><asrc=\"abc\",width=32>\n" +
"Decision 1:\n" +
"s0-'<'->s1\n" +
"s1-'/'->s2\n" +
"s1-ID->s2\n" +
"s2-'>'->s3\n" +
"s2-'='->s2\n" +
"s2-','->s2\n" +
"s2-ID->s2\n" +
"s2-STR->s2\n" +
"s2-INT->s2\n" +
"s3-EOF->s5^\n" +
"s3-'<'->s4^\n" +
"\n" +
"Decision 2:\n" +
"s0-'/'->:s1=>1\n" +
"s0-ID->:s2=>2\n" +
"\n" +
"Decision 3:\n" +
"s0-'>'->:s2=>2\n" +
"s0-'='->:s1=>1\n" +
"s0-','->:s1=>1\n" +
"s0-ID->:s1=>1\n" +
"s0-STR->:s1=>1\n" +
"s0-INT->:s1=>1\n", found);
}
/** lookahead prediction with '.' can be misleading since nongreedy. Lookahead
* that sees into a non-greedy loop, thinks it is greedy.
*/
@Test
public void testFindHTMLTags() throws Exception {
String grammar =
"grammar T;\n" +
"s @after {dumpDFA();} : ( .* (tag {System.out.println($tag.text);} |header) )* EOF;\n" +
"tag : '<' .+ '>' ;\n" +
"header : 'x' 'y' ;\n" +
"EQ : '=' ;\n" +
"COMMA : ',' ;\n" +
"ID : 'a'..'z'+ ;\n" +
"STR : '\"' .* '\"' ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = null;
System.out.println(grammar);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
",=foo <a x= 3>32skidoo<a><img>", true);
assertEquals("<ax=3>\n" +
"<a>\n" +
"<img>\n" +
"Decision 0:\n" + // .*
"s0-'<'->s2\n" +
"s0-'='->:s1=>1\n" +
"s0-','->:s1=>1\n" +
"s0-ID->:s1=>1\n" +
"s0-INT->:s1=>1\n" +
"s2-ID->s3\n" +
"s3-'x'->s4\n" +
"s3-'>'->:s5=>2\n" +
"s3-INT->s3\n" +
"s4-'='->s3\n" +
"\n" +
"Decision 3:\n" + // .+
"s0-'x'->:s1=>1\n" +
"s0-'>'->:s2=>2\n" +
"s0-'='->:s1=>1\n" +
"s0-ID->:s1=>1\n" +
"s0-INT->:s1=>1\n", found);
assertEquals(null,
this.stderrDuringParse);
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"x x<a>", true);
assertEquals("<a>\n" +
"Decision 0:\n" +
"s0-'x'->s1\n" +
"s0-'<'->s4\n" +
"s1-'x'->:s2=>1\n" +
"s1-'<'->:s3=>1\n" +
"s4-ID->s5\n" +
"s5-'>'->:s6=>2\n" +
"\n" +
"Decision 3:\n" +
"s0-'>'->:s2=>2\n" +
"s0-ID->:s1=>1\n", found);
// gets line 1:3 no viable alternative at input '>'. Why??
// oH! it sees .+ and figures it matches > so <> predicts tag CORRECT!
// Seeing '.' in a lookahead prediction can be misleading!!
found = execParser("T.g4", grammar, "TParser", "TLexer", "s",
"x <><a>", true);
assertEquals("<\n" +
"<a>\n" +
"Decision 0:\n" +
"s0-'x'->s1\n" +
"s0-'>'->:s6=>1\n" +
"s0-'<'->s3\n" +
"s1-'<'->:s2=>1\n" +
"s3-'>'->s4\n" +
"s3-ID->s4\n" +
"s4-'>'->:s7=>2\n" +
"s4-'<'->:s5=>2\n" +
"\n" +
"Decision 3:\n" +
"s0-'>'->:s1=>2\n" +
"s0-ID->:s2=>1\n", // doesn't match tag; null
found);
assertEquals("line 1:3 no viable alternative at input '>'\n",
this.stderrDuringParse);
}
}