diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java index 2c1cd5929..0b71c0f5e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java @@ -250,11 +250,9 @@ public abstract class ATNSimulator { int ndecisions = toInt(data[p++]); for (int i=1; i<=ndecisions; i++) { int s = toInt(data[p++]); - int isGreedy = toInt(data[p++]); DecisionState decState = (DecisionState)atn.states.get(s); atn.decisionToState.add(decState); decState.decision = i-1; - decState.isGreedy = isGreedy==1; } verifyATN(atn); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java index 7ed517e1f..161c978a1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java @@ -31,6 +31,4 @@ package org.antlr.v4.runtime.atn; public class DecisionState extends ATNState { public int decision = -1; - - public boolean isGreedy = true; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 74ca22113..3c98641f4 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -384,13 +384,12 @@ public class ParserATNSimulator extends ATNSimulator { ", outerContext="+outerContext.toString(parser)); } DecisionState decState = atn.getDecisionState(dfa.decision); - boolean greedy = decState.isGreedy; boolean loopsSimulateTailRecursion = SLL_loopsSimulateTailRecursion; boolean fullCtx = false; ATNConfigSet s0_closure = computeStartState(dfa.atnStartState, ParserRuleContext.EMPTY, - greedy, loopsSimulateTailRecursion, + loopsSimulateTailRecursion, fullCtx); dfa.s0 = addDFAState(dfa, new DFAState(s0_closure)); @@ -428,7 +427,6 @@ public class ParserATNSimulator extends ATNSimulator { DFAState s = s0; DecisionState decState = atn.getDecisionState(dfa.decision); - boolean greedy = decState.isGreedy; int t = input.LA(1); loop: @@ -440,14 +438,13 @@ public class ParserATNSimulator extends ATNSimulator { boolean fullCtx = true; ATNConfigSet s0_closure = computeStartState(dfa.atnStartState, outerContext, - greedy, loopsSimulateTailRecursion, + loopsSimulateTailRecursion, fullCtx); retry_with_context_from_dfa++; int alt = execATNWithFullContext(dfa, s, s0_closure, input, startIndex, outerContext, - ATN.INVALID_ALT_NUMBER, - greedy); + ATN.INVALID_ALT_NUMBER); return alt; } if ( s.isAcceptState ) { @@ -598,13 +595,11 @@ public class ParserATNSimulator extends ATNSimulator { int t = input.LA(1); DecisionState decState = atn.getDecisionState(dfa.decision); - boolean greedy = decState.isGreedy; while (true) { // while more work boolean loopsSimulateTailRecursion = SLL_loopsSimulateTailRecursion; // System.out.println("REACH "+getLookaheadName(input)); ATNConfigSet reach = computeReachSet(previous, t, - greedy, loopsSimulateTailRecursion, false); if ( reach==null ) { @@ -658,75 +653,44 @@ public class ParserATNSimulator extends ATNSimulator { // boolean cont = needMoreLookaheadLL(reach); D.configs.conflictingAlts = getConflictingAlts(reach); if ( D.configs.conflictingAlts!=null ) { - if ( greedy ) { - // CONFLICT, GREEDY (TYPICAL SITUATION) - if ( outerContext == ParserRuleContext.EMPTY || // in grammar start rule - !D.configs.dipsIntoOuterContext || // didn't fall out of rule - SLL ) // forcing SLL only - { - // SPECIAL CASE WHERE SLL KNOWS CONFLICT IS AMBIGUITY - if ( !D.configs.hasSemanticContext ) { - reportAmbiguity(dfa, D, startIndex, input.index(), - D.configs.conflictingAlts, D.configs); - } - D.isAcceptState = true; - D.prediction = D.configs.conflictingAlts.getMinElement(); - if ( debug ) System.out.println("RESOLVED TO "+D.prediction+" for "+D); - predictedAlt = D.prediction; - // Falls through to check predicates below - } - else { - // SLL CONFLICT; RETRY WITH FULL LL CONTEXT - // (it's possible SLL with preds could resolve to single alt - // which would mean we could avoid full LL, but not worth - // code complexity.) - if ( debug ) System.out.println("RETRY with outerContext="+outerContext); - // don't look up context in cache now since we're just creating state D - loopsSimulateTailRecursion = true; - ATNConfigSet s0_closure = - computeStartState(dfa.atnStartState, - outerContext, - greedy, - loopsSimulateTailRecursion, - true); - predictedAlt = execATNWithFullContext(dfa, D, s0_closure, - input, startIndex, - outerContext, - D.configs.conflictingAlts.getMinElement(), - greedy); - // not accept state: isCtxSensitive - D.requiresFullContext = true; // always force DFA to ATN simulate - D.prediction = ATN.INVALID_ALT_NUMBER; - addDFAEdge(dfa, previousD, t, D); - return predictedAlt; // all done with preds, etc... + // CONFLICT, GREEDY (TYPICAL SITUATION) + if ( outerContext == ParserRuleContext.EMPTY || // in grammar start rule + !D.configs.dipsIntoOuterContext || // didn't fall out of rule + SLL ) // forcing SLL only + { + // SPECIAL CASE WHERE SLL KNOWS CONFLICT IS AMBIGUITY + if ( !D.configs.hasSemanticContext ) { + reportAmbiguity(dfa, D, startIndex, input.index(), + D.configs.conflictingAlts, D.configs); } + D.isAcceptState = true; + D.prediction = D.configs.conflictingAlts.getMinElement(); + if ( debug ) System.out.println("RESOLVED TO "+D.prediction+" for "+D); + predictedAlt = D.prediction; + // Falls through to check predicates below } else { - // CONFLICT, NONGREEDY (ATYPICAL SITUATION) - // upon ambiguity for nongreedy, default to exit branch to avoid inf loop - // this handles case where we find ambiguity that stops DFA construction - // before a config hits rule stop state. Was leaving prediction blank. - int exitAlt = 2; - // when ambig or ctx sens or nongreedy or .* loop hitting rule stop - D.isAcceptState = true; - D.prediction = predictedAlt = exitAlt; - } - } - } - - if ( !greedy ) { - int exitAlt = 2; - if ( predictedAlt != ATN.INVALID_ALT_NUMBER && configWithAltAtStopState(reach, 1) ) { - if ( debug ) System.out.println("nongreedy loop but unique alt "+D.configs.uniqueAlt+" at "+reach); - // reaches end via .* means nothing after. - D.isAcceptState = true; - D.prediction = predictedAlt = exitAlt; - } - else {// if we reached end of rule via exit branch and decision nongreedy, we matched - if ( configWithAltAtStopState(reach, exitAlt) ) { - if ( debug ) System.out.println("nongreedy at stop state for exit branch"); - D.isAcceptState = true; - D.prediction = predictedAlt = exitAlt; + // SLL CONFLICT; RETRY WITH FULL LL CONTEXT + // (it's possible SLL with preds could resolve to single alt + // which would mean we could avoid full LL, but not worth + // code complexity.) + if ( debug ) System.out.println("RETRY with outerContext="+outerContext); + // don't look up context in cache now since we're just creating state D + loopsSimulateTailRecursion = true; + ATNConfigSet s0_closure = + computeStartState(dfa.atnStartState, + outerContext, + loopsSimulateTailRecursion, + true); + predictedAlt = execATNWithFullContext(dfa, D, s0_closure, + input, startIndex, + outerContext, + D.configs.conflictingAlts.getMinElement()); + // not accept state: isCtxSensitive + D.requiresFullContext = true; // always force DFA to ATN simulate + D.prediction = ATN.INVALID_ALT_NUMBER; + addDFAEdge(dfa, previousD, t, D); + return predictedAlt; // all done with preds, etc... } } } @@ -789,15 +753,14 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull ATNConfigSet s0, @NotNull TokenStream input, int startIndex, ParserRuleContext outerContext, - int SLL_min_alt, // todo: is this in D as min ambig alts? - boolean greedy) + int SLL_min_alt) // todo: is this in D as min ambig alts? { // caller must have write lock on dfa retry_with_context++; reportAttemptingFullContext(dfa, s0, startIndex, input.index()); if ( debug || debug_list_atn_decisions ) { - System.out.println("execATNWithFullContext "+s0+", greedy="+greedy); + System.out.println("execATNWithFullContext "+s0); } boolean fullCtx = true; ATNConfigSet reach = null; @@ -808,7 +771,7 @@ public class ParserATNSimulator extends ATNSimulator { // System.out.println("LL REACH "+getLookaheadName(input)+ // " from configs.size="+previous.size()+ // " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); - reach = computeReachSet(previous, t, greedy, true, fullCtx); + reach = computeReachSet(previous, t, true, fullCtx); if ( reach==null ) { // if any configs in previous dipped into outer context, that // means that input up to t actually finished entry rule @@ -854,7 +817,6 @@ public class ParserATNSimulator extends ATNSimulator { } protected ATNConfigSet computeReachSet(ATNConfigSet closure, int t, - boolean greedy, boolean loopsSimulateTailRecursion, boolean fullCtx) { @@ -890,7 +852,7 @@ public class ParserATNSimulator extends ATNSimulator { } else { for (ATNConfig c : intermediate) { - closure(c, reach, closureBusy, false, greedy, + closure(c, reach, closureBusy, false, loopsSimulateTailRecursion, fullCtx); } } @@ -902,7 +864,6 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull public ATNConfigSet computeStartState(@NotNull ATNState p, @Nullable RuleContext ctx, - boolean greedy, boolean loopsSimulateTailRecursion, boolean fullCtx) { @@ -914,7 +875,7 @@ public class ParserATNSimulator extends ATNSimulator { ATNState target = p.transition(i).target; ATNConfig c = new ATNConfig(target, i+1, initialContext); Set closureBusy = new HashSet(); - closure(c, configs, closureBusy, true, greedy, + closure(c, configs, closureBusy, true, loopsSimulateTailRecursion, fullCtx); } @@ -1091,12 +1052,11 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull ATNConfigSet configs, @NotNull Set closureBusy, boolean collectPredicates, - boolean greedy, boolean loopsSimulateTailRecursion, boolean fullCtx) { final int initialDepth = 0; - closureCheckingStopStateAndLoopRecursion(config, configs, closureBusy, collectPredicates, greedy, + closureCheckingStopStateAndLoopRecursion(config, configs, closureBusy, collectPredicates, loopsSimulateTailRecursion, fullCtx, initialDepth); @@ -1106,7 +1066,6 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull ATNConfigSet configs, @NotNull Set closureBusy, boolean collectPredicates, - boolean greedy, boolean loopsSimulateTailRecursion, boolean fullCtx, int depth) @@ -1116,13 +1075,6 @@ public class ParserATNSimulator extends ATNSimulator { if ( !closureBusy.add(config) ) return; // avoid infinite recursion if ( config.state instanceof RuleStopState ) { - if ( !greedy ) { - // don't see past end of a rule for any nongreedy decision - if ( debug ) System.out.println("NONGREEDY at stop state of "+ - getRuleName(config.state.ruleIndex)); - configs.add(config, mergeCache); - return; - } // We hit rule end. If we have context info, use it // run thru all possible stack tops in ctx if ( config.context!=null && !config.context.isEmpty() ) { @@ -1131,7 +1083,7 @@ public class ParserATNSimulator extends ATNSimulator { // we have no context info, just chase follow links (if greedy) if ( debug ) System.out.println("FALLING off rule "+ getRuleName(config.state.ruleIndex)); - closure_(config, configs, closureBusy, collectPredicates, greedy, + closure_(config, configs, closureBusy, collectPredicates, loopsSimulateTailRecursion, fullCtx, depth); continue; } @@ -1146,7 +1098,7 @@ public class ParserATNSimulator extends ATNSimulator { // Make sure we track that we are now out of context. c.reachesIntoOuterContext = config.reachesIntoOuterContext; assert depth > Integer.MIN_VALUE; - closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, collectPredicates, greedy, + closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, collectPredicates, loopsSimulateTailRecursion, fullCtx, depth - 1); } @@ -1179,7 +1131,7 @@ public class ParserATNSimulator extends ATNSimulator { } } - closure_(config, configs, closureBusy, collectPredicates, greedy, + closure_(config, configs, closureBusy, collectPredicates, loopsSimulateTailRecursion, fullCtx, depth); } @@ -1188,7 +1140,6 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull ATNConfigSet configs, @NotNull Set closureBusy, boolean collectPredicates, - boolean greedy, boolean loopsSimulateTailRecursion, boolean fullCtx, int depth) @@ -1233,7 +1184,7 @@ public class ParserATNSimulator extends ATNSimulator { } } - closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, continueCollecting, greedy, + closureCheckingStopStateAndLoopRecursion(c, configs, closureBusy, continueCollecting, loopsSimulateTailRecursion, fullCtx, newDepth); } diff --git a/tool/playground/T.g b/tool/playground/T.g index 345155ff1..3cf82b68e 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,8 +1,4 @@ grammar T; - -tokens {} -s : INT+ ; - -ID : [a-z]+ ; -INT : [0-9]+ ; -WS : [ \t\n]+ -> skip ; +s : ID ; +ID : 'a'..'z'+ ; +WS : (' '|'\n') {skip();} ; diff --git a/tool/src/org/antlr/v4/automata/ATNSerializer.java b/tool/src/org/antlr/v4/automata/ATNSerializer.java index 6781055ae..274e8b4af 100644 --- a/tool/src/org/antlr/v4/automata/ATNSerializer.java +++ b/tool/src/org/antlr/v4/automata/ATNSerializer.java @@ -226,7 +226,6 @@ public class ATNSerializer { data.add(ndecisions); for (DecisionState decStartState : atn.decisionToState) { data.add(decStartState.stateNumber); - data.add(decStartState.isGreedy?1:0); } return data; } @@ -300,8 +299,7 @@ public class ATNSerializer { int ndecisions = ATNSimulator.toInt(data[p++]); for (int i=1; i<=ndecisions; i++) { int s = ATNSimulator.toInt(data[p++]); - int isGreedy = ATNSimulator.toInt(data[p++]); - buf.append(i-1).append(":").append(s).append(" ").append(isGreedy).append("\n"); + buf.append(i-1).append(":").append(s).append("\n"); } return buf.toString(); } diff --git a/tool/src/org/antlr/v4/automata/ParserATNFactory.java b/tool/src/org/antlr/v4/automata/ParserATNFactory.java index 590a93d90..37a136e57 100644 --- a/tool/src/org/antlr/v4/automata/ParserATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ParserATNFactory.java @@ -454,8 +454,7 @@ public class ParserATNFactory implements ATNFactory { epsilon(blkEnd, loop); // blk can see loop back BlockAST blkAST = (BlockAST)plusAST.getChild(0); - loop.isGreedy = isGreedy(blkAST); - if ( !g.isLexer() || loop.isGreedy ) { + if ( !g.isLexer() || isGreedy(blkAST) ) { epsilon(loop, blkStart); // loop back to start epsilon(loop, end); // or exit } @@ -494,8 +493,7 @@ public class ParserATNFactory implements ATNFactory { end.loopBackState = loop; BlockAST blkAST = (BlockAST)starAST.getChild(0); - entry.isGreedy = isGreedy(blkAST); - if ( !g.isLexer() || entry.isGreedy ) { + if ( !g.isLexer() || isGreedy(blkAST) ) { epsilon(entry, blkStart); // loop enter edge (alt 1) epsilon(entry, end); // bypass loop edge (alt 2) } diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index 1289a25bb..4ec33f086 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -137,7 +137,18 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { @Override public void modeDef(GrammarAST m, GrammarAST ID) { - checkMode(ID.token); + if ( !g.isLexer() ) { + g.tool.errMgr.grammarError(ErrorType.MODE_NOT_IN_LEXER, g.fileName, + ID.token, ID.token.getText(), g); + } + } + + @Override + public void wildcardRef(GrammarAST ref) { + if ( !g.isLexer() ) { + g.tool.errMgr.grammarError(ErrorType.WILDCARD_IN_PARSER, g.fileName, + ref.getToken(), ref.getText(), g); + } } @Override @@ -258,13 +269,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { } } - void checkMode(Token modeNameToken) { - if ( !g.isLexer() ) { - g.tool.errMgr.grammarError(ErrorType.MODE_NOT_IN_LEXER, g.fileName, - modeNameToken, modeNameToken.getText(), g); - } - } - void checkNumPrequels(List options, List imports, List tokens) @@ -334,7 +338,14 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { { boolean ok = true; if ( parent.getType()==ANTLRParser.BLOCK ) { - if ( !Grammar.subruleOptions.contains(optionID.getText()) ) { // block + if ( g.isLexer() && Grammar.LexerSubruleOptions.contains(optionID.getText()) ) { // block + g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, + g.fileName, + optionID, + optionID.getText()); + ok = false; + } + if ( !g.isLexer() && Grammar.ParserSubruleOptions.contains(optionID.getText()) ) { // block g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, g.fileName, optionID, diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index b679d9411..30635de4e 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -136,7 +136,7 @@ public enum ErrorType { IMPORT_NAME_CLASH(113, " grammar and imported grammar both generate ", ErrorSeverity.ERROR), AST_OP_WITH_NON_AST_OUTPUT_OPTION(114, " ", ErrorSeverity.ERROR), AST_OP_IN_ALT_WITH_REWRITE(115, "", ErrorSeverity.ERROR), - WILDCARD_AS_ROOT(116, "", ErrorSeverity.ERROR), +// WILDCARD_AS_ROOT(116, "", ErrorSeverity.ERROR), CONFLICTING_OPTION_IN_TREE_FILTER(117, "", ErrorSeverity.ERROR), ALL_OPS_NEED_SAME_ASSOC(118, "all operators of alt of left-recursive rule must have same associativity", ErrorSeverity.WARNING), LEFT_RECURSION_CYCLES(119, "The following sets of rules are mutually left-recursive }; separator=\", \">]}; separator=\" and \">", ErrorSeverity.ERROR), @@ -149,6 +149,7 @@ public enum ErrorType { IMPLICIT_STRING_DEFINITION(126, "cannot create implicit token for string literal in non-combined grammar", ErrorSeverity.ERROR), // ALIAS_REASSIGNMENT(127, "token literal aliased to new token name ", ErrorSeverity.WARNING), ATTRIBUTE_IN_LEXER_ACTION(128, "attribute references not allowed in lexer actions: $", ErrorSeverity.ERROR), + WILDCARD_IN_PARSER(129, "wildcard '.' not allowed in parsers", ErrorSeverity.ERROR), /** Documentation comment is unterminated */ //UNTERMINATED_DOC_COMMENT(, "", ErrorSeverity.ERROR), diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index cac85cc79..03ec2643c 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -82,7 +82,10 @@ public class Grammar implements AttributeResolver { public static final Set ruleOptions = new HashSet() {{ }}; - public static final Set subruleOptions = new HashSet() {{ + public static final Set ParserSubruleOptions = new HashSet() {{ + }}; + + public static final Set LexerSubruleOptions = new HashSet() {{ add("greedy"); }}; diff --git a/tool/test/org/antlr/v4/test/TestASTStructure.java b/tool/test/org/antlr/v4/test/TestASTStructure.java index 1abe74bf7..f22c83f9c 100644 --- a/tool/test/org/antlr/v4/test/TestASTStructure.java +++ b/tool/test/org/antlr/v4/test/TestASTStructure.java @@ -71,7 +71,7 @@ public class TestASTStructure { // gunit test on line 18 RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n tokens { A, B }\n @header {foo}\n a : A;\n ", 18); Object actual = ((Tree)rstruct.getTree()).toStringTree(); - Object expecting = "(PARSER_GRAMMAR P (tokens { A (= B '33')) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))"; + Object expecting = "(PARSER_GRAMMAR P (tokens { A B) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))"; assertEquals("testing rule grammarSpec", expecting, actual); } @@ -79,7 +79,7 @@ public class TestASTStructure { // gunit test on line 30 RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n @header {foo}\n tokens { A,B }\n a : A;\n ", 30); Object actual = ((Tree)rstruct.getTree()).toStringTree(); - Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A (= B '33')) (RULES (RULE a (BLOCK (ALT A)))))"; + Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A B) (RULES (RULE a (BLOCK (ALT A)))))"; assertEquals("testing rule grammarSpec", expecting, actual); } @@ -113,9 +113,9 @@ public class TestASTStructure { @Test public void test_rule3() throws Exception { // gunit test on line 60 - RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n public a[int i] returns [int y]\n options {backtrack=true;}\n @init {blort}\n : ID ;\n ", 60); + RuleReturnScope rstruct = (RuleReturnScope)execParser("rule", "\n a[int i] returns [int y]\n @init {blort}\n : ID ;\n ", 60); Object actual = ((Tree)rstruct.getTree()).toStringTree(); - Object expecting = "(RULE a (RULEMODIFIERS public) int i (returns int y) (OPTIONS (= backtrack true)) (@ init {blort}) (BLOCK (ALT ID)))"; + Object expecting = "(RULE a int i (returns int y) (@ init {blort}) (BLOCK (ALT ID)))"; assertEquals("testing rule rule", expecting, actual); } diff --git a/tool/test/org/antlr/v4/test/TestATNSerialization.java b/tool/test/org/antlr/v4/test/TestATNSerialization.java index c9e94cf02..c2928972c 100644 --- a/tool/test/org/antlr/v4/test/TestATNSerialization.java +++ b/tool/test/org/antlr/v4/test/TestATNSerialization.java @@ -169,7 +169,7 @@ public class TestATNSerialization extends BaseTest { "5->2 EPSILON 0,0,0\n" + "5->3 EPSILON 0,0,0\n" + "6->1 EPSILON 0,0,0\n" + - "0:5 1\n"; + "0:5\n"; ATN atn = createATN(g); String result = ATNSerializer.getDecoded(g, atn); assertEquals(expecting, result); @@ -204,7 +204,7 @@ public class TestATNSerialization extends BaseTest { "8->3 EPSILON 0,0,0\n" + "8->5 EPSILON 0,0,0\n" + "9->1 EPSILON 0,0,0\n" + - "0:8 1\n"; + "0:8\n"; ATN atn = createATN(g); String result = ATNSerializer.getDecoded(g, atn); assertEquals(expecting, result); @@ -236,7 +236,7 @@ public class TestATNSerialization extends BaseTest { "6->7 EPSILON 0,0,0\n" + "7->8 ATOM 2,0,0\n" + "8->1 EPSILON 0,0,0\n" + - "0:5 1\n"; + "0:5\n"; ATN atn = createATN(g); String result = ATNSerializer.getDecoded(g, atn); assertEquals(expecting, result); @@ -298,7 +298,7 @@ public class TestATNSerialization extends BaseTest { "6->2 EPSILON 0,0,0\n" + "7->8 ATOM 98,0,0\n" + "8->4 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -321,7 +321,7 @@ public class TestATNSerialization extends BaseTest { "1->3 EPSILON 0,0,0\n" + "3->4 RANGE 48,57,0\n" + "4->2 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -346,7 +346,7 @@ public class TestATNSerialization extends BaseTest { "3->4 ATOM 97,0,0\n" + "4->5 ATOM -1,0,0\n" + "5->2 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -374,8 +374,8 @@ public class TestATNSerialization extends BaseTest { "4->6 SET 0,0,0\n" + "5->4 EPSILON 0,0,0\n" + "6->2 EPSILON 0,0,0\n" + - "0:0 1\n" + - "1:5 1\n"; + "0:0\n" + + "1:5\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -405,8 +405,8 @@ public class TestATNSerialization extends BaseTest { "6->4 EPSILON 0,0,0\n" + "6->7 EPSILON 0,0,0\n" + "7->2 EPSILON 0,0,0\n" + - "0:0 1\n" + - "1:6 1\n"; + "0:0\n" + + "1:6\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -453,7 +453,7 @@ public class TestATNSerialization extends BaseTest { "12->13 ATOM 99,0,0\n" + "13->14 ACTION 2,1,0\n" + "14->6 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -477,7 +477,7 @@ public class TestATNSerialization extends BaseTest { "1->3 EPSILON 0,0,0\n" + "3->4 NOT_SET 0,0,0\n" + "4->2 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -501,7 +501,7 @@ public class TestATNSerialization extends BaseTest { "1->3 EPSILON 0,0,0\n" + "3->4 SET 0,0,0\n" + "4->2 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -525,7 +525,7 @@ public class TestATNSerialization extends BaseTest { "1->3 EPSILON 0,0,0\n" + "3->4 NOT_SET 0,0,0\n" + "4->2 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -586,9 +586,9 @@ public class TestATNSerialization extends BaseTest { "18->19 WILDCARD 0,0,0\n" + "19->20 ACTION 2,1,0\n" + "20->7 EPSILON 0,0,0\n" + - "0:0 1\n" + - "1:1 1\n" + - "2:11 1\n"; + "0:0\n" + + "1:1\n" + + "2:11\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -615,7 +615,7 @@ public class TestATNSerialization extends BaseTest { "3->4 NOT_SET 0,0,0\n" + "4->5 NOT_SET 1,0,0\n" + "5->2 EPSILON 0,0,0\n" + - "0:0 1\n"; + "0:0\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -671,8 +671,8 @@ public class TestATNSerialization extends BaseTest { "15->7 EPSILON 0,0,0\n" + "16->17 ATOM 100,0,0\n" + "17->9 EPSILON 0,0,0\n" + - "0:0 1\n" + - "1:1 1\n"; + "0:0\n" + + "1:1\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); @@ -721,9 +721,9 @@ public class TestATNSerialization extends BaseTest { "12->6 EPSILON 0,0,0\n" + "13->14 ATOM 99,0,0\n" + "14->8 EPSILON 0,0,0\n" + - "0:0 1\n" + - "1:1 1\n" + - "2:2 1\n"; + "0:0\n" + + "1:1\n" + + "2:2\n"; ATN atn = createATN(lg); String result = ATNSerializer.getDecoded(lg, atn); assertEquals(expecting, result); diff --git a/tool/test/org/antlr/v4/test/TestNonGreedyLoops.java b/tool/test/org/antlr/v4/test/TestNonGreedyLoops.java deleted file mode 100644 index 861aaa681..000000000 --- a/tool/test/org/antlr/v4/test/TestNonGreedyLoops.java +++ /dev/null @@ -1,623 +0,0 @@ -/* - [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.antlr.v4.test; - -import org.junit.Test; - -public class TestNonGreedyLoops extends BaseTest { - @Test public void testNongreedyLoopOnEndIsNop() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : any ID EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "any : .* ;\n"+ - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "x", true); - assertEquals("x\n" + - "Decision 0:\n" + - "s0-ID->:s1=>2\n", found); - assertEquals(null, this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "34 x", true); - assertEquals("34x\n" + - "Decision 0:\n" + - "s0-INT->:s1=>2\n", found); - assertEquals("line 1:0 extraneous input '34' expecting ID\n", this.stderrDuringParse); - } - - @Test public void testNongreedyPlusLoopOnEndIsNop() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : any ID EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "any : .+ ;\n"+ // .+ on end of rule always gives no viable alt. can't bypass but can't match - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "x", true); - assertEquals("x\n" + - "Decision 0:\n" + - "s0-ID->:s1=>2\n", found); - assertEquals("line 1:0 no viable alternative at input 'x'\n", this.stderrDuringParse); - } - - @Test public void testNongreedyLoopInOtherRule() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : a {System.out.println(\"alt 1\");} | b {System.out.println(\"alt 2\");} ;\n" + - "a : .* ID ;\n"+ - "b : .* INT ;\n"+ - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "x", true); - assertEquals("alt 1\n" + - "Decision 0:\n" + - "s0-ID->s1\n" + - "s1-EOF->:s2=>1\n" + - "\n" + - "Decision 1:\n" + - "s0-ID->:s1=>2\n", found); - assertEquals(null, this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "34", true); - assertEquals("alt 2\n" + - "Decision 0:\n" + - "s0-INT->s1\n" + - "s1-EOF->:s2=>2\n" + - "\n" + - "Decision 2:\n" + - "s0-INT->:s1=>2\n", found); - assertEquals(null, this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "34 x", true); - assertEquals("alt 1\n" + - "Decision 0:\n" + - "s0-INT->s1\n" + - "s1-ID->s2\n" + - "s2-EOF->:s3=>1\n" + - "\n" + - "Decision 1:\n" + - "s0-INT->:s1=>1\n" + - "s0-ID->:s2=>2\n", found); - assertEquals(null, this.stderrDuringParse); - } - - @Test public void testNongreedyPlusLoopInOtherRule() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : a {System.out.println(\"alt 1\");} | b {System.out.println(\"alt 2\");} ;\n" + - "a : .+ ID ;\n"+ - "b : .+ INT ;\n"+ - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "2 3 x", true); - assertEquals("alt 1\n" + - "Decision 0:\n" + - "s0-INT->s1\n" + - "s1-INT->s2\n" + - "s2-ID->s3\n" + - "s3-EOF->:s4=>1\n" + - "\n" + - "Decision 1:\n" + - "s0-INT->:s1=>1\n" + - "s0-ID->:s2=>2\n", found); - assertEquals(null, this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "2 3", true); - assertEquals("alt 2\n" + - "Decision 0:\n" + - "s0-INT->s1\n" + - "s1-INT->s2\n" + - "s2-EOF->:s3=>2\n" + - "\n" + - "Decision 2:\n" + - "s0-INT->:s1=>2\n", found); - assertEquals("line 1:0 no viable alternative at input '2'\n", this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "a b c 3", true); - assertEquals("alt 2\n" + - "Decision 0:\n" + - "s0-ID->s1\n" + - "s1-ID->s2\n" + - "s2-INT->s3\n" + - "s2-ID->s2\n" + - "s3-EOF->:s4=>2\n" + - "\n" + - "Decision 2:\n" + - "s0-INT->:s2=>2\n" + - "s0-ID->:s1=>1\n", found); - assertEquals(null, this.stderrDuringParse); - } - - @Test public void testNongreedyLoopInOneAlt() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : a {System.out.println(\"alt 1\");} EOF | b {System.out.println(\"alt 2\");} EOF ;\n" + - "a : .* ;\n"+ // s comes here upon ID but then bypasses, error on EOF - "b : INT ;\n"+ - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "x", true); - assertEquals("alt 1\n" + - "Decision 0:\n" + - "s0-ID->:s1=>1\n", found); - assertNull(this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "34", true); - assertEquals("alt 1\n" + - "Decision 0:\n" + - "s0-INT->s1\n" + - "s1-EOF->:s2=>1\n", found); // resolves INT EOF to alt 1 from s since ambig 'tween a and b - assertEquals("line 1:2 reportAmbiguity d=0: ambigAlts={1..2}, input='34'\n", - this.stderrDuringParse); - } - - @Test public void testNongreedyLoopCantSeeEOF() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : block EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "block : '{' .* '}' ;\n"+ - "EQ : '=' ;\n" + - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String input = - "{ }"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("{}\n" + - "Decision 0:\n" + - "s0-'}'->:s1=>2\n", found); - input = - "{a b { }"; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("{ab{}\n" + - "Decision 0:\n" + - "s0-'{'->:s1=>1\n" + - "s0-'}'->:s2=>2\n" + - "s0-ID->:s1=>1\n", found); - input = - "{ } a 2 { }"; // FAILS to match since it terminates loop at first { } - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("", found); // should not print output; resync kills rest of input til '}' then returns normally - } - - @Test public void testNongreedyLoop() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "ifstat : 'if' '(' .* ')' block ;\n" + - "block : '{' '}' ;\n"+ - "EQ : '=' ;\n" + - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String input = - "if ( x=34 ) { } ;"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("if(x=34){};\n" + - "Decision 0:\n" + - "s0-')'->s2\n" + - "s0-'='->:s1=>1\n" + - "s0-INT->:s1=>1\n" + - "s0-ID->:s1=>1\n" + - "s2-'{'->s3\n" + - "s3-'}'->:s4=>2\n", found); - input = - "if ( ))) ) { } ;"; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("if()))){};\n" + - "Decision 0:\n" + - "s0-')'->s1\n" + - "s1-'{'->s3\n" + - "s1-')'->:s2=>1\n" + - "s3-'}'->:s4=>2\n", found); - input = - "if (() { } a 2) { } ;"; // The first { } should match block so should stop - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("", found); // should not finish to print output - } - - @Test public void testNongreedyLoopPassingThroughAnotherNongreedy() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "ifstat : 'if' '(' .* ')' block ;\n" + - "block : '{' (block|.)* '}' ;\n"+ - "EQ : '=' ;\n" + - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String input = - "if ( x=34 ) { {return a} b 34 } ;"; - String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("if(x=34){{returna}b34};\n" + - "Decision 0:\n" + - "s0-')'->s2\n" + - "s0-'='->:s1=>1\n" + - "s0-INT->:s1=>1\n" + - "s0-ID->:s1=>1\n" + - "s2-'{'->s3\n" + - "s3-'{'->s4\n" + - "s4-'}'->:s5=>2\n" + - "s4-ID->s4\n" + - "\n" + - "Decision 1:\n" + - "s0-'{'->:s1=>1\n" + - "s0-INT->:s2=>2\n" + - "s0-ID->:s2=>2\n" + - "\n" + - "Decision 2:\n" + - "s0-'{'->:s1=>1\n" + - "s0-'}'->:s3=>2\n" + - "s0-INT->:s2=>1\n" + - "s0-ID->:s2=>1\n", found); - - input = - "if ( ()) ) { {return a} b 34 } ;"; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("if(())){{returna}b34};\n" + - "Decision 0:\n" + - "s0-')'->s2\n" + - "s0-'('->:s1=>1\n" + - "s2-'{'->s4\n" + - "s2-')'->:s3=>1\n" + - "s4-'{'->s5\n" + - "s5-'}'->:s6=>2\n" + - "s5-ID->s5\n" + - "\n" + - "Decision 1:\n" + - "s0-'{'->:s1=>1\n" + - "s0-INT->:s2=>2\n" + - "s0-ID->:s2=>2\n" + - "\n" + - "Decision 2:\n" + - "s0-'{'->:s1=>1\n" + - "s0-'}'->:s3=>2\n" + - "s0-INT->:s2=>1\n" + - "s0-ID->:s2=>1\n", found); - } - - @Test public void testStatLoopNongreedyNotNecessary() throws Exception { - // EOF on end means LL(*) can identify when to stop the loop. - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : stat* ID '=' ID ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "stat : 'if' '(' INT ')' stat\n" + - " | 'return' INT ';'\n" + - " | ID '=' (INT|ID) ';'\n" + - " | block\n" + - " ;\n" + - "block : '{' stat* '}' ;\n"+ - "EQ : '=' ;\n" + - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String input = - "x=1; a=b;"; - String found = null; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("x=1;a=b;\n" + - "Decision 0:\n" + - "s0-ID->s1\n" + - "s1-'='->s2\n" + - "s2-INT->:s3=>1\n" + - "s2-ID->s4\n" + - "s4-';'->s5\n" + - "s5-EOF->:s6=>2\n", found); - input = - "if ( 1 ) { x=3; { return 4; } } return 99; abc=def;"; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("if(1){x=3;{return4;}}return99;abc=def;\n" + - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'return'->:s2=>1\n" + - "s0-ID->s3\n" + - "s3-'='->s4\n" + - "s4-ID->s5\n" + - "s5-';'->s6\n" + - "s6-EOF->:s7=>2\n", found); - input = - "x=1; a=3;"; // FAILS to match since it can't match last element - execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - // can't match EOF to ID '=' '3' ';' - assertEquals("line 1:9 no viable alternative at input ''\n", - this.stderrDuringParse); - - input = - "x=1; a=b; z=3;"; // FAILS to match since it can't match last element - execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("line 1:14 no viable alternative at input ''\n", - this.stderrDuringParse); - // should not finish to print output - } - - @Test public void testStatLoopNongreedyNecessary() throws Exception { - // stops scanning ahead at end of rule s since decision is nongreedy. - // this says: "match statements until we see a=b; assignment; ignore any - // statements that follow." - String grammar = - "grammar T;\n" + - "random : s ;" + // call s so s isn't followed by EOF directly - "s @after {dumpDFA();} : (options {greedy=false;} : stat)* ID '=' ID ';'\n" + - " {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "stat : 'if' '(' INT ')' stat\n" + - " | 'return' INT ';'\n" + - " | ID '=' (INT|ID) ';'\n" + - " | block\n" + - " ;\n" + - "block : '{' stat* '}' ;\n"+ - "EQ : '=' ;\n" + - "INT : '0'..'9'+ ;\n" + - "ID : 'a'..'z'+ ;\n" + - "WS : (' '|'\\n')+ {skip();} ;\n"; - String input = - "x=1; a=b; x=y;"; - String found = null; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("x=1;a=b;\n" + - "Decision 0:\n" + - "s0-ID->s1\n" + - "s1-'='->s2\n" + - "s2-INT->:s3=>1\n" + - "s2-ID->s4\n" + - "s4-';'->:s5=>2\n", found); // ignores x=1 that follows first a=b assignment - input = - "if ( 1 ) { x=3; { return 4; } } return 99; abc=def;"; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("if(1){x=3;{return4;}}return99;abc=def;\n" + - "Decision 0:\n" + - "s0-'if'->:s1=>1\n" + - "s0-'return'->:s2=>1\n" + - "s0-ID->s3\n" + - "s3-'='->s4\n" + - "s4-ID->s5\n" + - "s5-';'->:s6=>2\n", found); - input = - "x=1; a=3;"; // FAILS to match since it can't match either stat - execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - // can't match EOF to ID '=' '0' ';' - assertEquals("line 1:9 no viable alternative at input ''\n", - this.stderrDuringParse); - input = - "x=1; a=b; z=3;"; // stops at a=b; ignores z=3; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - input, true); - assertEquals("x=1;a=b;\n" + - "Decision 0:\n" + - "s0-ID->s1\n" + - "s1-'='->s2\n" + - "s2-INT->:s3=>1\n" + - "s2-ID->s4\n" + - "s4-';'->:s5=>2\n", found); // should not finish all input - } - - @Test public void testHTMLTags() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : (item)+ {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" + - "item : tag | . ;\n" + - "tag : '<' '/'? .* '>' ;\n" + - "EQ : '=' ;\n" + - "COMMA : ',' ;\n" + - "ID : 'a'..'z'+ ;\n" + - "STR : '\"' .* '\"' ;\n" + - "INT : '0'..'9'+;\n" + - "WS : (' '|'\\n') {skip();} ;\n"; - - String found = null; - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "foo", true); - assertEquals("foo\n" + - "Decision 1:\n" + - "s0-'<'->s1\n" + - "s0-ID->:s5=>2\n" + - "s1-'/'->s2\n" + - "s1-ID->s2\n" + - "s2-'>'->s3\n" + - "s2-ID->s2\n" + - "s3-EOF->s6^\n" + - "s3-'<'->s4^\n" + - "s3-ID->s3\n" + - "\n" + - "Decision 2:\n" + - "s0-'/'->:s2=>1\n" + - "s0-ID->:s1=>2\n" + - "\n" + - "Decision 3:\n" + - "s0-'>'->:s2=>2\n" + - "s0-ID->:s1=>1\n", found); - assertEquals("line 1:6 reportAttemptingFullContext d=1, input='foo<'\n" + - "line 1:6 reportAmbiguity d=1: ambigAlts={1..2}, input='foo<'\n" + - "line 1:10 reportAttemptingFullContext d=1, input=''\n" + - "line 1:10 reportAmbiguity d=1: ambigAlts={1..2}, input=''\n" + - "line 1:7 reportAmbiguity d=2: ambigAlts={1..2}, input='/'\n", - this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "", true); - assertEquals("\n" + - "Decision 1:\n" + - "s0-'<'->s1\n" + - "s1-'/'->s2\n" + - "s1-ID->s2\n" + - "s2-'>'->s3\n" + - "s2-ID->s2\n" + - "s3-EOF->s5^\n" + - "s3-'<'->s4^\n" + - "\n" + - "Decision 2:\n" + - "s0-'/'->:s2=>1\n" + - "s0-ID->:s1=>2\n" + - "\n" + - "Decision 3:\n" + - "s0-'>'->:s2=>2\n" + - "s0-ID->:s1=>1\n", found); - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "", true); - assertEquals("\n" + - "Decision 1:\n" + - "s0-'<'->s1\n" + - "s1-'/'->s2\n" + - "s1-ID->s2\n" + - "s2-'>'->s3\n" + - "s2-'='->s2\n" + - "s2-','->s2\n" + - "s2-ID->s2\n" + - "s2-STR->s2\n" + - "s2-INT->s2\n" + - "s3-EOF->s5^\n" + - "s3-'<'->s4^\n" + - "\n" + - "Decision 2:\n" + - "s0-'/'->:s1=>1\n" + - "s0-ID->:s2=>2\n" + - "\n" + - "Decision 3:\n" + - "s0-'>'->:s2=>2\n" + - "s0-'='->:s1=>1\n" + - "s0-','->:s1=>1\n" + - "s0-ID->:s1=>1\n" + - "s0-STR->:s1=>1\n" + - "s0-INT->:s1=>1\n", found); - } - - /** lookahead prediction with '.' can be misleading since nongreedy. Lookahead - * that sees into a non-greedy loop, thinks it is greedy. - */ - @Test - public void testFindHTMLTags() throws Exception { - String grammar = - "grammar T;\n" + - "s @after {dumpDFA();} : ( .* (tag {System.out.println($tag.text);} |header) )* EOF;\n" + - "tag : '<' .+ '>' ;\n" + - "header : 'x' 'y' ;\n" + - "EQ : '=' ;\n" + - "COMMA : ',' ;\n" + - "ID : 'a'..'z'+ ;\n" + - "STR : '\"' .* '\"' ;\n" + - "INT : '0'..'9'+;\n" + - "WS : (' '|'\\n') {skip();} ;\n"; - - String found = null; - System.out.println(grammar); - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - ",=foo 32skidoo", true); - assertEquals("\n" + - "\n" + - "\n" + - "Decision 0:\n" + // .* - "s0-'<'->s2\n" + - "s0-'='->:s1=>1\n" + - "s0-','->:s1=>1\n" + - "s0-ID->:s1=>1\n" + - "s0-INT->:s1=>1\n" + - "s2-ID->s3\n" + - "s3-'x'->s4\n" + - "s3-'>'->:s5=>2\n" + - "s3-INT->s3\n" + - "s4-'='->s3\n" + - "\n" + - "Decision 3:\n" + // .+ - "s0-'x'->:s1=>1\n" + - "s0-'>'->:s2=>2\n" + - "s0-'='->:s1=>1\n" + - "s0-ID->:s1=>1\n" + - "s0-INT->:s1=>1\n", found); - assertEquals(null, - this.stderrDuringParse); - - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "x x", true); - assertEquals("\n" + - "Decision 0:\n" + - "s0-'x'->s1\n" + - "s0-'<'->s4\n" + - "s1-'x'->:s2=>1\n" + - "s1-'<'->:s3=>1\n" + - "s4-ID->s5\n" + - "s5-'>'->:s6=>2\n" + - "\n" + - "Decision 3:\n" + - "s0-'>'->:s2=>2\n" + - "s0-ID->:s1=>1\n", found); - // gets line 1:3 no viable alternative at input '>'. Why?? - // oH! it sees .+ and figures it matches > so <> predicts tag CORRECT! - // Seeing '.' in a lookahead prediction can be misleading!! - found = execParser("T.g4", grammar, "TParser", "TLexer", "s", - "x <>", true); - assertEquals("<\n" + - "\n" + - "Decision 0:\n" + - "s0-'x'->s1\n" + - "s0-'>'->:s6=>1\n" + - "s0-'<'->s3\n" + - "s1-'<'->:s2=>1\n" + - "s3-'>'->s4\n" + - "s3-ID->s4\n" + - "s4-'>'->:s7=>2\n" + - "s4-'<'->:s5=>2\n" + - "\n" + - "Decision 3:\n" + - "s0-'>'->:s1=>2\n" + - "s0-ID->:s2=>1\n", // doesn't match tag; null - found); - assertEquals("line 1:3 no viable alternative at input '>'\n", - this.stderrDuringParse); - } -}