From 66da1f723af570079806f7a638c4a09228c32e69 Mon Sep 17 00:00:00 2001 From: parrt Date: Sun, 24 Jul 2011 11:48:43 -0800 Subject: [PATCH] got unit tests working again for interp; got args funcs in output [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8902] --- .../antlr/v4/runtime/ParserRuleContext.java | 4 - .../src/org/antlr/v4/runtime/Recognizer.java | 8 +- .../src/org/antlr/v4/runtime/RuleContext.java | 13 --- .../org/antlr/v4/runtime/atn/ATNConfig.java | 8 +- .../v4/runtime/atn/LexerATNSimulator.java | 6 +- .../v4/runtime/atn/ParserATNSimulator.java | 9 +- .../v4/runtime/atn/PredicateTransition.java | 2 +- tool/playground/T.g | 70 +++++++++++---- tool/playground/TestT.java | 4 +- .../v4/tool/templates/LeftRecursiveRules.stg | 2 +- .../v4/tool/templates/codegen/Java/Java.stg | 78 ++++++++--------- .../v4/codegen/OutputModelController.java | 86 ++++++++++++++----- .../org/antlr/v4/codegen/model/ArgAction.java | 42 +++++++++ .../src/org/antlr/v4/codegen/model/Lexer.java | 8 +- .../org/antlr/v4/codegen/model/Parser.java | 12 +-- .../v4/codegen/model/RuleArgFunction.java | 39 +++++++++ tool/src/org/antlr/v4/parse/ANTLRParser.g | 45 +++++++++- .../org/antlr/v4/parse/GrammarTreeVisitor.g | 6 +- .../v4/parse/LeftRecursiveRuleAnalyzer.java | 18 ++-- .../antlr/v4/parse/LeftRecursiveRuleWalker.g | 39 ++++++--- .../v4/semantics/BasicSemanticChecks.java | 28 +++--- .../antlr/v4/semantics/SemanticPipeline.java | 12 +-- .../antlr/v4/semantics/SymbolCollector.java | 5 +- tool/src/org/antlr/v4/tool/DOTGenerator.java | 24 +++--- tool/src/org/antlr/v4/tool/Grammar.java | 36 ++++++-- tool/src/org/antlr/v4/tool/GrammarAST.java | 14 ++- tool/src/org/antlr/v4/tool/LexerGrammar.java | 3 +- tool/src/org/antlr/v4/tool/Rule.java | 12 ++- .../org/antlr/v4/test/TestATNInterpreter.java | 15 ++-- .../v4/test/TestATNParserPrediction.java | 8 ++ .../org/antlr/v4/test/TestLeftRecursion.java | 9 +- 31 files changed, 470 insertions(+), 195 deletions(-) create mode 100644 tool/src/org/antlr/v4/codegen/model/ArgAction.java create mode 100644 tool/src/org/antlr/v4/codegen/model/RuleArgFunction.java diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java index 6a6202062..98a46c118 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java @@ -61,12 +61,8 @@ public class ParserRuleContext extends RuleContext { super(parent, parent!=null ? parent.s : -1, stateNumber); } - @Override public Object getTree() { return tree; } - @Override public ST getTemplate() { return st; } - @Override public Token getStart() { return start; } - @Override public Token getStop() { return stop; } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java index b710205da..e869bfb5f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java @@ -218,11 +218,15 @@ public class Recognizer { // subclass needs to override these if there are sempreds or actions // that the ATN interp needs to execute - public boolean _sempred(RuleContext _localctx, int ruleIndex, int actionIndex) { + public boolean sempred(RuleContext _localctx, int ruleIndex, int actionIndex) { return true; } /** In lexer, both indexes are same; one action per rule. */ - public void _action(RuleContext _localctx, int ruleIndex, int actionIndex) { + public void action(RuleContext _localctx, int ruleIndex, int actionIndex) { + } + + public ParserRuleContext args(RuleContext _localctx, int s, int ruleIndex, int actionIndex) { + return new ParserRuleContext(_localctx, s); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java index 75d605e74..a55d5bc94 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java @@ -29,7 +29,6 @@ package org.antlr.v4.runtime; import org.antlr.v4.runtime.atn.*; -import org.stringtemplate.v4.ST; /** Rules can return start/stop info as well as possible trees and templates. * Each context knows about invoking context and pointer into ATN so we @@ -68,18 +67,6 @@ public class RuleContext { */ protected int cachedHashCode; - /** Return the start token or tree */ - public Object getStart() { return null; } - - /** Return the stop token or tree */ - public Object getStop() { return null; } - - /** Has a value potentially if output=AST; */ - public Object getTree() { return null; } - - /** Has a value potentially if output=template */ - public ST getTemplate() { return null; } - public RuleContext() {} // public RuleContext(RuleContext parent) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java index a531a2427..884449a4a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java @@ -129,10 +129,10 @@ public class ATNConfig { public String toString(Recognizer recog, boolean showAlt) { StringBuffer buf = new StringBuffer(); - if ( state.ruleIndex>0 ) { - if ( recog!=null ) buf.append(recog.getRuleNames()[state.ruleIndex]+":"); - else buf.append(state.ruleIndex+":"); - } +// if ( state.ruleIndex>=0 ) { +// if ( recog!=null ) buf.append(recog.getRuleNames()[state.ruleIndex]+":"); +// else buf.append(state.ruleIndex+":"); +// } buf.append(state); if ( showAlt ) { buf.append("|"); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 830dad9fb..95c6f2ad8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -148,7 +148,7 @@ public class LexerATNSimulator extends ATNSimulator { recog.getRuleNames()[prevAcceptState.ruleIndex]+ ":"+ actionIndex); } - if ( actionIndex>=0 ) recog._action(null, prevAcceptState.ruleIndex, actionIndex); + if ( actionIndex>=0 ) recog.action(null, prevAcceptState.ruleIndex, actionIndex); } input.seek(prevAcceptMarker); return prevAcceptState.prediction; @@ -237,7 +237,7 @@ public class LexerATNSimulator extends ATNSimulator { else System.out.println("ACTION "+ruleIndex+":"+ruleIndex); } int actionIndex = atn.ruleToActionIndex[ruleIndex]; - if ( actionIndex>=0 ) recog._action(null, ruleIndex, actionIndex); + if ( actionIndex>=0 ) recog.action(null, ruleIndex, actionIndex); return ttype; } @@ -338,7 +338,7 @@ public class LexerATNSimulator extends ATNSimulator { } else if ( t.getClass() == PredicateTransition.class ) { PredicateTransition pt = (PredicateTransition)t; - if ( recog._sempred(null, pt.ruleIndex, pt.predIndex) ) { + if ( recog.sempred(null, pt.ruleIndex, pt.predIndex) ) { c = new ATNConfig(config, t.target); c.traversedPredicate = true; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 618f41b6d..a67b06942 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -37,7 +37,7 @@ import org.stringtemplate.v4.misc.MultiMap; import java.util.*; public class ParserATNSimulator extends ATNSimulator { - public static boolean debug = false; + public static boolean debug = true; public static boolean dfa_debug = false; public static int ATN_failover = 0; @@ -465,7 +465,10 @@ public class ParserATNSimulator extends ATNSimulator { ATNState p = config.state; RuleContext newContext = new RuleContext(config.context, p.stateNumber, t.target.stateNumber); +// RuleContext xx = parser.args(config.context, t.target.stateNumber, t.target.ruleIndex, -999); +// xx.invokingState = p.stateNumber; c = new ATNConfig(config, t.target, newContext); +// c = new ATNConfig(config, t.target, xx); } else if ( t instanceof PredicateTransition ) { PredicateTransition pt = (PredicateTransition)t; @@ -473,7 +476,7 @@ public class ParserATNSimulator extends ATNSimulator { // preds are epsilon if we're not doing preds. // if we are doing preds, pred must eval to true if ( !evalPreds || - (evalPreds && parser._sempred(originalContext, pt.ruleIndex, pt.predIndex)) ) { + (evalPreds && parser.sempred(originalContext, pt.ruleIndex, pt.predIndex)) ) { c = new ATNConfig(config, t.target); c.traversedPredicate = true; } @@ -484,7 +487,7 @@ public class ParserATNSimulator extends ATNSimulator { if ( debug ) System.out.println("ACTION edge "+at.ruleIndex+":"+at.actionIndex); if ( at.actionIndex>=0 ) { if ( debug ) System.out.println("DO ACTION "+at.ruleIndex+":"+at.actionIndex); - parser._action(originalContext, at.ruleIndex, at.actionIndex); + parser.action(originalContext, at.ruleIndex, at.actionIndex); } else { // non-forced action traversed to get to t.target diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java index df2bfb952..40767e7a5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java @@ -52,7 +52,7 @@ public class PredicateTransition extends Transition { public boolean isEpsilon() { return true; } public String toString() { - return "pred-"+ruleIndex+":"+predIndex; + return "pred_"+ruleIndex+":"+predIndex; } } diff --git a/tool/playground/T.g b/tool/playground/T.g index 801449473..2329cc431 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,18 +1,56 @@ grammar T; -//options {output=AST;} - -a : a PLUS a - | INT - ; - -/* -a : a_[0] ; -a_[int _p] : a_primary ( {$_p <= 2}? PLUS a{} )* +options {output=AST;} +s : e EOF ; +expressionList + : e (','! e)* ; -a_primary : INT ; -*/ -/* -a : a_[0] ; -a_[int _p] : a_primary ( {_p <= 2}? B )* ; -a_primary : A ; -*/ +e : '('! e ')'! + | 'this' + | 'super' + | INT + | ID + | type '.'^ 'class' + | e '.'^ ID + | e '.'^ 'this' + | e '.'^ 'super' '('^ expressionList? ')'! + | e '.'^ 'new'^ ID '('! expressionList? ')'! + | 'new'^ type ( '(' expressionList? ')'! | ('[' e ']'!)+) + | e '['^ e ']'! + | '('^ type ')'! e + | e ('++'^ | '--'^) + | e '('^ expressionList? ')'! + | ('+'^|'-'^|'++'^|'--'^) e + | ('~'^|'!'^) e + | e ('*'^|'/'^|'%'^) e + | e ('+'^|'-'^) e + | e ('<'^ '<' | '>'^ '>' '>' | '>'^ '>') e + | e ('<='^ | '>='^ | '>'^ | '<'^) e + | e 'instanceof'^ e + | e ('=='^ | '!='^) e + | e '&'^ e + | e '^'^ e + | e '|'^ e + | e '&&'^ e + | e '||'^ e + | e '?' e ':' e + | e ('='^ + |'+='^ + |'-='^ + |'*='^ + |'/='^ + |'&='^ + |'|='^ + |'^='^ + |'>>='^ + |'>>>='^ + |'<<='^ + |'%='^) e + ; +type: ID + | ID '['^ ']'! + | 'int' + | 'int' '['^ ']'! + ; +ID : ('a'..'z'|'A'..'Z'|'_'|'$')+; +INT : '0'..'9'+ ; +WS : (' '|'\n') {skip();} ; diff --git a/tool/playground/TestT.java b/tool/playground/TestT.java index 94301b49b..0a200aace 100644 --- a/tool/playground/TestT.java +++ b/tool/playground/TestT.java @@ -13,7 +13,7 @@ public class TestT { TLexer t = new TLexer(new ANTLRFileStream(args[0])); CommonTokenStream tokens = new CommonTokenStream(t); TParser p = new TParser(tokens); - ParserRuleContext ret = p.a(); + ParserRuleContext ret = p.s(); System.out.println(((Tree)ret.tree).toStringTree()); } @@ -47,7 +47,7 @@ public class TestT { ATN atn = f.createATN(); DOTGenerator dot = new DOTGenerator(g); - System.out.println(dot.getDOT(atn.ruleToStartState[g.getRule("d").index])); + System.out.println(dot.getDOT(atn.ruleToStartState[g.getRule("e_").index])); } public static class IntTokenStream implements TokenStream { diff --git a/tool/resources/org/antlr/v4/tool/templates/LeftRecursiveRules.stg b/tool/resources/org/antlr/v4/tool/templates/LeftRecursiveRules.stg index 719214bb0..811674edd 100644 --- a/tool/resources/org/antlr/v4/tool/templates/LeftRecursiveRules.stg +++ b/tool/resources/org/antlr/v4/tool/templates/LeftRecursiveRules.stg @@ -62,7 +62,7 @@ recRule(ruleName, precArgDef, argName, alts, setResultAction, buildAST, } ( - + () )* ; >> diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index c689dc2a6..3dc08531b 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -35,7 +35,7 @@ import java.util.ArrayList; >> -Parser(parser, scopes, funcs, atn, actionFuncs, sempredFuncs) ::= << +Parser(parser, scopes, funcs, atn, argFuncs, actionFuncs, sempredFuncs) ::= << public class extends Parser { public static final int =}; separator=", ", wrap, anchor>; @@ -58,30 +58,40 @@ public class extends Parser { @Override public ATN getATN() { return _ATN; } - + } >> -dumpActions(recog, actionFuncs, sempredFuncs) ::= << - - public void _action(RuleContext _localctx, int ruleIndex, int predIndex) { +dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= << + + public ParserRuleContext args(RuleContext _localctx, int s, int ruleIndex, int actionIndex) { switch ( ruleIndex ) { - : _action(()_localctx, predIndex);}; separator="\n"> + : return _arg(()_localctx, s, actionIndex);}; separator="\n"> + } + return new ParserRuleContext(_localctx, s); + } + + + + public void action(RuleContext _localctx, int ruleIndex, int actionIndex) { + switch ( ruleIndex ) { + : _action(()_localctx, actionIndex);}; separator="\n"> } } - + - public boolean _sempred(RuleContext _localctx, int ruleIndex, int predIndex) { + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch ( ruleIndex ) { - : return _sempred(()_localctx, predIndex);}; separator="\n"> } return true; } - + >> @@ -92,8 +102,19 @@ public (TokenStream input) { } >> +RuleArgFunction(r, actions) ::= << +/** arg computations for rules called FROM and evaluated in context of */ +public ParserRuleContext _arg(ParserRuleContext _localctx, int s, int actionIndex) { + switch ( actionIndex ) { + : return new (_localctx, s, );}; separator="\n"> + } + return null; +} +>> + RuleActionFunction(r, actions) ::= << -public void _action( _localctx, int actionIndex) { +public void _action( _localctx, int s, int actionIndex) { switch ( actionIndex ) { : break;}; separator="\n"> @@ -330,6 +351,8 @@ Action(a, chunks) ::= "" ForcedAction(a, chunks) ::= "" +ArgAction(a, chunks) ::= "" + SemPred(p, chunks) ::= << if (!()) throw new FailedPredicateException(this, input, "", """!>); >> @@ -575,7 +598,7 @@ public class extends Lexer { - + } >> @@ -588,37 +611,10 @@ public static final ATN _ATN = ATNSimulator.deserialize(_serializedATN.toCharArray()); static { org.antlr.v4.tool.DOTGenerator dot = new org.antlr.v4.tool.DOTGenerator(null); - //System.out.println(dot.getDOT(_ATN.decisionToATNState.get(0))); + //System.out.println(dot.getDOT(_ATN.decisionToState.get(0), rulesNames)); } >> -/* -actionMethod(name, ruleIndex, actions) ::= << -public void _actions(int action) { - System.out.println("exec action "+action); - switch ( action ) { - : - - break; - }> - } -}<\n> ->> - -sempredMethod(name, ruleIndex, preds) ::= << -public boolean _sempreds(int pred) { - switch ( pred ) { - : - return

; - }> - default : return false; - } -}<\n> ->> -*/ - /** Using a type to init value map, try to init a type; if not in table * must be an object, default value is "null". */ diff --git a/tool/src/org/antlr/v4/codegen/OutputModelController.java b/tool/src/org/antlr/v4/codegen/OutputModelController.java index 9d7b9e46e..4e940f2f6 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelController.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelController.java @@ -96,23 +96,7 @@ public class OutputModelController { Grammar g = delegate.getGrammar(); for (Rule r : g.rules.values()) { - String ctxType = gen.target.getRuleFunctionContextStructName(r); - for (ActionAST a : r.actions) { - if ( a instanceof PredAST ) { - PredAST p = (PredAST)a; - RuleSempredFunction rsf = new RuleSempredFunction(delegate, r, ctxType); - file.lexer.sempredFuncs.add(rsf); - rsf.actions.put(g.sempreds.get(p), new Action(delegate, p)); - } - else if ( a.getType()==ANTLRParser.ACTION || - a.getType()==ANTLRParser.FORCED_ACTION ) - { - // lexer sees {{...}} and {..} as same; neither are done until accept - RuleActionFunction raf = new RuleActionFunction(delegate, r, ctxType); - file.lexer.actionFuncs.add(raf); - raf.actions.put(g.actions.get(a), new ForcedAction(delegate, a)); - } - } + buildLexerRuleActions(file.lexer, r); } return file; @@ -168,21 +152,81 @@ public class OutputModelController { for (ActionAST a : r.actions) { if ( a instanceof PredAST ) { PredAST p = (PredAST)a; - RuleSempredFunction rsf = new RuleSempredFunction(delegate, r, function.ctxType); - parser.sempredFuncs.add(rsf); + RuleSempredFunction rsf = parser.sempredFuncs.get(r); + if ( rsf==null ) { + rsf = new RuleSempredFunction(delegate, r, function.ctxType); + parser.sempredFuncs.put(r, rsf); + } rsf.actions.put(g.sempreds.get(p), new Action(delegate, p)); } else if ( a.getType()==ANTLRParser.FORCED_ACTION ) { - RuleActionFunction raf = new RuleActionFunction(delegate, r, function.ctxType); - parser.actionFuncs.add(raf); + RuleActionFunction raf = parser.actionFuncs.get(r); + if ( raf==null ) { + raf = new RuleActionFunction(delegate, r, function.ctxType); + parser.actionFuncs.put(r, raf); + } raf.actions.put(g.actions.get(a), new ForcedAction(delegate, a)); } + else if ( a.getType()==ANTLRParser.ARG_ACTION ) { + RuleArgFunction raf = parser.argFuncs.get(r); + if ( raf==null ) { + raf = new RuleArgFunction(delegate, r, function.ctxType); + parser.argFuncs.put(r, raf); + } + GrammarAST callNode = (GrammarAST)a.getParent(); + String invokedRuleName = callNode.getText(); + Rule invokedRule = g.getRule(invokedRuleName); + String invokedCtxType = gen.target.getRuleFunctionContextStructName(invokedRule); + raf.actions.put(g.actions.get(a), new ArgAction(delegate, a, invokedCtxType)); + } } if ( function.ruleCtx.isEmpty() ) function.ruleCtx = null; popCurrentRule(); } + public void buildLexerRuleActions(Lexer lexer, Rule r) { + CodeGenerator gen = delegate.getGenerator(); + Grammar g = delegate.getGrammar(); + String ctxType = gen.target.getRuleFunctionContextStructName(r); + for (ActionAST a : r.actions) { + if ( a instanceof PredAST ) { + PredAST p = (PredAST)a; + RuleSempredFunction rsf = lexer.sempredFuncs.get(r); + if ( rsf==null ) { + rsf = new RuleSempredFunction(delegate, r, ctxType); + lexer.sempredFuncs.put(r, rsf); + } + rsf.actions.put(g.sempreds.get(p), new Action(delegate, p)); + } + else if ( a.getType()== ANTLRParser.ACTION || + a.getType()==ANTLRParser.FORCED_ACTION ) + { + RuleActionFunction raf = lexer.sempredFuncs.get(r); + if ( raf==null ) { + raf = new RuleActionFunction(delegate, r, ctxType); + lexer.actionFuncs.put(r, raf); + } + raf.actions.put(g.actions.get(a), new ForcedAction(delegate, a)); + } + + if ( a instanceof PredAST ) { + PredAST p = (PredAST)a; + RuleSempredFunction rsf = new RuleSempredFunction(delegate, r, ctxType); + lexer.sempredFuncs.put(r, rsf); + rsf.actions.put(g.sempreds.get(p), new Action(delegate, p)); + } + else if ( a.getType()==ANTLRParser.ACTION || + a.getType()==ANTLRParser.FORCED_ACTION ) + { + // lexer sees {{...}} and {..} as same; neither are done until accept + RuleActionFunction raf = new RuleActionFunction(delegate, r, ctxType); + lexer.actionFuncs.put(r, raf); + raf.actions.put(g.actions.get(a), new ForcedAction(delegate, a)); + } + } + } + public RuleFunction rule(Rule r) { RuleFunction rf = delegate.rule(r); for (CodeGeneratorExtension ext : extensions) rf = ext.rule(rf); diff --git a/tool/src/org/antlr/v4/codegen/model/ArgAction.java b/tool/src/org/antlr/v4/codegen/model/ArgAction.java new file mode 100644 index 000000000..172867a0c --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/ArgAction.java @@ -0,0 +1,42 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model; + +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.tool.GrammarAST; + +public class ArgAction extends Action { + /** Context type of invoked rule */ + public String ctxType; + public ArgAction(OutputModelFactory factory, GrammarAST ast, String ctxType) { + super(factory, ast); + this.ctxType = ctxType; + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/Lexer.java b/tool/src/org/antlr/v4/codegen/model/Lexer.java index c30f69d0f..9ea1f8c4d 100644 --- a/tool/src/org/antlr/v4/codegen/model/Lexer.java +++ b/tool/src/org/antlr/v4/codegen/model/Lexer.java @@ -43,10 +43,10 @@ public class Lexer extends OutputModelObject { public Collection modes; @ModelElement public SerializedATN atn; - @ModelElement public List actionFuncs = - new ArrayList(); - @ModelElement public List sempredFuncs = - new ArrayList(); + @ModelElement public LinkedHashMap actionFuncs = + new LinkedHashMap(); + @ModelElement public LinkedHashMap sempredFuncs = + new LinkedHashMap(); public Lexer(OutputModelFactory factory, LexerFile file) { this.factory = factory; diff --git a/tool/src/org/antlr/v4/codegen/model/Parser.java b/tool/src/org/antlr/v4/codegen/model/Parser.java index c46384484..4b53e71d4 100644 --- a/tool/src/org/antlr/v4/codegen/model/Parser.java +++ b/tool/src/org/antlr/v4/codegen/model/Parser.java @@ -30,7 +30,7 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.*; import java.util.*; @@ -44,10 +44,12 @@ public class Parser extends OutputModelObject { @ModelElement public List funcs = new ArrayList(); @ModelElement public SerializedATN atn; - @ModelElement public List actionFuncs = - new ArrayList(); - @ModelElement public List sempredFuncs = - new ArrayList(); + @ModelElement public LinkedHashMap argFuncs = + new LinkedHashMap(); + @ModelElement public LinkedHashMap actionFuncs = + new LinkedHashMap(); + @ModelElement public LinkedHashMap sempredFuncs = + new LinkedHashMap(); public Parser(OutputModelFactory factory, ParserFile file) { this.factory = factory; diff --git a/tool/src/org/antlr/v4/codegen/model/RuleArgFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleArgFunction.java new file mode 100644 index 000000000..360ee794f --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/RuleArgFunction.java @@ -0,0 +1,39 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.codegen.model; + +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.tool.Rule; + +public class RuleArgFunction extends RuleActionFunction { + public RuleArgFunction(OutputModelFactory factory, Rule r, String ctxType) { + super(factory, r, ctxType); + } +} diff --git a/tool/src/org/antlr/v4/parse/ANTLRParser.g b/tool/src/org/antlr/v4/parse/ANTLRParser.g index dc2c5dda1..bdb6cdf79 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRParser.g +++ b/tool/src/org/antlr/v4/parse/ANTLRParser.g @@ -146,6 +146,12 @@ boolean buildAST; // the method call from whence to obtain the AST for the parse. // grammarSpec +@after { +GrammarAST options = (GrammarAST)$tree.getFirstChildWithType(ANTLRParser.OPTIONS); +if ( options!=null ) { + Grammar.setNodeOptions($tree, options); +} +} : // The grammar itself can have a documenation comment, which is the // first terminal in the file. @@ -375,7 +381,13 @@ sync // grammar type, such as using returns in lexer rules and so on. rule @init { paraphrases.push("matching a rule"); } -@after { paraphrases.pop(); } +@after { + paraphrases.pop(); + GrammarAST options = (GrammarAST)$tree.getFirstChildWithType(ANTLRParser.OPTIONS); + if ( options!=null ) { + Grammar.setNodeOptions($tree, options); + } +} : // A rule may start with an optional documentation comment DOC_COMMENT? @@ -690,7 +702,16 @@ ebnfSuffix | PLUS -> POSITIVE_CLOSURE[$start] ; -atom: // Qualified reference delegate.rule. This must be +atom +@after { + if ( $tree.getType()==DOT ) { + GrammarAST options = (GrammarAST)$tree.getFirstChildWithType(ANTLRParser.OPTIONS); + if ( options!=null ) { + Grammar.setNodeOptions($tree, options); + } + } +} + : // Qualified reference delegate.rule. This must be // lexically contiguous (no spaces either side of the DOT) // otherwise it is two references with a wildcard in between // and not a qualified reference. @@ -756,6 +777,12 @@ setElement // of options, which apply only to that block. // block +@after { +GrammarAST options = (GrammarAST)$tree.getFirstChildWithType(ANTLRParser.OPTIONS); +if ( options!=null ) { + Grammar.setNodeOptions($tree, options); +} +} : LPAREN ( optionsSpec? ra+=ruleAction* COLON )? altList @@ -791,6 +818,12 @@ range ; terminal +@after { +GrammarAST options = (GrammarAST)$tree.getFirstChildWithType(ANTLRParser.ELEMENT_OPTIONS); +if ( options!=null ) { + Grammar.setNodeOptions($tree, options); +} +} : TOKEN_REF elementOptions? -> ^(TOKEN_REF elementOptions?) | STRING_LITERAL elementOptions? -> ^(STRING_LITERAL elementOptions?) ; @@ -798,7 +831,7 @@ terminal // Terminals may be adorned with certain options when // reference in the grammar: TOK<,,,> elementOptions - : LT elementOption (COMMA elementOption)* GT -> ^(ELEMENT_OPTIONS elementOption+) + : LT elementOption (COMMA elementOption)* GT -> ^(ELEMENT_OPTIONS[$LT,"ELEMENT_OPTIONS"] elementOption+) ; // WHen used with elements we can specify what the tree node type can @@ -860,6 +893,12 @@ rewriteTreeElement ; rewriteTreeAtom +@after { +GrammarAST options = (GrammarAST)$tree.getFirstChildWithType(ANTLRParser.OPTIONS); +if ( options!=null ) { + Grammar.setNodeOptions($tree, options); +} +} : TOKEN_REF elementOptions? ARG_ACTION? -> ^(TOKEN_REF elementOptions? ARG_ACTION?) // for imaginary nodes | RULE_REF | STRING_LITERAL elementOptions? -> ^(STRING_LITERAL elementOptions?) diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index bff0f1be6..172219d88 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -413,7 +413,11 @@ block ; ruleref - : ^(RULE_REF ARG_ACTION?) {ruleRef($RULE_REF, $ARG_ACTION);} + : ^(RULE_REF arg=ARG_ACTION?) + { + ruleRef($RULE_REF, $ARG_ACTION); + if ( $arg!=null ) actionInAlt((ActionAST)$arg); + } ; range diff --git a/tool/src/org/antlr/v4/parse/LeftRecursiveRuleAnalyzer.java b/tool/src/org/antlr/v4/parse/LeftRecursiveRuleAnalyzer.java index 2056b6f33..fb05d185f 100644 --- a/tool/src/org/antlr/v4/parse/LeftRecursiveRuleAnalyzer.java +++ b/tool/src/org/antlr/v4/parse/LeftRecursiveRuleAnalyzer.java @@ -90,14 +90,16 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { @Override public void setTokenPrec(GrammarAST t, int alt) { ASSOC assoc = ASSOC.left; - if ( ((TerminalAST)t).getOptions()!=null ) { - String a = ((TerminalAST)t).getOption("assoc"); - if ( a!=null ) { - if ( a.equals(ASSOC.right.toString()) ) { - assoc = ASSOC.right; - } - else { - tool.errMgr.toolError(ErrorType.ILLEGAL_OPTION_VALUE, "assoc", assoc); + if ( t instanceof GrammarASTWithOptions ) { + if ( ((GrammarASTWithOptions)t).getOptions()!=null ) { + String a = ((GrammarASTWithOptions)t).getOption("assoc"); + if ( a!=null ) { + if ( a.equals(ASSOC.right.toString()) ) { + assoc = ASSOC.right; + } + else { + tool.errMgr.toolError(ErrorType.ILLEGAL_OPTION_VALUE, "assoc", assoc); + } } } } diff --git a/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g b/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g index 8cc2441a4..4c635ce2b 100644 --- a/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g +++ b/tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g @@ -80,6 +80,18 @@ rec_rule returns [boolean isLeftRec] // {if ($ruleBlock.isLeftRec) $r.setType(PREC_RULE);} ; +exceptionGroup + : exceptionHandler* finallyClause? + ; + +exceptionHandler + : ^(CATCH ARG_ACTION ACTION) + ; + +finallyClause + : ^(FINALLY ACTION) + ; + ruleModifier : PUBLIC | PRIVATE @@ -150,21 +162,21 @@ token returns [GrammarAST t=null] | ^(PLUS_ASSIGN ID s=token {$t = $s.t;}) | ^(ROOT s=token {$t = $s.t;}) | ^(BANG s=token {$t = $s.t;}) - | b=STRING_LITERAL {$t = $b;} - | c=TOKEN_REF {$t = $c;} + | b=STRING_LITERAL {$t = $b;} + | ^(b=STRING_LITERAL elementOptions) {$t = $b;} + | ^(c=TOKEN_REF elementOptions) {$t = $c;} + | c=TOKEN_REF {$t = $c;} ; -exceptionGroup - : exceptionHandler* finallyClause? +elementOptions + : ^(ELEMENT_OPTIONS elementOption+) ; -exceptionHandler - : ^(CATCH ARG_ACTION ACTION) - ; - -finallyClause - : ^(FINALLY ACTION) - ; +elementOption + : ID + | ^(ASSIGN ID ID) + | ^(ASSIGN ID STRING_LITERAL) + ; element : ^(ROOT element) @@ -210,8 +222,11 @@ tree_ atom : ^(RULE_REF ARG_ACTION?) - | ^(TOKEN_REF ARG_ACTION?) + | ^(STRING_LITERAL elementOptions) | STRING_LITERAL + | ^(TOKEN_REF elementOptions) + | TOKEN_REF + | ^(WILDCARD elementOptions) | WILDCARD | ^(DOT ID element) ; diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index a32d2b68e..97f2812d5 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -196,12 +196,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { checkMode(ID.token); } - @Override - public void grammarOption(GrammarAST ID, String value) { - boolean ok = checkOptions(g.ast, ID.token, value); - if ( ok ) g.ast.setOption(ID.getText(), value); - } - @Override public void discoverRule(RuleAST rule, GrammarAST ID, List modifiers, @@ -217,19 +211,25 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { checkInvalidRuleRef(ref.token); } + @Override + public void grammarOption(GrammarAST ID, String value) { + boolean ok = checkOptions(g.ast, ID.token, value); + //if ( ok ) g.ast.setOption(ID.getText(), value); + } + @Override public void terminalOption(TerminalAST t, GrammarAST ID, GrammarAST value) { String v = null; if ( value!=null ) v = value.getText(); boolean ok = checkTokenOptions(ID, v); - if ( ok ) { - if ( v!=null ) { - t.setOption(ID.getText(), v); - } - else { - t.setOption(TerminalAST.defaultTokenOption, v); - } - } +// if ( ok ) { +// if ( v!=null ) { +// t.setOption(ID.getText(), v); +// } +// else { +// t.setOption(TerminalAST.defaultTokenOption, v); +// } +// } } @Override diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java index 91b023353..1dd9c4d7a 100644 --- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java +++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java @@ -160,14 +160,14 @@ public class SemanticPipeline { void assignTokenTypes(Grammar g, List tokensDefs, List tokenIDs, Set strings) { - Grammar G = g.getOutermostGrammar(); // put in root, even if imported + //Grammar G = g.getOutermostGrammar(); // put in root, even if imported // DEFINE tokens { X='x'; } ALIASES for (GrammarAST alias : tokensDefs) { if ( alias.getType()== ANTLRParser.ASSIGN ) { String name = alias.getChild(0).getText(); String lit = alias.getChild(1).getText(); - G.defineTokenAlias(name, lit); + g.defineTokenAlias(name, lit); } } @@ -182,11 +182,11 @@ public class SemanticPipeline { */ // DEFINE TOKEN TYPES FOR TOKEN REFS LIKE ID, INT - for (GrammarAST idAST : tokenIDs) { G.defineTokenName(idAST.getText()); } + for (GrammarAST idAST : tokenIDs) { g.defineTokenName(idAST.getText()); } // DEFINE TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';' - for (String s : strings) { G.defineStringLiteral(s); } - System.out.println("tokens="+G.tokenNameToTypeMap); - System.out.println("strings="+G.stringLiteralToTypeMap); + for (String s : strings) { g.defineStringLiteral(s); } + System.out.println("tokens="+g.tokenNameToTypeMap); + System.out.println("strings="+g.stringLiteralToTypeMap); } } diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index d91389014..9451149d7 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -35,7 +35,10 @@ import org.antlr.v4.tool.*; import java.util.*; /** Collects (create) rules, terminals, strings, actions, scopes etc... from AST - * side-effects: sets resolver field of asts for actions. + * side-effects: sets resolver field of asts for actions and + * defines predicates via definePredicateInAlt(), collects actions and stores + * in alts. + * TODO: remove side-effects! */ public class SymbolCollector extends GrammarTreeVisitor { /** which grammar are we checking */ diff --git a/tool/src/org/antlr/v4/tool/DOTGenerator.java b/tool/src/org/antlr/v4/tool/DOTGenerator.java index a61923637..129a0a892 100644 --- a/tool/src/org/antlr/v4/tool/DOTGenerator.java +++ b/tool/src/org/antlr/v4/tool/DOTGenerator.java @@ -164,7 +164,7 @@ public class DOTGenerator { public String getDOT(ATNState startState) { Set ruleNames = grammar.rules.keySet(); String[] names = new String[ruleNames.size()+1]; - int i = 1; + int i = 0; for (String s : ruleNames) names[i++] = s; return getDOT(startState, names); } @@ -196,17 +196,17 @@ public class DOTGenerator { // special case: if decision point, then line up the alt start states // unless it's an end of block - if ( s instanceof BlockStartState ) { - ST rankST = stlib.getInstanceOf("decision-rank"); - DecisionState alt = (DecisionState)s; - for (int i=0; i + } + } + } + public static Map getStringLiteralAliasesFromLexerRules(GrammarRootAST ast) { GrammarAST combinedRulesRoot = (GrammarAST)ast.getFirstChildWithType(ANTLRParser.RULES); @@ -736,10 +755,15 @@ public class Grammar implements AttributeResolver { } public Set getStringLiterals() { - // TODO: super inefficient way to get these. - SymbolCollector collector = new SymbolCollector(this); - collector.process(ast); // no side-effects; find strings - return collector.strings; + final Set strings = new HashSet(); + GrammarTreeVisitor collector = new GrammarTreeVisitor() { + @Override + public void stringRef(TerminalAST ref, GrammarAST options) { + strings.add(ref.getText()); + } + }; + collector.visitGrammar(ast); + return strings; } diff --git a/tool/src/org/antlr/v4/tool/GrammarAST.java b/tool/src/org/antlr/v4/tool/GrammarAST.java index b44756547..8c356ba3d 100644 --- a/tool/src/org/antlr/v4/tool/GrammarAST.java +++ b/tool/src/org/antlr/v4/tool/GrammarAST.java @@ -133,9 +133,19 @@ public class GrammarAST extends CommonTree { public void setType(int type) { token.setType(type); } +// +// @Override +// public String getText() { +// if ( textOverride!=null ) return textOverride; +// if ( token!=null ) { +// return token.getText(); +// } +// return ""; +// } public void setText(String text) { - textOverride = text; // don't alt tokens as others might see +// textOverride = text; // don't alt tokens as others might see + token.setText(text); // we delete surrounding tree, so ok to alter } // @Override @@ -170,7 +180,7 @@ public class GrammarAST extends CommonTree { int type = adaptor.getType(o); while ( type!=Token.EOF ) { buf.append(" "); - buf.append(o.token.getText()); + buf.append(o.getText()); nodes.consume(); o = (GrammarAST)nodes.LT(1); type = adaptor.getType(o); diff --git a/tool/src/org/antlr/v4/tool/LexerGrammar.java b/tool/src/org/antlr/v4/tool/LexerGrammar.java index ee262ad73..cbcef8b1c 100644 --- a/tool/src/org/antlr/v4/tool/LexerGrammar.java +++ b/tool/src/org/antlr/v4/tool/LexerGrammar.java @@ -41,7 +41,7 @@ public class LexerGrammar extends Grammar { public Grammar implicitLexerOwner; /** DEFAULT_MODE rules are added first due to grammar syntax order */ - public MultiMap modes = new MultiMap(); + public MultiMap modes; public LexerGrammar(Tool tool, GrammarRootAST ast) { super(tool, ast); @@ -62,6 +62,7 @@ public class LexerGrammar extends Grammar { @Override public void defineRule(Rule r) { super.defineRule(r); + if ( modes==null ) modes = new MultiMap(); modes.map(r.mode, r); } } diff --git a/tool/src/org/antlr/v4/tool/Rule.java b/tool/src/org/antlr/v4/tool/Rule.java index a12ea9705..fe8b4f3f0 100644 --- a/tool/src/org/antlr/v4/tool/Rule.java +++ b/tool/src/org/antlr/v4/tool/Rule.java @@ -133,16 +133,22 @@ public class Rule implements AttributeResolver { public void defineActionInAlt(int currentAlt, ActionAST actionAST) { actions.add(actionAST); alt[currentAlt].actions.add(actionAST); - if ( g.isLexer() || actionAST.getType()==ANTLRParser.FORCED_ACTION ) { + if ( g.isLexer() || actionAST.getType()==ANTLRParser.FORCED_ACTION || + actionAST.getType()==ANTLRParser.ARG_ACTION ) + { actionIndex = g.actions.size(); - g.actions.put(actionAST, actionIndex); + if ( g.actions.get(actionAST)==null ) { + g.actions.put(actionAST, actionIndex); + } } } public void definePredicateInAlt(int currentAlt, PredAST predAST) { actions.add(predAST); alt[currentAlt].actions.add(predAST); - g.sempreds.put(predAST, g.sempreds.size()); + if ( g.sempreds.get(predAST)==null ) { + g.sempreds.put(predAST, g.sempreds.size()); + } } public Attribute resolveRetvalOrProperty(String y) { diff --git a/tool/test/org/antlr/v4/test/TestATNInterpreter.java b/tool/test/org/antlr/v4/test/TestATNInterpreter.java index 7d9f00db2..4fbbacc26 100644 --- a/tool/test/org/antlr/v4/test/TestATNInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNInterpreter.java @@ -8,7 +8,11 @@ import org.junit.Test; import java.util.List; -/** */ + + // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH + // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH + // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH + public class TestATNInterpreter extends BaseTest { @Test public void testSimpleNoBlock() throws Exception { LexerGrammar lg = new LexerGrammar( @@ -157,10 +161,10 @@ public class TestATNInterpreter extends BaseTest { "D : 'd' ;\n"); Grammar g = new Grammar( "parser grammar T;\n"+ - "a : (A B | A B | D) C ;"); - checkMatchedAlt(lg, g, "abc", 1); - checkMatchedAlt(lg, g, "abcd", 1); - checkMatchedAlt(lg, g, "dc", 3); + "a : (A B | A B | C) D ;"); + checkMatchedAlt(lg, g, "abd", 1); + checkMatchedAlt(lg, g, "abdc", 1); + checkMatchedAlt(lg, g, "cd", 3); } @Test public void testAmbigAltChooseFirst2() throws Exception { @@ -244,6 +248,7 @@ public class TestATNInterpreter extends BaseTest { ); Grammar g = new Grammar( "parser grammar T;\n"+ + "tokens {A;B;C;LP;RP;INT;}\n" + "a : e B | e C ;\n" + "e : LP e RP\n" + " | INT\n" + diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 190633e54..f73ff2c5f 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -10,6 +10,10 @@ import org.junit.Test; import java.util.List; + // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH + // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH + // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH + public class TestATNParserPrediction extends BaseTest { @Test public void testAorB() throws Exception { LexerGrammar lg = new LexerGrammar( @@ -143,6 +147,7 @@ public class TestATNParserPrediction extends BaseTest { "C : 'c' ;\n"); Grammar g = new Grammar( "parser grammar T;\n"+ + "tokens {A;B;C;}\n" + "a : x B ;\n" + "b : x C ;\n" + "x : A | ;\n"); @@ -255,6 +260,7 @@ public class TestATNParserPrediction extends BaseTest { // AB predicted in both alts of e but in diff contexts. Grammar g = new Grammar( "parser grammar T;\n"+ + "tokens {A;B;C;}\n" + "a : e B ;\n" + "b : e A B ;\n" + "e : A | ;\n"); // TODO: try with three alts @@ -363,6 +369,7 @@ public class TestATNParserPrediction extends BaseTest { ); Grammar g = new Grammar( "parser grammar T;\n"+ + "tokens {A;B;C;LP;RP;INT;}\n" + "a : e B | e C ;\n" + "e : LP e RP\n" + " | INT\n" + @@ -421,6 +428,7 @@ public class TestATNParserPrediction extends BaseTest { ); Grammar g = new Grammar( "parser grammar T;\n"+ + "tokens {A;B;C;LP;RP;INT;}\n" + "a : e A | e A B ;\n" + "e : LP e RP\n" + " | INT\n" + diff --git a/tool/test/org/antlr/v4/test/TestLeftRecursion.java b/tool/test/org/antlr/v4/test/TestLeftRecursion.java index c2f672805..d88803a67 100644 --- a/tool/test/org/antlr/v4/test/TestLeftRecursion.java +++ b/tool/test/org/antlr/v4/test/TestLeftRecursion.java @@ -40,6 +40,7 @@ public class TestLeftRecursion extends BaseTest { String grammar = "grammar T;\n" + "options {output=AST;}\n" + + "s : e EOF ;\n" + // must indicate EOF can follow or 'a' won't match "e : e '*'^ e" + " | e '+'^ e" + " | e '?'^ e ':'! e" + @@ -66,6 +67,7 @@ public class TestLeftRecursion extends BaseTest { String grammar = "grammar T;\n" + "options {output=AST;}\n" + + "s : declarator EOF ;\n" + // must indicate EOF can follow "declarator\n" + " : declarator '['^ e ']'!\n" + " | declarator '['^ ']'!\n" + @@ -97,6 +99,7 @@ public class TestLeftRecursion extends BaseTest { String grammar = "grammar T;\n" + "options {output=AST;}\n" + + "s : declarator EOF ;\n" + // must indicate EOF can follow "declarator\n" + " : declarator '[' e ']' -> ^('[' declarator e)\n" + " | declarator '[' ']' -> ^('[' declarator)\n" + @@ -128,6 +131,7 @@ public class TestLeftRecursion extends BaseTest { String grammar = "grammar T;\n" + "options {output=AST;}\n" + + "s : e EOF ;\n" + // must indicate EOF can follow "e : e '.'^ ID\n" + " | e '.'^ 'this'\n" + " | '-'^ e\n" + @@ -160,6 +164,7 @@ public class TestLeftRecursion extends BaseTest { String grammar = "grammar T;\n" + "options {output=AST;}\n" + + "s : e EOF ;\n" + // must indicate EOF can follow "e : e '.' ID -> ^('.' e ID)\n" + " | e '.' 'this' -> ^('.' e 'this')\n" + " | '-' e -> ^('-' e)\n" + @@ -191,6 +196,7 @@ public class TestLeftRecursion extends BaseTest { String grammar = "grammar T;\n" + "options {output=AST;}\n" + + "s : e EOF ;\n" + // must indicate EOF can follow "e\n" + " : e '.'^ ID\n" + " | '-'^ e\n" + @@ -232,6 +238,7 @@ public class TestLeftRecursion extends BaseTest { String grammar = "grammar T;\n" + "options {output=AST;}\n" + + "s : e EOF ;\n" + // must indicate EOF can follow "expressionList\n" + " : e (','! e)*\n" + " ;\n" + @@ -245,7 +252,7 @@ public class TestLeftRecursion extends BaseTest { " | e '.'^ 'this'\n" + " | e '.'^ 'super' '('^ expressionList? ')'!\n" + " | e '.'^ 'new'^ ID '('! expressionList? ')'!\n" + - " | 'new'^ type ( '(' expressionList? ')'! | (options {k=1;}:'[' e ']'!)+)\n" + // ugly; simplified + " | 'new'^ type ( '(' expressionList? ')'! | ('[' e ']'!)+)\n" + " | e '['^ e ']'!\n" + " | '('^ type ')'! e\n" + " | e ('++'^ | '--'^)\n" +