From dc82edad024a2192afa45eed2c1b5a566c5be8bc Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 21 Jan 2012 15:01:05 -0800 Subject: [PATCH] Summary: we can have lexer commands like -> skip now. moved model.actions package to chunk type(foo) didn't match. ATNBuilder.g now tracks the outer alternative number and calls new factory commands on the lexer commands. Removed unnecessary resolveWithPredicate field from ATNConfig Added lexerActionIndex field to ATNConfig since we need to track whether we passed an action in an alternative will rule in the lexer. Renamed ruleIndex in DFAState and added the lexer action index so that we can execute lexer actions from the DFA. added functions to the grammar tree visitor for the lexer commands. Added templates for the lexer commands. Augmented the lexer ATN factory so that it constructs plain old actions from the lexer commands it finds. That way, the code generator doesn't know any different and generates an action. Augmented the lexer ATN simulator so that it fires a proper action index now. previously it only used the rule index, which of course doesn't work when you have more than one action in a rule. rm'd dup code from OutputModelController altered the epsilon edge removal optimization so that it could not remove actions in lexer rules. Added list of valid lexer commands in Rule. [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9877] --- .../src/org/antlr/v4/runtime/Recognizer.java | 1 - .../org/antlr/v4/runtime/atn/ATNConfig.java | 25 ++----- .../v4/runtime/atn/LexerATNSimulator.java | 37 ++++++----- .../v4/runtime/atn/v2ParserATNSimulator.java | 1 - .../org/antlr/v4/runtime/dfa/DFAState.java | 3 +- .../v4/tool/templates/codegen/Java/Java.stg | 17 +++-- .../src/org/antlr/v4/automata/ATNFactory.java | 14 ++-- .../antlr/v4/automata/LexerATNFactory.java | 65 ++++++++++++++++--- .../antlr/v4/automata/ParserATNFactory.java | 36 ++++++++-- .../antlr/v4/codegen/ActionTranslator.java | 2 +- .../v4/codegen/OutputModelController.java | 25 ++----- tool/src/org/antlr/v4/codegen/Target.java | 17 +---- .../org/antlr/v4/codegen/model/Action.java | 15 +++-- .../antlr/v4/codegen/model/InvokeRule.java | 2 +- .../org/antlr/v4/codegen/model/Parser.java | 9 +-- .../v4/codegen/model/RuleActionFunction.java | 1 + .../org/antlr/v4/codegen/model/SemPred.java | 10 ++- .../model/{actions => chunk}/ActionChunk.java | 2 +- .../model/{actions => chunk}/ActionText.java | 2 +- .../model/{actions => chunk}/ArgRef.java | 2 +- .../DefaultParserSuperClass.java | 2 +- .../model/{actions => chunk}/LabelRef.java | 2 +- .../{actions => chunk}/LexerSetAttr.java | 2 +- .../{actions => chunk}/ListLabelRef.java | 2 +- .../model/{actions => chunk}/LocalRef.java | 2 +- .../{actions => chunk}/NonLocalAttrRef.java | 2 +- .../{actions => chunk}/QRetValueRef.java | 2 +- .../model/{actions => chunk}/RetValueRef.java | 2 +- .../{actions => chunk}/RulePropertyRef.java | 2 +- .../RulePropertyRef_ctx.java | 2 +- .../RulePropertyRef_start.java | 2 +- .../RulePropertyRef_stop.java | 2 +- .../RulePropertyRef_text.java | 2 +- .../model/{actions => chunk}/SetAttr.java | 2 +- .../{actions => chunk}/SetNonLocalAttr.java | 2 +- .../ThisRulePropertyRef_ctx.java | 2 +- .../ThisRulePropertyRef_start.java | 2 +- .../ThisRulePropertyRef_stop.java | 2 +- .../ThisRulePropertyRef_text.java | 2 +- .../{actions => chunk}/TokenPropertyRef.java | 2 +- .../TokenPropertyRef_channel.java | 2 +- .../TokenPropertyRef_index.java | 2 +- .../TokenPropertyRef_int.java | 2 +- .../TokenPropertyRef_line.java | 2 +- .../TokenPropertyRef_pos.java | 2 +- .../TokenPropertyRef_text.java | 2 +- .../TokenPropertyRef_type.java | 2 +- .../model/{actions => chunk}/TokenRef.java | 2 +- tool/src/org/antlr/v4/parse/ANTLRParser.g | 2 +- tool/src/org/antlr/v4/parse/ATNBuilder.g | 38 ++++++++++- .../org/antlr/v4/parse/GrammarTreeVisitor.g | 12 ++-- tool/src/org/antlr/v4/tool/Rule.java | 36 +++++++--- .../test/org/antlr/v4/test/TestLexerExec.java | 61 ++++++++++++++++- 53 files changed, 337 insertions(+), 154 deletions(-) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ActionChunk.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ActionText.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ArgRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/DefaultParserSuperClass.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/LabelRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/LexerSetAttr.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ListLabelRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/LocalRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/NonLocalAttrRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/QRetValueRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/RetValueRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/RulePropertyRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/RulePropertyRef_ctx.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/RulePropertyRef_start.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/RulePropertyRef_stop.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/RulePropertyRef_text.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/SetAttr.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/SetNonLocalAttr.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ThisRulePropertyRef_ctx.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ThisRulePropertyRef_start.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ThisRulePropertyRef_stop.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/ThisRulePropertyRef_text.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef_channel.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef_index.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef_int.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef_line.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef_pos.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef_text.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenPropertyRef_type.java (97%) rename tool/src/org/antlr/v4/codegen/model/{actions => chunk}/TokenRef.java (97%) diff --git a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java index 242aa2b46..f3a31bde3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java @@ -129,7 +129,6 @@ public abstract class Recognizer { return true; } - /** In lexer, both indexes are same; one action per rule. */ public void action(@Nullable RuleContext _localctx, int ruleIndex, int actionIndex) { } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java index 3605d2bcc..b75948213 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java @@ -57,18 +57,6 @@ public class ATNConfig { @Nullable public RuleContext context; - /** - * Indicates that we have reached this ATN configuration after - * traversing a predicate transition. This is important because we - * cannot cache DFA states derived from such configurations - * otherwise predicates would not get executed again (DFAs don't - * have predicated edges in v4). - */ - //public boolean traversedPredicate; // TODO: don't need - - /** Ignore this config when examining config sets */ -// public boolean resolved; - /** * We cannot execute predicates dependent upon local context unless * we know for sure we are in the correct context. Because there is @@ -82,18 +70,12 @@ public class ATNConfig { */ public int reachesIntoOuterContext; + /** Capture lexer action we traverse */ + public int lexerActionIndex = -1; // TOOD: move to subclass + @NotNull public SemanticContext semanticContext = SemanticContext.NONE; - /** This bit is used to indicate a semantic predicate will be - * used to resolve the conflict. Essentially, this is used - * as an "ignore" bit so that upon a set of conflicting configurations, - * such as (s|2|p) and (s|3|q), I can set (s|3) to resolved=true (and any - * other configuration associated with alt 3) to make it look like that set - * uniquely predicts an alt. - */ - protected boolean resolveWithPredicate; - public ATNConfig(@NotNull ATNState state, int alt, @Nullable RuleContext context) @@ -134,6 +116,7 @@ public class ATNConfig { this.context = context; this.reachesIntoOuterContext = c.reachesIntoOuterContext; this.semanticContext = semanticContext; + this.lexerActionIndex = c.lexerActionIndex; } // public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 0410e2af9..5490cb2d8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -259,8 +259,10 @@ public class LexerATNSimulator extends ATNSimulator { throw new LexerNoViableAltException(recog, input, startIndex, s.configset); } - int ruleIndex = dfaPrevAccept.state.ruleIndex; - accept(input, ruleIndex, dfaPrevAccept); + int ruleIndex = dfaPrevAccept.state.lexerRuleIndex; + int actionIndex = dfaPrevAccept.state.lexerActionIndex; + accept(input, ruleIndex, actionIndex, + dfaPrevAccept.index, dfaPrevAccept.line, dfaPrevAccept.charPos); tracePredict(dfaPrevAccept.state.prediction); return dfaPrevAccept.state.prediction; } @@ -342,7 +344,8 @@ public class LexerATNSimulator extends ATNSimulator { } int ruleIndex = atnPrevAccept.config.state.ruleIndex; - accept(input, ruleIndex, atnPrevAccept); + accept(input, ruleIndex, atnPrevAccept.config.lexerActionIndex, + atnPrevAccept.index, atnPrevAccept.line, atnPrevAccept.charPos); return atn.ruleToTokenType[ruleIndex]; } @@ -377,19 +380,20 @@ public class LexerATNSimulator extends ATNSimulator { } } - protected void accept(@NotNull CharStream input, int ruleIndex, @NotNull ExecState prevAccept) { + protected void accept(@NotNull CharStream input, int ruleIndex, int actionIndex, + int index, int line, int charPos) + { if ( debug ) { - System.out.format("ACTION %s:%d\n", recog != null ? recog.getRuleNames()[ruleIndex] : ruleIndex, ruleIndex); + System.out.format("ACTION %s:%d\n", recog != null ? recog.getRuleNames()[ruleIndex] : ruleIndex, actionIndex); } - int actionIndex = atn.ruleToActionIndex[ruleIndex]; if ( actionIndex>=0 && recog!=null ) recog.action(null, ruleIndex, actionIndex); // seek to after last char in token - traceSeek(prevAccept.index); - input.seek(prevAccept.index); - line = prevAccept.line; - charPositionInLine = prevAccept.charPos; + traceSeek(index); + input.seek(index); + this.line = line; + this.charPositionInLine = charPos; consume(input); } @@ -530,6 +534,7 @@ public class LexerATNSimulator extends ATNSimulator { // ignore actions; just exec one per rule upon accept else if ( t.getClass() == ActionTransition.class ) { c = new ATNConfig(config, t.target); + c.lexerActionIndex = ((ActionTransition)t).actionIndex; } else if ( t.isEpsilon() ) { c = new ATNConfig(config, t.target); @@ -611,7 +616,7 @@ public class LexerATNSimulator extends ATNSimulator { future. Rather than creating collections of semantic predicates like v3 and testing them on prediction, v4 will test them on the fly all the time using the ATN not the DFA. This is slower but - semantically it's not use that often. One of the key elements to + semantically it's not used that often. One of the key elements to this predicate mechanism is not adding DFA states that see predicates immediately afterwards in the ATN. For example, @@ -639,14 +644,16 @@ public class LexerATNSimulator extends ATNSimulator { { firstConfigWithRuleStopState = c; } - if ( c.semanticContext!=null && c.semanticContext!=SemanticContext.NONE ) traversedPredicate = true; -// if ( c.traversedPredicate ) traversedPredicate = true; + if ( c.semanticContext!=null && c.semanticContext!=SemanticContext.NONE ) { + traversedPredicate = true; + } } if ( firstConfigWithRuleStopState!=null ) { newState.isAcceptState = true; - newState.ruleIndex = firstConfigWithRuleStopState.state.ruleIndex; - newState.prediction = atn.ruleToTokenType[newState.ruleIndex]; + newState.lexerRuleIndex = firstConfigWithRuleStopState.state.ruleIndex; + newState.lexerActionIndex = firstConfigWithRuleStopState.lexerActionIndex; + newState.prediction = atn.ruleToTokenType[newState.lexerRuleIndex]; } if ( traversedPredicate ) return null; // cannot cache diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java index 429c191cc..9068c1c47 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java @@ -759,7 +759,6 @@ public class v2ParserATNSimulator extends ATNSimulator { for (ATNConfig c : configs) { if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) { altToPred[c.alt] = SemanticContext.or(altToPred[c.alt], c.semanticContext); - c.resolveWithPredicate = true; nPredAlts++; } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java index 16107c13c..5248ba330 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java @@ -80,7 +80,8 @@ public class DFAState { public int prediction; // if accept state, what ttype do we match? is "else" clause if predicated - public int ruleIndex; // if accept, exec what action? + public int lexerRuleIndex = -1; // if accept, exec action in what rule? + public int lexerActionIndex = -1; // if accept, exec what action? // todo: rename as unique? public boolean complete; // all alts predict "prediction" diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index b2d6ce3bf..3171b2410 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -434,6 +434,17 @@ setState(); if (!()) throw new FailedPredicateException(this, ); >> +// lexer actions are not associated with model objects + +LexerSkipCommand() ::= "skip();" +LexerMoreCommand() ::= "more();" +LexerPopMode() ::= "popMode();" + +LexerTypeCommand(arg) ::= "type = ;" +LexerChannelCommand(arg) ::= "channel = ;" +LexerModeCommand(arg) ::= "mode = ;" +LexerPushModeCommand(arg) ::= "pushMode();" + DefaultParserSuperClass(s) ::= "Parser" ActionText(t) ::= "" @@ -482,8 +493,7 @@ TokenListDecl(t) ::= "List\ = new ArrayList\();" RuleContextDecl(r) ::= " ;" RuleContextListDecl(rdecl) ::= "List\<> = new ArrayList\<>();" -/** Default RuleContext type name for a Parser rule */ -ParserRuleContext() ::= "ParserRuleContext\" +LexerRuleContext() ::= "RuleContext" /** The rule context name is the rule followed by a suffix; e.g., * r becomes rContext. @@ -538,7 +548,7 @@ AttributeDecl(d) ::= "" /** If we don't know location of label def x, use this template */ labelref(x) ::= "_localctx." -// used for left-recursive rules +// used for left-recursive rules recRuleDefArg() ::= "int _p" recRuleArg() ::= "$_p" recRuleAltPredicate(ruleName,opPrec) ::= " >= " @@ -592,7 +602,6 @@ Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= << public class extends Lexer { public static final int =}; separator=", ", wrap, anchor>; - // Lexer modes = ;}; separator="\n"> public static final String[] tokenNames = { diff --git a/tool/src/org/antlr/v4/automata/ATNFactory.java b/tool/src/org/antlr/v4/automata/ATNFactory.java index 975335a5c..3fab41715 100644 --- a/tool/src/org/antlr/v4/automata/ATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ATNFactory.java @@ -58,6 +58,8 @@ public interface ATNFactory { void setCurrentRuleName(String name); + void setCurrentOuterAlt(int alt); + Handle rule(GrammarAST ruleAST, String name, Handle blk); ATNState newState(); @@ -77,8 +79,6 @@ public interface ATNFactory { Handle range(GrammarAST a, GrammarAST b); -// Handle not(GrammarAST a); - /** For a non-lexer, just build a simple token reference atom. * For a lexer, a string is a sequence of char to match. That is, * "fog" is treated as 'f' 'o' 'g' not as a single transition in @@ -116,11 +116,11 @@ public interface ATNFactory { /** Build what amounts to an epsilon transition with an action. * The action goes into ATN though it is ignored during analysis. - * It slows things down a bit, but I must ignore predicates after - * having seen an action (5-5-2008). */ Handle action(ActionAST action); + Handle action(String action); + Handle alt(List els); /** From A|B|..|Z alternative block build @@ -214,4 +214,10 @@ public interface ATNFactory { * (^(. .+) | .) to be safe. */ Handle wildcardTree(GrammarAST associatedAST); + + Handle lexerAltCommands(ATNFactory.Handle alt, ATNFactory.Handle cmds); + + String lexerCallCommand(GrammarAST ID, GrammarAST arg); + + String lexerCommand(GrammarAST ID); } diff --git a/tool/src/org/antlr/v4/automata/LexerATNFactory.java b/tool/src/org/antlr/v4/automata/LexerATNFactory.java index 0534f33b1..40ac8c0e5 100644 --- a/tool/src/org/antlr/v4/automata/LexerATNFactory.java +++ b/tool/src/org/antlr/v4/automata/LexerATNFactory.java @@ -29,23 +29,35 @@ package org.antlr.v4.automata; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; +import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.misc.CharSupport; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; import org.antlr.v4.tool.ast.ActionAST; import org.antlr.v4.tool.ast.GrammarAST; import org.antlr.v4.tool.ast.TerminalAST; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; import java.util.List; import java.util.Set; public class LexerATNFactory extends ParserATNFactory { - public LexerATNFactory(LexerGrammar g) { super(g); } + public STGroup codegenTemplates; + public LexerATNFactory(LexerGrammar g) { + super(g); + // use codegen to get correct language templates for lexer commands + String language = Grammar.getLanguageOption(g.ast); + CodeGenerator gen = new CodeGenerator(g.tool, null, language); + codegenTemplates = gen.templates; + } public ATN createATN() { // BUILD ALL START STATES (ONE PER MODE) @@ -87,13 +99,48 @@ public class LexerATNFactory extends ParserATNFactory { @Override public Handle action(ActionAST action) { -// Handle h = super.action(action); -// ActionTransition a = (ActionTransition)h.left.transition(0); -// a.actionIndex = g.actions.get(action); -// return h; - // no actions in lexer ATN; just one on end and we exec via action number - ATNState x = newState(action); - return new Handle(x, x); // return just one blank state + ATNState left = newState(action); + ATNState right = newState(action); + boolean isCtxDependent = false; + int actionIndex = g.lexerActions.get(action); + ActionTransition a = + new ActionTransition(right, currentRule.index, actionIndex, isCtxDependent); + left.addTransition(a); + action.atnState = left; + Handle h = new Handle(left, right); + return h; + } + + @Override + public Handle action(String action) { + // define action AST for this rule as if we had found in grammar + ActionAST ast = new ActionAST(new CommonToken(ANTLRParser.ACTION, action)); + currentRule.defineActionInAlt(currentOuterAlt, ast); + return action(ast); + } + + @Override + public Handle lexerAltCommands(Handle alt, Handle cmds) { + Handle h = new Handle(alt.left, cmds.right); + epsilon(alt.right, cmds.left); + return h; + } + + @Override + public String lexerCallCommand(GrammarAST ID, GrammarAST arg) { + ST cmdST = codegenTemplates.getInstanceOf("Lexer" + + CharSupport.capitalize(ID.getText())+ + "Command"); + cmdST.add("arg", arg.getText()); + return cmdST.render(); + } + + @Override + public String lexerCommand(GrammarAST ID) { + ST cmdST = codegenTemplates.getInstanceOf("Lexer" + + CharSupport.capitalize(ID.getText())+ + "Command"); + return cmdST.render(); } @Override @@ -126,7 +173,7 @@ public class LexerATNFactory extends ParserATNFactory { } if ( invert ) { // TODO: what? should be chars not token types - IntervalSet notSet = (IntervalSet)set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType()); + IntervalSet notSet = set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType()); left.addTransition(new NotSetTransition(right, set, notSet)); } else { diff --git a/tool/src/org/antlr/v4/automata/ParserATNFactory.java b/tool/src/org/antlr/v4/automata/ParserATNFactory.java index 91e789a9d..b88488965 100644 --- a/tool/src/org/antlr/v4/automata/ParserATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ParserATNFactory.java @@ -70,8 +70,10 @@ public class ParserATNFactory implements ATNFactory { // we have p-x->q for x in {rule, action, pred, token, ...} // if edge out of q is single epsilon to block end // we can strip epsilon p-x->q-eps->r - if ( q.getNumberOfTransitions()==1 && q.transition(0).isEpsilon() ) { - ATNState r = q.transition(0).target; + Transition trans = q.transition(0); + if ( q.getNumberOfTransitions()==1 && trans.isEpsilon() && + !(trans instanceof ActionTransition) ) { + ATNState r = trans.target; if ( r instanceof BlockEndState || r instanceof PlusLoopbackState || r instanceof StarLoopbackState ) @@ -99,6 +101,8 @@ public class ParserATNFactory implements ATNFactory { public Rule currentRule; + public int currentOuterAlt; + public ParserATNFactory(@NotNull Grammar g) { this.g = g; atn = new ATN(); } public ATN createATN() { @@ -120,7 +124,7 @@ public class ParserATNFactory implements ATNFactory { ATNBuilder b = new ATNBuilder(nodes,this); try { setCurrentRuleName(r.name); - Handle h = b.block(null); + Handle h = b.ruleBlock(null); rule(r.ast, r.name, h); } catch (RecognitionException re) { @@ -133,6 +137,11 @@ public class ParserATNFactory implements ATNFactory { this.currentRule = g.getRule(name); } + @Override + public void setCurrentOuterAlt(int alt) { + currentOuterAlt = alt; + } + /* start->ruleblock->end */ public Handle rule(GrammarAST ruleAST, String name, Handle blk) { Rule r = g.getRule(name); @@ -170,7 +179,7 @@ public class ParserATNFactory implements ATNFactory { set.add(ttype); } if ( invert ) { - IntervalSet notSet = (IntervalSet)set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType()); + IntervalSet notSet = set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType()); left.addTransition(new NotSetTransition(right, set, notSet)); } else { @@ -286,6 +295,11 @@ public class ParserATNFactory implements ATNFactory { return new Handle(left, right); } + @Override + public Handle action(String action) { + return null; + } + /** From A|B|..|Z alternative block build * * o->o-A->o->o (last ATNState is BlockEndState pointed to by all alts) @@ -619,4 +633,18 @@ public class ParserATNFactory implements ATNFactory { return false; } + @Override + public Handle lexerAltCommands(Handle alt, Handle cmds) { + return null; + } + + @Override + public String lexerCallCommand(GrammarAST ID, GrammarAST arg) { + return null; + } + + @Override + public String lexerCommand(GrammarAST ID) { + return null; + } } diff --git a/tool/src/org/antlr/v4/codegen/ActionTranslator.java b/tool/src/org/antlr/v4/codegen/ActionTranslator.java index cb52618e1..4bfb2dac2 100644 --- a/tool/src/org/antlr/v4/codegen/ActionTranslator.java +++ b/tool/src/org/antlr/v4/codegen/ActionTranslator.java @@ -32,7 +32,7 @@ package org.antlr.v4.codegen; import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.Token; import org.antlr.v4.codegen.model.RuleFunction; -import org.antlr.v4.codegen.model.actions.*; +import org.antlr.v4.codegen.model.chunk.*; import org.antlr.v4.parse.ActionSplitter; import org.antlr.v4.parse.ActionSplitterListener; import org.antlr.v4.tool.Attribute; diff --git a/tool/src/org/antlr/v4/codegen/OutputModelController.java b/tool/src/org/antlr/v4/codegen/OutputModelController.java index 88733829e..6e0a12248 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelController.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelController.java @@ -269,10 +269,15 @@ public class OutputModelController { function.postamble = rulePostamble(function, r); } - public void buildLexerRuleActions(Lexer lexer, Rule r) { + public void buildLexerRuleActions(Lexer lexer, final Rule r) { CodeGenerator gen = delegate.getGenerator(); Grammar g = delegate.getGrammar(); String ctxType = gen.target.getRuleFunctionContextStructName(r); + RuleActionFunction raf = lexer.actionFuncs.get(r); + if ( raf==null ) { + raf = new RuleActionFunction(delegate, r, ctxType); + lexer.actionFuncs.put(r, raf); + } for (ActionAST a : r.actions) { if ( a instanceof PredAST ) { PredAST p = (PredAST)a; @@ -284,24 +289,6 @@ public class OutputModelController { rsf.actions.put(g.sempreds.get(p), new Action(delegate, p)); } else if ( a.getType()== ANTLRParser.ACTION ) { - RuleActionFunction raf = lexer.sempredFuncs.get(r); - if ( raf==null ) { - raf = new RuleActionFunction(delegate, r, ctxType); - lexer.actionFuncs.put(r, raf); - } - raf.actions.put(g.lexerActions.get(a), new Action(delegate, a)); - } - - if ( a instanceof PredAST ) { - PredAST p = (PredAST)a; - RuleSempredFunction rsf = new RuleSempredFunction(delegate, r, ctxType); - lexer.sempredFuncs.put(r, rsf); - rsf.actions.put(g.sempreds.get(p), new Action(delegate, p)); - } - else if ( a.getType()==ANTLRParser.ACTION ) { - // lexer sees {{...}} and {..} as same; neither are done until accept - RuleActionFunction raf = new RuleActionFunction(delegate, r, ctxType); - lexer.actionFuncs.put(r, raf); raf.actions.put(g.lexerActions.get(a), new Action(delegate, a)); } } diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 1abcf8448..b109ef166 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -339,15 +339,9 @@ public class Target { public String getRuleFunctionContextStructName(Rule r) { if ( r.g.isLexer() ) { - return gen.templates.getInstanceOf("ParserRuleContext").render(); + return gen.templates.getInstanceOf("LexerRuleContext").render(); } return r.name+gen.templates.getInstanceOf("RuleContextNameSuffix").render(); -// boolean hasNoExternallyVisibleElements = -// r.args==null && r.retvals==null && r.scope==null && r.getLabelNames()==null; -// if ( hasNoExternallyVisibleElements ) { -// return gen.templates.getInstanceOf("ParserRuleContext").render(); -// } -// return r.name+"_ctx"; } /** If we know which actual function, we can provide the actual ctx type. @@ -358,16 +352,9 @@ public class Target { public String getRuleFunctionContextStructName(RuleFunction function) { Rule r = function.rule; if ( r.g.isLexer() ) { - return gen.templates.getInstanceOf("ParserRuleContext").render(); + return gen.templates.getInstanceOf("LexerRuleContext").render(); } return r.name+gen.templates.getInstanceOf("RuleContextNameSuffix").render(); -// boolean hasNoExternallyVisibleElements = -// r.args==null && r.retvals==null && r.scope==null && r.getLabelNames()==null; -// -// if ( hasNoExternallyVisibleElements && function.ruleCtx.isEmpty() ) { -// return gen.templates.getInstanceOf("ParserRuleContext").render(); -// } -// return r.name+"_ctx"; } // should be same for all refs to same token like $ID within single rule function diff --git a/tool/src/org/antlr/v4/codegen/model/Action.java b/tool/src/org/antlr/v4/codegen/model/Action.java index c3a80e48e..96a3e53dd 100644 --- a/tool/src/org/antlr/v4/codegen/model/Action.java +++ b/tool/src/org/antlr/v4/codegen/model/Action.java @@ -32,7 +32,8 @@ package org.antlr.v4.codegen.model; import org.antlr.runtime.CommonToken; import org.antlr.v4.codegen.ActionTranslator; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.codegen.model.actions.ActionChunk; +import org.antlr.v4.codegen.model.chunk.ActionChunk; +import org.antlr.v4.codegen.model.chunk.ActionText; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.tool.ast.ActionAST; import org.antlr.v4.tool.ast.GrammarAST; @@ -57,9 +58,15 @@ public class Action extends RuleElement { public Action(OutputModelFactory factory, String action) { super(factory,null); - RuleFunction rf = factory.getCurrentRuleFunction(); ActionAST ast = new ActionAST(new CommonToken(ANTLRParser.ACTION, action)); - ast.resolver = rf.rule; - chunks = ActionTranslator.translateActionChunk(factory, rf, action, ast); + RuleFunction rf = factory.getCurrentRuleFunction(); + if ( rf!=null ) { // we can translate + ast.resolver = rf.rule; + chunks = ActionTranslator.translateActionChunk(factory, rf, action, ast); + } + else { + chunks = new ArrayList(); + chunks.add(new ActionText(action)); + } } } diff --git a/tool/src/org/antlr/v4/codegen/model/InvokeRule.java b/tool/src/org/antlr/v4/codegen/model/InvokeRule.java index c4b622bdb..c358ed585 100644 --- a/tool/src/org/antlr/v4/codegen/model/InvokeRule.java +++ b/tool/src/org/antlr/v4/codegen/model/InvokeRule.java @@ -32,7 +32,7 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.ActionTranslator; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.codegen.model.actions.ActionChunk; +import org.antlr.v4.codegen.model.chunk.ActionChunk; import org.antlr.v4.codegen.model.decl.Decl; import org.antlr.v4.codegen.model.decl.RuleContextDecl; import org.antlr.v4.codegen.model.decl.RuleContextListDecl; diff --git a/tool/src/org/antlr/v4/codegen/model/Parser.java b/tool/src/org/antlr/v4/codegen/model/Parser.java index 6004f1c05..a1ae7d6f6 100644 --- a/tool/src/org/antlr/v4/codegen/model/Parser.java +++ b/tool/src/org/antlr/v4/codegen/model/Parser.java @@ -31,10 +31,11 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.OutputModelFactory; -import org.antlr.v4.codegen.model.actions.ActionChunk; -import org.antlr.v4.codegen.model.actions.ActionText; -import org.antlr.v4.codegen.model.actions.DefaultParserSuperClass; -import org.antlr.v4.tool.*; +import org.antlr.v4.codegen.model.chunk.ActionChunk; +import org.antlr.v4.codegen.model.chunk.ActionText; +import org.antlr.v4.codegen.model.chunk.DefaultParserSuperClass; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.Rule; import java.util.*; diff --git a/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java index a9d9b60d0..a1e266f1e 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java @@ -39,6 +39,7 @@ public class RuleActionFunction extends OutputModelObject { public String ctxType; public int ruleIndex; + /** Map actionIndex to Action */ @ModelElement public LinkedHashMap actions = new LinkedHashMap(); diff --git a/tool/src/org/antlr/v4/codegen/model/SemPred.java b/tool/src/org/antlr/v4/codegen/model/SemPred.java index 41a23f570..2cc131b88 100644 --- a/tool/src/org/antlr/v4/codegen/model/SemPred.java +++ b/tool/src/org/antlr/v4/codegen/model/SemPred.java @@ -29,9 +29,13 @@ package org.antlr.v4.codegen.model; -import org.antlr.v4.codegen.*; -import org.antlr.v4.codegen.model.actions.ActionChunk; -import org.antlr.v4.tool.ast.*; +import org.antlr.v4.codegen.ActionTranslator; +import org.antlr.v4.codegen.CodeGenerator; +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.codegen.model.chunk.ActionChunk; +import org.antlr.v4.tool.ast.ActionAST; +import org.antlr.v4.tool.ast.GrammarAST; +import org.antlr.v4.tool.ast.PredAST; import java.util.List; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ActionChunk.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ActionChunk.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java index cf5aa5c9b..d2d600ab0 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ActionChunk.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; import org.antlr.v4.codegen.model.OutputModelObject; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ActionText.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ActionText.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java index 794007262..edc0d7366 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ActionText.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class ActionText extends ActionChunk { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ArgRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ArgRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java index 13dfa5659..a2f760b03 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ArgRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class ArgRef extends LocalRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/DefaultParserSuperClass.java b/tool/src/org/antlr/v4/codegen/model/chunk/DefaultParserSuperClass.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/DefaultParserSuperClass.java rename to tool/src/org/antlr/v4/codegen/model/chunk/DefaultParserSuperClass.java index 678f97e7c..2e8e9cc0c 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/DefaultParserSuperClass.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/DefaultParserSuperClass.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; public class DefaultParserSuperClass extends ActionChunk { } diff --git a/tool/src/org/antlr/v4/codegen/model/actions/LabelRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/LabelRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java index bce42c5cb..7cba9448e 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/LabelRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; public class LabelRef extends ActionChunk { public String name; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/LexerSetAttr.java b/tool/src/org/antlr/v4/codegen/model/chunk/LexerSetAttr.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/LexerSetAttr.java rename to tool/src/org/antlr/v4/codegen/model/chunk/LexerSetAttr.java index 7228867fd..5271f853e 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/LexerSetAttr.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/LexerSetAttr.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; import java.util.List; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ListLabelRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ListLabelRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java index acf2768a6..e2e0943ae 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ListLabelRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; public class ListLabelRef extends LabelRef { public ListLabelRef(String name) { super(name); } diff --git a/tool/src/org/antlr/v4/codegen/model/actions/LocalRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/LocalRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java index eb5ac5728..5d7edb7e7 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/LocalRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; public class LocalRef extends ActionChunk { public String name; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/NonLocalAttrRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/NonLocalAttrRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java index db3d08390..344c3d8ee 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/NonLocalAttrRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; public class NonLocalAttrRef extends ActionChunk { public String ruleName; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/QRetValueRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/QRetValueRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java index 48b936872..0b88d5435 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/QRetValueRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class QRetValueRef extends RetValueRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/RetValueRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/RetValueRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java index e5c8a2f7f..7886c3239 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/RetValueRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class RetValueRef extends ActionChunk { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java index 52dee08d5..f889fc1b5 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class RulePropertyRef extends ActionChunk { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_ctx.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_ctx.java rename to tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java index b9d1a0e42..3bbfc5b78 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_ctx.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; public class RulePropertyRef_ctx extends RulePropertyRef { public RulePropertyRef_ctx(String label) { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_start.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_start.java rename to tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java index 3c02aa405..e68b3070f 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_start.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class RulePropertyRef_start extends RulePropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_stop.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_stop.java rename to tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java index 8601b3a54..8a102d95c 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_stop.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class RulePropertyRef_stop extends RulePropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_text.java rename to tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java index 8a87d629f..b2fc94b49 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/RulePropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class RulePropertyRef_text extends RulePropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/SetAttr.java b/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/SetAttr.java rename to tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java index 4e2d38621..d63febc10 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/SetAttr.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; import org.antlr.v4.codegen.model.ModelElement; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/SetNonLocalAttr.java b/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/SetNonLocalAttr.java rename to tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java index 4e98415a8..333fadede 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/SetNonLocalAttr.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; import java.util.List; diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_ctx.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_ctx.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java index cea4da867..d6b6da0c8 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_ctx.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; public class ThisRulePropertyRef_ctx extends RulePropertyRef { public ThisRulePropertyRef_ctx(String label) { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_start.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_start.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java index a32be77e3..93aa8429c 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_start.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class ThisRulePropertyRef_start extends RulePropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_stop.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_stop.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java index 0cfe627fc..0503f913f 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_stop.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class ThisRulePropertyRef_stop extends RulePropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_text.java rename to tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java index c92cc3994..14193cc5a 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/ThisRulePropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class ThisRulePropertyRef_text extends RulePropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java index 2ae825e00..f46ee9c04 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef extends ActionChunk { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_channel.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_channel.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java index 4760d7a40..eb2d41c5d 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_channel.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef_channel extends TokenPropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_index.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_index.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java index 3e057fb12..17cce4b29 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_index.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef_index extends TokenPropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_int.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_int.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java index 9954334ac..802d1b3ea 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_int.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef_int extends TokenPropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_line.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_line.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java index 2f8a7436f..69859069d 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_line.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef_line extends TokenPropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_pos.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_pos.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java index a9d32fa70..2ddff8ac1 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_pos.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef_pos extends TokenPropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_text.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java index f95a4a8a9..0b06c9a74 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef_text extends TokenPropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_type.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_type.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java index 4ccced4a6..b6a36ca4b 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenPropertyRef_type.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenPropertyRef_type extends TokenPropertyRef { diff --git a/tool/src/org/antlr/v4/codegen/model/actions/TokenRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java similarity index 97% rename from tool/src/org/antlr/v4/codegen/model/actions/TokenRef.java rename to tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java index 49293b0fd..ad47340c6 100644 --- a/tool/src/org/antlr/v4/codegen/model/actions/TokenRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java @@ -27,7 +27,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.antlr.v4.codegen.model.actions; +package org.antlr.v4.codegen.model.chunk; /** */ public class TokenRef extends ActionChunk { diff --git a/tool/src/org/antlr/v4/parse/ANTLRParser.g b/tool/src/org/antlr/v4/parse/ANTLRParser.g index 61dc27b9d..df401436a 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRParser.g +++ b/tool/src/org/antlr/v4/parse/ANTLRParser.g @@ -648,7 +648,7 @@ lexerAction ; lexerActionExpr - : ID + : id | INT ; diff --git a/tool/src/org/antlr/v4/parse/ATNBuilder.g b/tool/src/org/antlr/v4/parse/ATNBuilder.g index 6aa4b7d00..e1fef64ec 100644 --- a/tool/src/org/antlr/v4/parse/ATNBuilder.g +++ b/tool/src/org/antlr/v4/parse/ATNBuilder.g @@ -75,6 +75,21 @@ import org.antlr.v4.automata.ATNFactory; dummy : block[null] ; // avoid error about no start rule +ruleBlock[GrammarAST ebnfRoot] returns [ATNFactory.Handle p] +@init { + List alts = new ArrayList(); + int alt = 1; + factory.setCurrentOuterAlt(alt); +} + : ^(BLOCK + (^(OPTIONS .+))? + ( a=alternative + {alts.add($a.p); factory.setCurrentOuterAlt(++alt);} + )+ + ) + {$p = factory.block((BlockAST)$BLOCK, ebnfRoot, alts);} + ; + block[GrammarAST ebnfRoot] returns [ATNFactory.Handle p] @init {List alts = new ArrayList();} : ^(BLOCK (^(OPTIONS .+))? (a=alternative {alts.add($a.p);})+) @@ -83,11 +98,32 @@ block[GrammarAST ebnfRoot] returns [ATNFactory.Handle p] alternative returns [ATNFactory.Handle p] @init {List els = new ArrayList();} - : ^(LEXER_ALT_ACTION a=alternative .*) {$p = $a.p;} + : ^(LEXER_ALT_ACTION a=alternative lexerCommands) + {$p = factory.lexerAltCommands($a.p,$lexerCommands.p);} | ^(ALT EPSILON) {$p = factory.epsilon($EPSILON);} | ^(ALT (e=element {els.add($e.p);})+) {$p = factory.alt(els);} ; +lexerCommands returns [ATNFactory.Handle p] +@init {StringBuilder cmds = new StringBuilder();} + : (c=lexerCommand {cmds.append($c.cmd+" ");})+ + { + $p = factory.action(cmds.toString()); + } + ; + +lexerCommand returns [String cmd] + : ^(LEXER_ACTION_CALL ID lexerCommandExpr) + {$cmd = factory.lexerCallCommand($ID, $lexerCommandExpr.start);} + | ID + {$cmd = factory.lexerCommand($ID);} + ; + +lexerCommandExpr + : ID + | INT + ; + element returns [ATNFactory.Handle p] : labeledElement {$p = $labeledElement.p;} | atom {$p = $atom.p;} diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index 3e7e32256..da917eb89 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -153,6 +153,8 @@ public void wildcardRef(GrammarAST ref) { } public void actionInAlt(ActionAST action) { } public void sempredInAlt(PredAST pred) { } public void label(GrammarAST op, GrammarAST ID, GrammarAST element) { } +public void lexerCallCommand(int outerAltNumber, GrammarAST ID, GrammarAST arg) { } +public void lexerCommand(int outerAltNumber, GrammarAST ID) { } public void traceIn(String ruleName, int ruleIndex) { System.err.println("enter "+ruleName+": "+input.LT(1)); @@ -357,7 +359,7 @@ outerAlternative ; lexerAlternative - : ^(LEXER_ALT_ACTION lexerElements lexerAction+) + : ^(LEXER_ALT_ACTION lexerElements lexerCommand+) | lexerElements ; @@ -404,12 +406,14 @@ alternative | ^(ALT EPSILON) ; -lexerAction - : ^(LEXER_ACTION_CALL lexerActionExpr) +lexerCommand + : ^(LEXER_ACTION_CALL ID lexerCommandExpr) + {lexerCallCommand(currentOuterAltNumber, $ID, $lexerCommandExpr.start);} | ID + {lexerCommand(currentOuterAltNumber, $ID);} ; -lexerActionExpr +lexerCommandExpr : ID | INT ; diff --git a/tool/src/org/antlr/v4/tool/Rule.java b/tool/src/org/antlr/v4/tool/Rule.java index 8b06fafd6..d9edeeec6 100644 --- a/tool/src/org/antlr/v4/tool/Rule.java +++ b/tool/src/org/antlr/v4/tool/Rule.java @@ -75,12 +75,25 @@ public class Rule implements AttributeResolver { add(new Attribute("int")); }}; - public String name; + public static Set validLexerCommands = new HashSet() {{ + // CALLS + add("mode"); + add("pushMode"); + add("type"); + add("channel"); + + // ACTIONS + add("popMode"); + add("skip"); + add("more"); + }}; + + public String name; public List modifiers; - public RuleAST ast; - public AttributeDict args; - public AttributeDict retvals; + public RuleAST ast; + public AttributeDict args; + public AttributeDict retvals; public AttributeDict locals; /** In which grammar does this rule live? */ @@ -123,7 +136,7 @@ public class Rule implements AttributeResolver { /** All rules have unique index 0..n-1 */ public int index; - public int actionIndex = -1; // if lexer; 0..n-1 + public int actionIndex = -1; // if lexer; 0..n-1 for n actions in a rule public Rule(Grammar g, String name, RuleAST ast, int numberOfAlts) { this.g = g; @@ -138,10 +151,15 @@ public class Rule implements AttributeResolver { actions.add(actionAST); alt[currentAlt].actions.add(actionAST); if ( g.isLexer() ) { - actionIndex = g.lexerActions.size(); - if ( g.lexerActions.get(actionAST)==null ) { - g.lexerActions.put(actionAST, actionIndex); - } + defineLexerAction(actionAST); + } + } + + /** Lexer actions are numbered across rules 0..n-1 */ + public void defineLexerAction(ActionAST actionAST) { + actionIndex = g.lexerActions.size(); + if ( g.lexerActions.get(actionAST)==null ) { + g.lexerActions.put(actionAST, actionIndex); } } diff --git a/tool/test/org/antlr/v4/test/TestLexerExec.java b/tool/test/org/antlr/v4/test/TestLexerExec.java index 21316c554..6ca8c0ad4 100644 --- a/tool/test/org/antlr/v4/test/TestLexerExec.java +++ b/tool/test/org/antlr/v4/test/TestLexerExec.java @@ -44,11 +44,70 @@ public class TestLexerExec extends BaseTest { assertEquals(expecting, found); } + @Test public void testSkipCommand() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+ + "WS : (' '|'\\n') -> skip ;"; + String found = execLexer("L.g", grammar, "L", "34 34"); + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<3>,1:0]\n" + + "[@1,3:4='34',<3>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + assertEquals(expecting, found); + } + + @Test public void testMoreCommand() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+ + "WS : '#' -> more ;"; + String found = execLexer("L.g", grammar, "L", "34#10"); + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<3>,1:0]\n" + + "[@1,2:4='#10',<3>,1:2]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + assertEquals(expecting, found); + } + + @Test public void testTypeCommand() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+ + "HASH : '#' -> type(HASH) ;"; + String found = execLexer("L.g", grammar, "L", "34#"); + String expecting = + "I\n" + + "[@0,0:1='34',<3>,1:0]\n" + + "[@1,2:2='#',<4>,1:2]\n" + + "[@2,3:2='',<-1>,1:3]\n"; + assertEquals(expecting, found); + } + + @Test public void testCombinedCommand() throws Exception { + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+ + "HASH : '#' -> type(HASH), skip, more ;"; + String found = execLexer("L.g", grammar, "L", "34#11"); + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<3>,1:0]\n" + + "[@1,2:4='#11',<3>,1:2]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + assertEquals(expecting, found); + } + @Test public void testLexerMode() throws Exception { String grammar = "lexer grammar L;\n" + "STRING_START : '\"' {pushMode(STRING_MODE); more();} ;\n" + - "WS : ' '|'\n' {skip();} ;\n"+ + "WS : (' '|'\n') {skip();} ;\n"+ "mode STRING_MODE;\n"+ "STRING : '\"' {popMode();} ;\n"+ "ANY : . {more();} ;\n";