diff --git a/runtime/Java/src/org/antlr/v4/runtime/nfa/Bytecode.java b/runtime/Java/src/org/antlr/v4/runtime/nfa/Bytecode.java index 1cacc8e81..5d9b1a999 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/nfa/Bytecode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/nfa/Bytecode.java @@ -6,8 +6,12 @@ import java.util.List; /** */ public class Bytecode { public static final int MAX_OPNDS = 3; // Or single opnd indicating variable number - public static final int OPND_SIZE_IN_BYTES = 2; - public enum OperandType { NONE, CHAR, ADDR, INT, VARARGS } + public static final int ADDR_SIZE = 2; + public enum OperandType { + NONE(0), BYTE(1), CHAR(2), ADDR(ADDR_SIZE), SHORT(2), INT(4), VARARGS(0); + public int sizeInBytes; + OperandType(int sizeInBytes) { this.sizeInBytes = sizeInBytes; } + } public static class Instruction { String name; // E.g., "load_str", "new" @@ -38,22 +42,26 @@ public class Bytecode { public static final short ACCEPT = 1; public static final short JMP = 2; public static final short SPLIT = 3; - public static final short MATCH = 4; - public static final short RANGE = 5; + public static final short MATCH8 = 4; + public static final short MATCH16 = 5; + public static final short RANGE8 = 6; + public static final short RANGE16 = 7; /** Used for disassembly; describes instruction set */ public static Instruction[] instructions = new Instruction[] { null, // - new Instruction("accept", OperandType.INT), // index is the opcode + new Instruction("accept", OperandType.SHORT), // index is the opcode new Instruction("jmp", OperandType.ADDR), new Instruction("split", OperandType.VARARGS), - new Instruction("match", OperandType.CHAR), - new Instruction("range", OperandType.CHAR, OperandType.CHAR) + new Instruction("match8", OperandType.BYTE), + new Instruction("match16", OperandType.CHAR), + new Instruction("range8", OperandType.BYTE, OperandType.BYTE), + new Instruction("range16", OperandType.CHAR, OperandType.CHAR) }; - public static String disassemble(byte[] code) { + public static String disassemble(byte[] code, int start) { StringBuilder buf = new StringBuilder(); - int i=0; + int i=start; while (i operands = new ArrayList(); - for (int i=0; iopcode){ + case Char: + if(*sp != pc->c) + return 0; + pc++; + sp++; + continue; + case Match: + return 1; + case Jmp: + pc = pc->x; + continue; + case Split: + if(recursiveloop(pc->x, sp)) + return 1; + pc = pc->y; + continue; + } + assert(0); + return -1; +} + */ +public class NFA { + public byte[] code; + Map ruleToAddr; + + public NFA(byte[] code, Map ruleToAddr) { + this.code = code; + this.ruleToAddr = ruleToAddr; + } + + public int exec(CharStream input, String ruleName) { + return exec(input, ruleToAddr.get(ruleName)); + } + + public int exec(CharStream input) { return exec(input, 0); } - /* - for(;;){ - switch(pc->opcode){ - case Char: - if(*sp != pc->c) - return 0; - pc++; - sp++; - continue; - case Match: - return 1; - case Jmp: - pc = pc->x; - continue; - case Split: - if(recursiveloop(pc->x, sp)) - return 1; - pc = pc->y; - continue; - } - assert(0); - return -1; - } - */ public int exec(CharStream input, int ip) { while ( ip < code.length ) { int c = input.LA(1); @@ -38,18 +51,24 @@ public class Interpreter { short opcode = code[ip]; ip++; // move to next instruction or first byte of operand switch (opcode) { - case Bytecode.MATCH : - int o = getShort(code, ip); - ip += 2; - if ( c != o ) return 0; + case Bytecode.MATCH8 : + if ( c != code[ip] ) return 0; + ip++; input.consume(); break; - case Bytecode.RANGE : - int from = getShort(code, ip); + case Bytecode.MATCH16 : + if ( c != getShort(code, ip) ) return 0; ip += 2; - int to = getShort(code, ip); + input.consume(); + break; + case Bytecode.RANGE8 : + if ( ccode[ip+1] ) return 0; ip += 2; - if ( cto ) return 0; + input.consume(); + break; + case Bytecode.RANGE16 : + if ( cgetShort(code, ip+2) ) return 0; + ip += 4; input.consume(); break; case Bytecode.ACCEPT : @@ -67,7 +86,7 @@ public class Interpreter { for (int i=1; i<=nopnds-1; i++) { int addr = getShort(code, ip); ip += 2; - System.out.println("try alt "+i+" at "+addr); + //System.out.println("try alt "+i+" at "+addr); int m = input.mark(); int r = exec(input, addr); if ( r>0 ) { input.release(m); return r; } @@ -76,7 +95,7 @@ public class Interpreter { // try final alternative (w/o recursion) int addr = getShort(code, ip); ip = addr; - System.out.println("try alt "+nopnds+" at "+addr); + //System.out.println("try alt "+nopnds+" at "+addr); continue; default : throw new RuntimeException("invalid instruction @ "+ip+": "+opcode); @@ -87,12 +106,10 @@ public class Interpreter { void trace(int ip) { String instr = Bytecode.disassembleInstruction(code, ip); - System.out.println(instr); - } + System.out.println(instr); + } public static int getShort(byte[] memory, int index) { - int b1 = memory[index++]&0xFF; // mask off sign-extended bits - int b2 = memory[index++]&0xFF; - return b1<<(8*1) | b2; - } + return (memory[index]&0xFF) <<(8*1) | (memory[index+1]&0xFF); // prevent sign extension with mask + } } diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 42536dda4..3be8886f1 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -7,6 +7,7 @@ import org.antlr.v4.automata.DFA; import org.antlr.v4.automata.LexerNFAFactory; import org.antlr.v4.automata.NFAFactory; import org.antlr.v4.automata.ParserNFAFactory; +import org.antlr.v4.codegen.CodeGenPipeline; import org.antlr.v4.parse.ANTLRLexer; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.parse.GrammarASTAdaptor; @@ -407,6 +408,8 @@ public class Tool { if ( generate_DFA_dot ) generateDFAs(g); // GENERATE CODE + CodeGenPipeline gen = new CodeGenPipeline(g); + gen.process(); } // TODO: Move to ast manipulation class? diff --git a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java index 3a9531340..029a4a5c2 100644 --- a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java +++ b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java @@ -68,7 +68,7 @@ public class AnalysisPipeline { System.out.print("DFA="+dfa); if ( !dfa.valid() ) { - System.out.print("invalid DFA"); + System.out.println("invalid DFA"); } conv.issueAmbiguityWarnings(); diff --git a/tool/src/org/antlr/v4/automata/DFA.java b/tool/src/org/antlr/v4/automata/DFA.java index c1a29b266..40303731a 100644 --- a/tool/src/org/antlr/v4/automata/DFA.java +++ b/tool/src/org/antlr/v4/automata/DFA.java @@ -113,9 +113,9 @@ public class DFA { public boolean valid() { return - converter.danglingStates.size()==0 && - converter.abortedDueToMultipleRecursiveAltsAt ==null && - converter.recursionOverflowState ==null; + converter.danglingStates.size()==0; +// converter.abortedDueToMultipleRecursiveAltsAt ==null && +// converter.recursionOverflowState ==null; } public String toString() { diff --git a/tool/src/org/antlr/v4/automata/ParserNFAFactory.java b/tool/src/org/antlr/v4/automata/ParserNFAFactory.java index 151fd26e0..b90880b76 100644 --- a/tool/src/org/antlr/v4/automata/ParserNFAFactory.java +++ b/tool/src/org/antlr/v4/automata/ParserNFAFactory.java @@ -259,6 +259,9 @@ public class ParserNFAFactory implements NFAFactory { } Handle first = els.get(0); Handle last = els.get(els.size()-1); + if ( first==null || last==null ) { + System.out.println("huh?"); + } return new Handle(first.left, last.right); } diff --git a/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java new file mode 100644 index 000000000..e6d6f419a --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java @@ -0,0 +1,26 @@ +package org.antlr.v4.codegen; + +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.v4.runtime.nfa.NFA; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; + +public class CodeGenPipeline { + Grammar g; + public CodeGenPipeline(Grammar g) { + this.g = g; + } + public void process() { + if ( g.isLexer() ) processLexer(); + } + + void processLexer() { + LexerGrammar lg = (LexerGrammar)g; + for (String modeName : lg.modes.keySet()) { // for each mode + NFA nfa = NFABytecodeGenerator.getBytecode(lg, modeName); + ANTLRStringStream input = new ANTLRStringStream("32ab"); + int ttype = nfa.exec(input); System.out.println("ttype="+ttype); + ttype = nfa.exec(input); System.out.println("ttype="+ttype); + } + } +} diff --git a/tool/src/org/antlr/v4/codegen/NFABytecodeGenerator.java b/tool/src/org/antlr/v4/codegen/NFABytecodeGenerator.java index 8fd53cbd8..d0c045157 100644 --- a/tool/src/org/antlr/v4/codegen/NFABytecodeGenerator.java +++ b/tool/src/org/antlr/v4/codegen/NFABytecodeGenerator.java @@ -2,30 +2,47 @@ package org.antlr.v4.codegen; import org.antlr.runtime.RecognizerSharedState; import org.antlr.runtime.Token; +import org.antlr.runtime.tree.CommonTreeNodeStream; import org.antlr.runtime.tree.TreeNodeStream; +import org.antlr.v4.parse.ANTLRParser; +import org.antlr.v4.parse.GrammarASTAdaptor; import org.antlr.v4.runtime.nfa.Bytecode; +import org.antlr.v4.runtime.nfa.NFA; import org.antlr.v4.runtime.tree.TreeParser; +import org.antlr.v4.tool.GrammarAST; +import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.Rule; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** http://swtch.com/~rsc/regexp/regexp2.html */ public class NFABytecodeGenerator extends TreeParser { public abstract static class Instr { - public short opcode; public int addr; - public int nBytes; - public Instr(short opcode, int nBytes) { this.opcode = opcode; this.nBytes = nBytes; } - public void write(byte[] code) { code[addr] = (byte)opcode; } + public abstract short opcode(); + public abstract int nBytes(); + public int charSize(int a, int b) { return Math.max(charSize(a), charSize(b)); } + public int charSize(int c) { + if ( c<=0xFF ) return 1; + if ( c<=0xFFFF ) return 2; + return 4; + } + public void write(byte[] code) { code[addr] = (byte)opcode(); } } public static class MatchInstr extends Instr { Token token; - char c; - public MatchInstr(Token t, char c) { super(Bytecode.MATCH, 3); this.token = t; this.c = c; } + int c; + public MatchInstr(Token t, int c) { super(); this.token = t; this.c = c; } + public short opcode() { return charSize(c)==1?Bytecode.MATCH8:Bytecode.MATCH16; }; + public int nBytes() { return 1+charSize(c); } public void write(byte[] code) { super.write(code); - writeShort(code, addr+1, (short)c); + if ( charSize(c)==1 ) code[addr+1] = (byte)(c&0xFF); + else writeShort(code, addr+1, (short)c); } @Override @@ -37,33 +54,41 @@ public class NFABytecodeGenerator extends TreeParser { } public static class RangeInstr extends Instr { - Token a, b; - char start, stop; - public RangeInstr(Token a, Token b) { - super(Bytecode.RANGE, 1+2*Bytecode.OPND_SIZE_IN_BYTES); - this.a = a; - this.b = b; - start = (char)Target.getCharValueFromGrammarCharLiteral(a.getText()); - stop = (char)Target.getCharValueFromGrammarCharLiteral(b.getText()); + Token start, stop; + int a, b; + public RangeInstr(Token start, Token stop) { + this.start = start; + this.stop = stop; + a = (char)Target.getCharValueFromGrammarCharLiteral(start.getText()); + b = (char)Target.getCharValueFromGrammarCharLiteral(stop.getText()); } + public short opcode() { return charSize(a, b)==1?Bytecode.RANGE8:Bytecode.RANGE16; }; + public int nBytes() { return 1+2*charSize(a, b); } public void write(byte[] code) { super.write(code); - writeShort(code, addr+1, (short)start); - writeShort(code, addr+1+Bytecode.OPND_SIZE_IN_BYTES, (short)stop); + if ( charSize(a,b)==1 ) { + code[addr+1] = (byte)(a&0xFF); + code[addr+2] = (byte)(b&0xFF); + } + else { + writeShort(code, addr+1, (short)a); + writeShort(code, addr+1+charSize(a,b), (short)b); + } } @Override public String toString() { - return addr+":RangeInstr{"+start+".."+stop+"}"; + return addr+":RangeInstr{"+ a +".."+ b +"}"; } } public static class AcceptInstr extends Instr { int ruleIndex; public AcceptInstr(int ruleIndex) { - super(Bytecode.ACCEPT, 3); this.ruleIndex = ruleIndex; } + public short opcode() { return Bytecode.ACCEPT; }; + public int nBytes() { return 1+2; } public void write(byte[] code) { super.write(code); writeShort(code, addr+1, (short)ruleIndex); @@ -73,7 +98,8 @@ public class NFABytecodeGenerator extends TreeParser { public static class JumpInstr extends Instr { int target; - public JumpInstr() { super(Bytecode.JMP, 3); } + public short opcode() { return Bytecode.JMP; }; + public int nBytes() { return 1+Bytecode.ADDR_SIZE; } public void write(byte[] code) { super.write(code); writeShort(code, addr+1, (short)target); @@ -89,15 +115,18 @@ public class NFABytecodeGenerator extends TreeParser { public static class SplitInstr extends Instr { List addrs = new ArrayList(); - public SplitInstr(int nAlts) { super(Bytecode.SPLIT, 1+2+nAlts*2); } + int nAlts; + public SplitInstr(int nAlts) { this.nAlts = nAlts; } + public short opcode() { return Bytecode.SPLIT; }; + public int nBytes() { return 1+2+nAlts*Bytecode.ADDR_SIZE; } public void write(byte[] code) { super.write(code); int a = addr + 1; writeShort(code, a, (short)addrs.size()); - a += Bytecode.OPND_SIZE_IN_BYTES; + a += 2; for (int x : addrs) { writeShort(code, a, (short)x); - a += Bytecode.OPND_SIZE_IN_BYTES; + a += Bytecode.ADDR_SIZE; } } @@ -122,7 +151,7 @@ public class NFABytecodeGenerator extends TreeParser { public void emit(Instr I) { I.addr = ip; - ip += I.nBytes; + ip += I.nBytes(); instrs.add(I); } @@ -133,9 +162,24 @@ public class NFABytecodeGenerator extends TreeParser { } } - public byte[] getCode() { + /** Given any block of alts, return list of instruction objects */ + public static List getInstructions(GrammarAST blk, int acceptValue) { + GrammarASTAdaptor adaptor = new GrammarASTAdaptor(); + CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk); + NFABytecodeTriggers gen = new NFABytecodeTriggers(nodes); + try { + gen.block(); + gen.emit(new NFABytecodeGenerator.AcceptInstr(acceptValue)); + } + catch (Exception e){ + e.printStackTrace(System.err); + } + return gen.instrs; + } + + public static byte[] getByteCode(List instrs) { Instr last = instrs.get(instrs.size() - 1); - int size = last.addr + last.nBytes; + int size = last.addr + last.nBytes(); byte[] code = new byte[size]; for (Instr I : instrs) { I.write(code); @@ -143,6 +187,42 @@ public class NFABytecodeGenerator extends TreeParser { return code; } + public static NFA getBytecode(LexerGrammar lg, String modeName) { + GrammarASTAdaptor adaptor = new GrammarASTAdaptor(); + NFABytecodeTriggers gen = new NFABytecodeTriggers(null); + + // add split for s0 to hook up rules (fill in operands as we gen rules) + int numRules = lg.modes.get(modeName).size(); + int numFragmentRules = 0; + for (Rule r : lg.modes.get(modeName)) { if ( r.isFragment() ) numFragmentRules++; } + SplitInstr s0 = new SplitInstr(numRules - numFragmentRules); + gen.emit(s0); + + Map ruleToAddr = new HashMap(); + for (Rule r : lg.modes.get(modeName)) { // for each rule in mode + GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK); + CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk); + gen.setTreeNodeStream(nodes); + ruleToAddr.put(r.name, gen.ip); + if ( !r.isFragment() ) s0.addrs.add(gen.ip); + try { + gen.block(); + int ruleTokenType = lg.getTokenType(r.name); + gen.emit(new NFABytecodeGenerator.AcceptInstr(ruleTokenType)); + } + catch (Exception e){ + e.printStackTrace(System.err); + } + } + byte[] code = NFABytecodeGenerator.getByteCode(gen.instrs); + System.out.println("all:"); + System.out.println(Bytecode.disassemble(code)); + System.out.println("rule addrs="+ruleToAddr); + + NFA nfa = new NFA(code, ruleToAddr); + return nfa; + } + /** Write value at index into a byte array highest to lowest byte, * left to right. */ @@ -151,30 +231,4 @@ public class NFABytecodeGenerator extends TreeParser { memory[index+1] = (byte)(value&0xFF); } - /* CODE TO GENERATE NFA BYTECODES - // testing code gen concept - GrammarASTAdaptor adaptor = new GrammarASTAdaptor(); - for (Rule r : lg.modes.get(modeName)) { - GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK); - CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk); - NFABytecodeTriggers gen = new NFABytecodeTriggers(nodes); - try { - gen.block(); - gen.emit(new NFABytecodeGenerator.AcceptInstr(r.index)); - System.out.println("code=\n"+gen.instrs); - byte[] code = gen.getCode(); - System.out.println(Bytecode.disassemble(code)); - Interpreter in = new Interpreter(code); - String s = "i"; - ANTLRStringStream input = new ANTLRStringStream(s); - int rule = in.exec(input, 0); - System.out.println(s+" matched rule "+rule+" leaving off at index="+input.index()); - } - catch (Exception e){ - e.printStackTrace(System.err); - } - } - - */ - } diff --git a/tool/src/org/antlr/v4/codegen/NFABytecodeTriggers.java b/tool/src/org/antlr/v4/codegen/NFABytecodeTriggers.java index bb9f0c88a..6242b9e54 100644 --- a/tool/src/org/antlr/v4/codegen/NFABytecodeTriggers.java +++ b/tool/src/org/antlr/v4/codegen/NFABytecodeTriggers.java @@ -1,4 +1,4 @@ -// $ANTLR ${project.version} ${buildNumber} NFABytecodeTriggers.g 2010-04-21 13:15:49 +// $ANTLR ${project.version} ${buildNumber} NFABytecodeTriggers.g 2010-04-29 12:03:06 package org.antlr.v4.codegen; diff --git a/tool/src/org/antlr/v4/semantics/CollectSymbols.g b/tool/src/org/antlr/v4/semantics/CollectSymbols.g index 89c6715cd..1b61e015d 100644 --- a/tool/src/org/antlr/v4/semantics/CollectSymbols.g +++ b/tool/src/org/antlr/v4/semantics/CollectSymbols.g @@ -73,7 +73,7 @@ import org.stringtemplate.v4.misc.MultiMap; @members { Rule currentRule; -String currentMode = Grammar.DEFAULT_MODE_NAME; +String currentMode = LexerGrammar.DEFAULT_MODE_NAME; int currentAlt = 1; // 1..n public List rules = new ArrayList(); public List rulerefs = new ArrayList(); @@ -149,6 +149,7 @@ rule : ^( RULE name=ID (options {greedy=false;}:.)* (^(RULEMODIFIERS (m=. {modifiers.add($m);})+))? + (^(AT ID ACTION))* ^(BLOCK .+) .* ) diff --git a/tool/src/org/antlr/v4/semantics/CollectSymbols.java b/tool/src/org/antlr/v4/semantics/CollectSymbols.java index b18a99c1d..f7426085e 100644 --- a/tool/src/org/antlr/v4/semantics/CollectSymbols.java +++ b/tool/src/org/antlr/v4/semantics/CollectSymbols.java @@ -1,4 +1,4 @@ -// $ANTLR ${project.version} ${buildNumber} CollectSymbols.g 2010-04-19 17:33:29 +// $ANTLR ${project.version} ${buildNumber} CollectSymbols.g 2010-04-28 18:42:02 /* [The "BSD license"] @@ -37,7 +37,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; - /** Collects rules, terminals, strings, actions, scopes etc... from AST * Side-effects: None */ @@ -670,7 +669,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { // $ANTLR start "rule" - // CollectSymbols.g:147:1: rule : ^( RULE name= ID ( options {greedy=false; } : . )* ( ^( RULEMODIFIERS (m= . )+ ) )? ^( BLOCK ( . )+ ) ( . )* ) ; + // CollectSymbols.g:147:1: rule : ^( RULE name= ID ( options {greedy=false; } : . )* ( ^( RULEMODIFIERS (m= . )+ ) )? ( ^( AT ID ACTION ) )* ^( BLOCK ( . )+ ) ( . )* ) ; public final void rule() throws RecognitionException { GrammarAST name=null; GrammarAST RULE8=null; @@ -678,8 +677,8 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { List modifiers = new ArrayList(); try { - // CollectSymbols.g:149:2: ( ^( RULE name= ID ( options {greedy=false; } : . )* ( ^( RULEMODIFIERS (m= . )+ ) )? ^( BLOCK ( . )+ ) ( . )* ) ) - // CollectSymbols.g:149:6: ^( RULE name= ID ( options {greedy=false; } : . )* ( ^( RULEMODIFIERS (m= . )+ ) )? ^( BLOCK ( . )+ ) ( . )* ) + // CollectSymbols.g:149:2: ( ^( RULE name= ID ( options {greedy=false; } : . )* ( ^( RULEMODIFIERS (m= . )+ ) )? ( ^( AT ID ACTION ) )* ^( BLOCK ( . )+ ) ( . )* ) ) + // CollectSymbols.g:149:6: ^( RULE name= ID ( options {greedy=false; } : . )* ( ^( RULEMODIFIERS (m= . )+ ) )? ( ^( AT ID ACTION ) )* ^( BLOCK ( . )+ ) ( . )* ) { RULE8=(GrammarAST)match(input,RULE,FOLLOW_RULE_in_rule382); if (state.failed) return ; @@ -761,46 +760,42 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { } - match(input,BLOCK,FOLLOW_BLOCK_in_rule441); if (state.failed) return ; - - match(input, Token.DOWN, null); if (state.failed) return ; - // CollectSymbols.g:152:17: ( . )+ - int cnt8=0; + // CollectSymbols.g:152:9: ( ^( AT ID ACTION ) )* loop8: do { int alt8=2; int LA8_0 = input.LA(1); - if ( ((LA8_0>=SEMPRED && LA8_0<=ALT_REWRITE)) ) { + if ( (LA8_0==AT) ) { alt8=1; } - else if ( (LA8_0==UP) ) { - alt8=2; - } switch (alt8) { case 1 : - // CollectSymbols.g:152:17: . + // CollectSymbols.g:152:10: ^( AT ID ACTION ) { - matchAny(input); if (state.failed) return ; + match(input,AT,FOLLOW_AT_in_rule442); if (state.failed) return ; + + match(input, Token.DOWN, null); if (state.failed) return ; + match(input,ID,FOLLOW_ID_in_rule444); if (state.failed) return ; + match(input,ACTION,FOLLOW_ACTION_in_rule446); if (state.failed) return ; + + match(input, Token.UP, null); if (state.failed) return ; } break; default : - if ( cnt8 >= 1 ) break loop8; - if (state.backtracking>0) {state.failed=true; return ;} - EarlyExitException eee = - new EarlyExitException(8, input); - throw eee; + break loop8; } - cnt8++; } while (true); + match(input,BLOCK,FOLLOW_BLOCK_in_rule460); if (state.failed) return ; - match(input, Token.UP, null); if (state.failed) return ; - // CollectSymbols.g:153:9: ( . )* + match(input, Token.DOWN, null); if (state.failed) return ; + // CollectSymbols.g:153:17: ( . )+ + int cnt9=0; loop9: do { int alt9=2; @@ -816,7 +811,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { switch (alt9) { case 1 : - // CollectSymbols.g:153:9: . + // CollectSymbols.g:153:17: . { matchAny(input); if (state.failed) return ; @@ -824,7 +819,42 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { break; default : - break loop9; + if ( cnt9 >= 1 ) break loop9; + if (state.backtracking>0) {state.failed=true; return ;} + EarlyExitException eee = + new EarlyExitException(9, input); + throw eee; + } + cnt9++; + } while (true); + + + match(input, Token.UP, null); if (state.failed) return ; + // CollectSymbols.g:154:9: ( . )* + loop10: + do { + int alt10=2; + int LA10_0 = input.LA(1); + + if ( ((LA10_0>=SEMPRED && LA10_0<=ALT_REWRITE)) ) { + alt10=1; + } + else if ( (LA10_0==UP) ) { + alt10=2; + } + + + switch (alt10) { + case 1 : + // CollectSymbols.g:154:9: . + { + matchAny(input); if (state.failed) return ; + + } + break; + + default : + break loop10; } } while (true); @@ -835,7 +865,6 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { int numAlts = RULE8.getFirstChildWithType(BLOCK).getChildCount(); Rule r = new Rule(g, (name!=null?name.getText():null), (GrammarASTWithOptions)RULE8, numAlts); if ( g.isLexer() ) r.mode = currentMode; - else r.mode = LexerGrammar.DEFAULT_MODE_NAME; if ( modifiers.size()>0 ) r.modifiers = modifiers; rules.add(r); currentRule = r; @@ -908,7 +937,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { // CollectSymbols.g:173:2: ( RULE ) // CollectSymbols.g:173:4: RULE { - match(input,RULE,FOLLOW_RULE_in_finishRule508); if (state.failed) return ; + match(input,RULE,FOLLOW_RULE_in_finishRule527); if (state.failed) return ; if ( state.backtracking==1 ) { currentRule = null; } @@ -941,11 +970,11 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { if (state.backtracking>0) {state.failed=true; return ;} throw new FailedPredicateException(input, "ruleNamedAction", "inContext(\"RULE\")"); } - match(input,AT,FOLLOW_AT_in_ruleNamedAction524); if (state.failed) return ; + match(input,AT,FOLLOW_AT_in_ruleNamedAction543); if (state.failed) return ; match(input, Token.DOWN, null); if (state.failed) return ; - ID9=(GrammarAST)match(input,ID,FOLLOW_ID_in_ruleNamedAction526); if (state.failed) return ; - ACTION10=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_ruleNamedAction528); if (state.failed) return ; + ID9=(GrammarAST)match(input,ID,FOLLOW_ID_in_ruleNamedAction545); if (state.failed) return ; + ACTION10=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_ruleNamedAction547); if (state.failed) return ; match(input, Token.UP, null); if (state.failed) return ; if ( state.backtracking==1 ) { @@ -983,7 +1012,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { if (state.backtracking>0) {state.failed=true; return ;} throw new FailedPredicateException(input, "ruleAction", "inContext(\"RULE ...\")&&!inContext(\"SCOPE\")&&\n\t\t !inContext(\"CATCH\")&&!inContext(\"FINALLY\")&&!inContext(\"AT\")"); } - ACTION11=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_ruleAction548); if (state.failed) return ; + ACTION11=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_ruleAction567); if (state.failed) return ; if ( state.backtracking==1 ) { currentRule.alt[currentAlt].actions.add((ActionAST)ACTION11); @@ -1014,11 +1043,11 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { // CollectSymbols.g:195:2: ( ^( CATCH ARG_ACTION ACTION ) ) // CollectSymbols.g:195:4: ^( CATCH ARG_ACTION ACTION ) { - match(input,CATCH,FOLLOW_CATCH_in_exceptionHandler564); if (state.failed) return ; + match(input,CATCH,FOLLOW_CATCH_in_exceptionHandler583); if (state.failed) return ; match(input, Token.DOWN, null); if (state.failed) return ; - match(input,ARG_ACTION,FOLLOW_ARG_ACTION_in_exceptionHandler566); if (state.failed) return ; - ACTION12=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_exceptionHandler568); if (state.failed) return ; + match(input,ARG_ACTION,FOLLOW_ARG_ACTION_in_exceptionHandler585); if (state.failed) return ; + ACTION12=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_exceptionHandler587); if (state.failed) return ; match(input, Token.UP, null); if (state.failed) return ; if ( state.backtracking==1 ) { @@ -1051,10 +1080,10 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { // CollectSymbols.g:203:2: ( ^( FINALLY ACTION ) ) // CollectSymbols.g:203:4: ^( FINALLY ACTION ) { - match(input,FINALLY,FOLLOW_FINALLY_in_finallyClause585); if (state.failed) return ; + match(input,FINALLY,FOLLOW_FINALLY_in_finallyClause604); if (state.failed) return ; match(input, Token.DOWN, null); if (state.failed) return ; - ACTION13=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_finallyClause587); if (state.failed) return ; + ACTION13=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_finallyClause606); if (state.failed) return ; match(input, Token.UP, null); if (state.failed) return ; if ( state.backtracking==1 ) { @@ -1091,7 +1120,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { if (state.backtracking>0) {state.failed=true; return ;} throw new FailedPredicateException(input, "ruleArg", "inContext(\"RULE\")"); } - ARG_ACTION14=(GrammarAST)match(input,ARG_ACTION,FOLLOW_ARG_ACTION_in_ruleArg607); if (state.failed) return ; + ARG_ACTION14=(GrammarAST)match(input,ARG_ACTION,FOLLOW_ARG_ACTION_in_ruleArg626); if (state.failed) return ; if ( state.backtracking==1 ) { currentRule.args = ScopeParser.parseTypeList((ARG_ACTION14!=null?ARG_ACTION14.getText():null)); @@ -1122,10 +1151,10 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { // CollectSymbols.g:219:2: ( ^( RETURNS ARG_ACTION ) ) // CollectSymbols.g:219:4: ^( RETURNS ARG_ACTION ) { - match(input,RETURNS,FOLLOW_RETURNS_in_ruleReturns624); if (state.failed) return ; + match(input,RETURNS,FOLLOW_RETURNS_in_ruleReturns643); if (state.failed) return ; match(input, Token.DOWN, null); if (state.failed) return ; - ARG_ACTION15=(GrammarAST)match(input,ARG_ACTION,FOLLOW_ARG_ACTION_in_ruleReturns626); if (state.failed) return ; + ARG_ACTION15=(GrammarAST)match(input,ARG_ACTION,FOLLOW_ARG_ACTION_in_ruleReturns645); if (state.failed) return ; match(input, Token.UP, null); if (state.failed) return ; if ( state.backtracking==1 ) { @@ -1165,25 +1194,25 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { throw new FailedPredicateException(input, "ruleScopeSpec", "inContext(\"RULE\")"); } // CollectSymbols.g:228:3: ( ^( SCOPE ACTION ) | ^( SCOPE (ids+= ID )+ ) ) - int alt11=2; - int LA11_0 = input.LA(1); + int alt12=2; + int LA12_0 = input.LA(1); - if ( (LA11_0==SCOPE) ) { - int LA11_1 = input.LA(2); + if ( (LA12_0==SCOPE) ) { + int LA12_1 = input.LA(2); - if ( (LA11_1==DOWN) ) { - int LA11_2 = input.LA(3); + if ( (LA12_1==DOWN) ) { + int LA12_2 = input.LA(3); - if ( (LA11_2==ACTION) ) { - alt11=1; + if ( (LA12_2==ACTION) ) { + alt12=1; } - else if ( (LA11_2==ID) ) { - alt11=2; + else if ( (LA12_2==ID) ) { + alt12=2; } else { if (state.backtracking>0) {state.failed=true; return ;} NoViableAltException nvae = - new NoViableAltException("", 11, 2, input); + new NoViableAltException("", 12, 2, input); throw nvae; } @@ -1191,7 +1220,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { else { if (state.backtracking>0) {state.failed=true; return ;} NoViableAltException nvae = - new NoViableAltException("", 11, 1, input); + new NoViableAltException("", 12, 1, input); throw nvae; } @@ -1199,18 +1228,18 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { else { if (state.backtracking>0) {state.failed=true; return ;} NoViableAltException nvae = - new NoViableAltException("", 11, 0, input); + new NoViableAltException("", 12, 0, input); throw nvae; } - switch (alt11) { + switch (alt12) { case 1 : // CollectSymbols.g:228:5: ^( SCOPE ACTION ) { - match(input,SCOPE,FOLLOW_SCOPE_in_ruleScopeSpec649); if (state.failed) return ; + match(input,SCOPE,FOLLOW_SCOPE_in_ruleScopeSpec668); if (state.failed) return ; match(input, Token.DOWN, null); if (state.failed) return ; - ACTION16=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_ruleScopeSpec651); if (state.failed) return ; + ACTION16=(GrammarAST)match(input,ACTION,FOLLOW_ACTION_in_ruleScopeSpec670); if (state.failed) return ; match(input, Token.UP, null); if (state.failed) return ; if ( state.backtracking==1 ) { @@ -1226,26 +1255,26 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { case 2 : // CollectSymbols.g:234:5: ^( SCOPE (ids+= ID )+ ) { - match(input,SCOPE,FOLLOW_SCOPE_in_ruleScopeSpec664); if (state.failed) return ; + match(input,SCOPE,FOLLOW_SCOPE_in_ruleScopeSpec683); if (state.failed) return ; match(input, Token.DOWN, null); if (state.failed) return ; // CollectSymbols.g:234:16: (ids+= ID )+ - int cnt10=0; - loop10: + int cnt11=0; + loop11: do { - int alt10=2; - int LA10_0 = input.LA(1); + int alt11=2; + int LA11_0 = input.LA(1); - if ( (LA10_0==ID) ) { - alt10=1; + if ( (LA11_0==ID) ) { + alt11=1; } - switch (alt10) { + switch (alt11) { case 1 : // CollectSymbols.g:234:16: ids+= ID { - ids=(GrammarAST)match(input,ID,FOLLOW_ID_in_ruleScopeSpec668); if (state.failed) return ; + ids=(GrammarAST)match(input,ID,FOLLOW_ID_in_ruleScopeSpec687); if (state.failed) return ; if (list_ids==null) list_ids=new ArrayList(); list_ids.add(ids); @@ -1254,13 +1283,13 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { break; default : - if ( cnt10 >= 1 ) break loop10; + if ( cnt11 >= 1 ) break loop11; if (state.backtracking>0) {state.failed=true; return ;} EarlyExitException eee = - new EarlyExitException(10, input); + new EarlyExitException(11, input); throw eee; } - cnt10++; + cnt11++; } while (true); @@ -1353,30 +1382,30 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { throw new FailedPredicateException(input, "labeledElement", "inContext(\"RULE ...\")"); } // CollectSymbols.g:252:3: ( ^( ASSIGN id= ID e= . ) | ^( PLUS_ASSIGN id= ID e= . ) ) - int alt12=2; - int LA12_0 = input.LA(1); + int alt13=2; + int LA13_0 = input.LA(1); - if ( (LA12_0==ASSIGN) ) { - alt12=1; + if ( (LA13_0==ASSIGN) ) { + alt13=1; } - else if ( (LA12_0==PLUS_ASSIGN) ) { - alt12=2; + else if ( (LA13_0==PLUS_ASSIGN) ) { + alt13=2; } else { if (state.backtracking>0) {state.failed=true; return retval;} NoViableAltException nvae = - new NoViableAltException("", 12, 0, input); + new NoViableAltException("", 13, 0, input); throw nvae; } - switch (alt12) { + switch (alt13) { case 1 : // CollectSymbols.g:252:5: ^( ASSIGN id= ID e= . ) { - match(input,ASSIGN,FOLLOW_ASSIGN_in_labeledElement732); if (state.failed) return retval; + match(input,ASSIGN,FOLLOW_ASSIGN_in_labeledElement751); if (state.failed) return retval; match(input, Token.DOWN, null); if (state.failed) return retval; - id=(GrammarAST)match(input,ID,FOLLOW_ID_in_labeledElement736); if (state.failed) return retval; + id=(GrammarAST)match(input,ID,FOLLOW_ID_in_labeledElement755); if (state.failed) return retval; e=(GrammarAST)input.LT(1); matchAny(input); if (state.failed) return retval; @@ -1387,10 +1416,10 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { case 2 : // CollectSymbols.g:253:5: ^( PLUS_ASSIGN id= ID e= . ) { - match(input,PLUS_ASSIGN,FOLLOW_PLUS_ASSIGN_in_labeledElement748); if (state.failed) return retval; + match(input,PLUS_ASSIGN,FOLLOW_PLUS_ASSIGN_in_labeledElement767); if (state.failed) return retval; match(input, Token.DOWN, null); if (state.failed) return retval; - id=(GrammarAST)match(input,ID,FOLLOW_ID_in_labeledElement752); if (state.failed) return retval; + id=(GrammarAST)match(input,ID,FOLLOW_ID_in_labeledElement771); if (state.failed) return retval; e=(GrammarAST)input.LT(1); matchAny(input); if (state.failed) return retval; @@ -1436,23 +1465,23 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { try { // CollectSymbols.g:258:5: ({...}? STRING_LITERAL | TOKEN_REF ) - int alt13=2; - int LA13_0 = input.LA(1); + int alt14=2; + int LA14_0 = input.LA(1); - if ( (LA13_0==STRING_LITERAL) ) { - alt13=1; + if ( (LA14_0==STRING_LITERAL) ) { + alt14=1; } - else if ( (LA13_0==TOKEN_REF) ) { - alt13=2; + else if ( (LA14_0==TOKEN_REF) ) { + alt14=2; } else { if (state.backtracking>0) {state.failed=true; return retval;} NoViableAltException nvae = - new NoViableAltException("", 13, 0, input); + new NoViableAltException("", 14, 0, input); throw nvae; } - switch (alt13) { + switch (alt14) { case 1 : // CollectSymbols.g:258:7: {...}? STRING_LITERAL { @@ -1460,7 +1489,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { if (state.backtracking>0) {state.failed=true; return retval;} throw new FailedPredicateException(input, "terminal", "!inContext(\"TOKENS ASSIGN\")"); } - STRING_LITERAL17=(GrammarAST)match(input,STRING_LITERAL,FOLLOW_STRING_LITERAL_in_terminal778); if (state.failed) return retval; + STRING_LITERAL17=(GrammarAST)match(input,STRING_LITERAL,FOLLOW_STRING_LITERAL_in_terminal797); if (state.failed) return retval; if ( state.backtracking==1 ) { terminals.add(((GrammarAST)retval.start)); @@ -1476,7 +1505,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { case 2 : // CollectSymbols.g:266:7: TOKEN_REF { - TOKEN_REF18=(GrammarAST)match(input,TOKEN_REF,FOLLOW_TOKEN_REF_in_terminal793); if (state.failed) return retval; + TOKEN_REF18=(GrammarAST)match(input,TOKEN_REF,FOLLOW_TOKEN_REF_in_terminal812); if (state.failed) return retval; if ( state.backtracking==1 ) { terminals.add(TOKEN_REF18); @@ -1513,22 +1542,22 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { // CollectSymbols.g:278:7: ({...}?r= RULE_REF | r= RULE_REF ) { // CollectSymbols.g:278:7: ({...}?r= RULE_REF | r= RULE_REF ) - int alt14=2; - int LA14_0 = input.LA(1); + int alt15=2; + int LA15_0 = input.LA(1); - if ( (LA14_0==RULE_REF) ) { - int LA14_1 = input.LA(2); + if ( (LA15_0==RULE_REF) ) { + int LA15_1 = input.LA(2); if ( ((inContext("DOT ..."))) ) { - alt14=1; + alt15=1; } else if ( (true) ) { - alt14=2; + alt15=2; } else { if (state.backtracking>0) {state.failed=true; return ;} NoViableAltException nvae = - new NoViableAltException("", 14, 1, input); + new NoViableAltException("", 15, 1, input); throw nvae; } @@ -1536,11 +1565,11 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { else { if (state.backtracking>0) {state.failed=true; return ;} NoViableAltException nvae = - new NoViableAltException("", 14, 0, input); + new NoViableAltException("", 15, 0, input); throw nvae; } - switch (alt14) { + switch (alt15) { case 1 : // CollectSymbols.g:278:9: {...}?r= RULE_REF { @@ -1548,7 +1577,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { if (state.backtracking>0) {state.failed=true; return ;} throw new FailedPredicateException(input, "ruleref", "inContext(\"DOT ...\")"); } - r=(GrammarAST)match(input,RULE_REF,FOLLOW_RULE_REF_in_ruleref830); if (state.failed) return ; + r=(GrammarAST)match(input,RULE_REF,FOLLOW_RULE_REF_in_ruleref849); if (state.failed) return ; if ( state.backtracking==1 ) { qualifiedRulerefs.add((GrammarAST)r.getParent()); } @@ -1558,7 +1587,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { case 2 : // CollectSymbols.g:280:8: r= RULE_REF { - r=(GrammarAST)match(input,RULE_REF,FOLLOW_RULE_REF_in_ruleref843); if (state.failed) return ; + r=(GrammarAST)match(input,RULE_REF,FOLLOW_RULE_REF_in_ruleref862); if (state.failed) return ; } break; @@ -1607,7 +1636,7 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { "\21\3\uffff\1\12\1\14\1\uffff\1\11\2\uffff\1\1\1\uffff\1\2\3\uffff"+ "\1\10"; static final String DFA1_specialS = - "\11\uffff\1\3\1\4\1\1\23\uffff\1\2\1\0\1\uffff}>"; + "\11\uffff\1\2\1\4\1\3\23\uffff\1\0\1\1\1\uffff}>"; static final String[] DFA1_transitionS = { "\1\7\1\uffff\1\17\4\uffff\1\1\11\uffff\1\10\1\uffff\1\21\1\20"+ "\1\uffff\1\5\11\uffff\1\3\4\uffff\1\15\10\uffff\1\2\2\uffff"+ @@ -1685,36 +1714,6 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { int _s = s; switch ( s ) { case 0 : - int LA1_32 = input.LA(1); - - - int index1_32 = input.index(); - input.rewind(); - s = -1; - if ( ((inContext("TOKENS"))) ) {s = 4;} - - else if ( ((inContext("RULE ..."))) ) {s = 13;} - - - input.seek(index1_32); - if ( s>=0 ) return s; - break; - case 1 : - int LA1_11 = input.LA(1); - - - int index1_11 = input.index(); - input.rewind(); - s = -1; - if ( ((inContext("RESULT ..."))) ) {s = 12;} - - else if ( (true) ) {s = 22;} - - - input.seek(index1_11); - if ( s>=0 ) return s; - break; - case 2 : int LA1_31 = input.LA(1); @@ -1729,7 +1728,22 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { input.seek(index1_31); if ( s>=0 ) return s; break; - case 3 : + case 1 : + int LA1_32 = input.LA(1); + + + int index1_32 = input.index(); + input.rewind(); + s = -1; + if ( ((inContext("TOKENS"))) ) {s = 4;} + + else if ( ((inContext("RULE ..."))) ) {s = 13;} + + + input.seek(index1_32); + if ( s>=0 ) return s; + break; + case 2 : int LA1_9 = input.LA(1); @@ -1744,6 +1758,21 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { input.seek(index1_9); if ( s>=0 ) return s; break; + case 3 : + int LA1_11 = input.LA(1); + + + int index1_11 = input.index(); + input.rewind(); + s = -1; + if ( ((inContext("RESULT ..."))) ) {s = 12;} + + else if ( (true) ) {s = 22;} + + + input.seek(index1_11); + if ( s>=0 ) return s; + break; case 4 : int LA1_10 = input.LA(1); @@ -1768,40 +1797,51 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { } } static final String DFA5_eotS = - "\26\uffff"; + "\40\uffff"; static final String DFA5_eofS = - "\26\uffff"; + "\40\uffff"; static final String DFA5_minS = - "\1\4\2\2\1\uffff\2\4\2\2\1\uffff\1\3\3\2\3\4\2\2\2\3\1\uffff\1\3"; + "\1\4\3\2\1\uffff\3\4\1\2\1\3\1\2\1\uffff\2\3\1\uffff\4\2\4\4\1\2"+ + "\2\3\1\2\3\3\1\uffff\1\3"; static final String DFA5_maxS = - "\3\146\1\uffff\4\146\1\uffff\13\146\1\uffff\1\146"; + "\4\146\1\uffff\6\146\1\uffff\2\146\1\uffff\17\146\1\uffff\1\146"; static final String DFA5_acceptS = - "\3\uffff\1\1\4\uffff\1\2\13\uffff\1\2\1\uffff"; + "\4\uffff\1\1\6\uffff\1\2\2\uffff\1\2\17\uffff\1\2\1\uffff"; static final String DFA5_specialS = - "\26\uffff}>"; + "\40\uffff}>"; static final String[] DFA5_transitionS = { - "\107\3\1\1\1\3\1\2\31\3", - "\1\4\1\uffff\143\3", - "\1\5\1\uffff\143\3", + "\70\4\1\2\16\4\1\1\1\4\1\3\31\4", + "\1\5\1\uffff\143\4", + "\1\6\1\uffff\143\4", + "\1\7\1\uffff\143\4", "", - "\143\6", - "\143\7", - "\2\10\143\6", - "\1\10\1\11\143\7", + "\143\10", + "\123\4\1\11\17\4", + "\143\12", + "\2\13\143\10", + "\15\4\1\14\126\4", + "\1\13\1\15\143\12", "", - "\1\10\107\14\1\12\1\14\1\13\31\14", - "\1\15\1\10\107\14\1\12\1\14\1\13\31\14", - "\1\16\1\10\107\14\1\12\1\14\1\13\31\14", - "\1\17\1\10\107\14\1\12\1\14\1\13\31\14", - "\143\20", - "\143\21", - "\143\22", - "\1\3\1\23\143\20", - "\1\3\1\24\143\21", - "\1\25\143\22", - "\1\24\107\14\1\12\1\14\1\13\31\14", + "\1\16\143\4", + "\1\16\70\22\1\20\16\22\1\17\1\22\1\21\31\22", "", - "\1\24\107\14\1\12\1\14\1\13\31\14" + "\1\23\1\16\70\22\1\20\16\22\1\17\1\22\1\21\31\22", + "\1\24\1\16\70\22\1\20\16\22\1\17\1\22\1\21\31\22", + "\1\25\1\16\70\22\1\20\16\22\1\17\1\22\1\21\31\22", + "\1\26\1\16\70\22\1\20\16\22\1\17\1\22\1\21\31\22", + "\143\27", + "\123\31\1\30\17\31", + "\143\32", + "\143\31", + "\1\4\1\33\143\27", + "\1\35\14\31\1\34\126\31", + "\1\35\143\31", + "\1\4\1\36\143\32", + "\1\36\70\22\1\20\16\22\1\17\1\22\1\21\31\22", + "\1\37\143\31", + "\1\36\70\22\1\20\16\22\1\17\1\22\1\21\31\22", + "", + "\1\36\70\22\1\20\16\22\1\17\1\22\1\21\31\22" }; static final short[] DFA5_eot = DFA.unpackEncodedString(DFA5_eotS); @@ -1873,33 +1913,36 @@ public class CollectSymbols extends org.antlr.v4.runtime.tree.TreeFilter { public static final BitSet FOLLOW_RULE_in_rule382 = new BitSet(new long[]{0x0000000000000004L}); public static final BitSet FOLLOW_ID_in_rule394 = new BitSet(new long[]{0xFFFFFFFFFFFFFFF0L,0x0000007FFFFFFFFFL}); public static final BitSet FOLLOW_RULEMODIFIERS_in_rule418 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_BLOCK_in_rule441 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_set_in_setAlt484 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_RULE_in_finishRule508 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_AT_in_ruleNamedAction524 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ID_in_ruleNamedAction526 = new BitSet(new long[]{0x0000000000010000L}); - public static final BitSet FOLLOW_ACTION_in_ruleNamedAction528 = new BitSet(new long[]{0x0000000000000008L}); - public static final BitSet FOLLOW_ACTION_in_ruleAction548 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_CATCH_in_exceptionHandler564 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ARG_ACTION_in_exceptionHandler566 = new BitSet(new long[]{0x0000000000010000L}); - public static final BitSet FOLLOW_ACTION_in_exceptionHandler568 = new BitSet(new long[]{0x0000000000000008L}); - public static final BitSet FOLLOW_FINALLY_in_finallyClause585 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ACTION_in_finallyClause587 = new BitSet(new long[]{0x0000000000000008L}); - public static final BitSet FOLLOW_ARG_ACTION_in_ruleArg607 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_RETURNS_in_ruleReturns624 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ARG_ACTION_in_ruleReturns626 = new BitSet(new long[]{0x0000000000000008L}); - public static final BitSet FOLLOW_SCOPE_in_ruleScopeSpec649 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ACTION_in_ruleScopeSpec651 = new BitSet(new long[]{0x0000000000000008L}); - public static final BitSet FOLLOW_SCOPE_in_ruleScopeSpec664 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ID_in_ruleScopeSpec668 = new BitSet(new long[]{0x0000000000000008L,0x0000000000800000L}); - public static final BitSet FOLLOW_set_in_rewriteElement696 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_ASSIGN_in_labeledElement732 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ID_in_labeledElement736 = new BitSet(new long[]{0xFFFFFFFFFFFFFFF0L,0x0000007FFFFFFFFFL}); - public static final BitSet FOLLOW_PLUS_ASSIGN_in_labeledElement748 = new BitSet(new long[]{0x0000000000000004L}); - public static final BitSet FOLLOW_ID_in_labeledElement752 = new BitSet(new long[]{0xFFFFFFFFFFFFFFF0L,0x0000007FFFFFFFFFL}); - public static final BitSet FOLLOW_STRING_LITERAL_in_terminal778 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_TOKEN_REF_in_terminal793 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_RULE_REF_in_ruleref830 = new BitSet(new long[]{0x0000000000000002L}); - public static final BitSet FOLLOW_RULE_REF_in_ruleref843 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_in_rule442 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ID_in_rule444 = new BitSet(new long[]{0x0000000000010000L}); + public static final BitSet FOLLOW_ACTION_in_rule446 = new BitSet(new long[]{0x0000000000000008L}); + public static final BitSet FOLLOW_BLOCK_in_rule460 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_set_in_setAlt503 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_RULE_in_finishRule527 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_in_ruleNamedAction543 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ID_in_ruleNamedAction545 = new BitSet(new long[]{0x0000000000010000L}); + public static final BitSet FOLLOW_ACTION_in_ruleNamedAction547 = new BitSet(new long[]{0x0000000000000008L}); + public static final BitSet FOLLOW_ACTION_in_ruleAction567 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_CATCH_in_exceptionHandler583 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ARG_ACTION_in_exceptionHandler585 = new BitSet(new long[]{0x0000000000010000L}); + public static final BitSet FOLLOW_ACTION_in_exceptionHandler587 = new BitSet(new long[]{0x0000000000000008L}); + public static final BitSet FOLLOW_FINALLY_in_finallyClause604 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ACTION_in_finallyClause606 = new BitSet(new long[]{0x0000000000000008L}); + public static final BitSet FOLLOW_ARG_ACTION_in_ruleArg626 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_RETURNS_in_ruleReturns643 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ARG_ACTION_in_ruleReturns645 = new BitSet(new long[]{0x0000000000000008L}); + public static final BitSet FOLLOW_SCOPE_in_ruleScopeSpec668 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ACTION_in_ruleScopeSpec670 = new BitSet(new long[]{0x0000000000000008L}); + public static final BitSet FOLLOW_SCOPE_in_ruleScopeSpec683 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ID_in_ruleScopeSpec687 = new BitSet(new long[]{0x0000000000000008L,0x0000000000800000L}); + public static final BitSet FOLLOW_set_in_rewriteElement715 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_ASSIGN_in_labeledElement751 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ID_in_labeledElement755 = new BitSet(new long[]{0xFFFFFFFFFFFFFFF0L,0x0000007FFFFFFFFFL}); + public static final BitSet FOLLOW_PLUS_ASSIGN_in_labeledElement767 = new BitSet(new long[]{0x0000000000000004L}); + public static final BitSet FOLLOW_ID_in_labeledElement771 = new BitSet(new long[]{0xFFFFFFFFFFFFFFF0L,0x0000007FFFFFFFFFL}); + public static final BitSet FOLLOW_STRING_LITERAL_in_terminal797 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_TOKEN_REF_in_terminal812 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_RULE_REF_in_ruleref849 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_RULE_REF_in_ruleref862 = new BitSet(new long[]{0x0000000000000002L}); } \ No newline at end of file diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java index 69620eb36..6d07e2bd7 100644 --- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java +++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java @@ -84,13 +84,24 @@ public class SemanticPipeline { AttributeChecks.checkAllAttributeExpressions(g); // ASSIGN TOKEN TYPES - assignTokenTypes(g, collector, symcheck); + if ( g.isLexer() ) assignLexerTokenTypes(g, collector); + else assignTokenTypes(g, collector, symcheck); UseDefAnalyzer usedef = new UseDefAnalyzer(); usedef.checkRewriteElementsPresentOnLeftSide(g, collector.rules); } - public void assignTokenTypes(Grammar g, CollectSymbols collector, SymbolChecks symcheck) { + void assignLexerTokenTypes(Grammar g, CollectSymbols collector) { + Grammar G = g.getOutermostGrammar(); // put in root, even if imported + for (GrammarAST def : collector.tokensDefs) { + if ( def.getType()== ANTLRParser.ID ) G.defineTokenName(def.getText()); + } + for (Rule r : g.rules.values()) { + if ( !r.isFragment() ) G.defineTokenName(r.name); + } + } + + void assignTokenTypes(Grammar g, CollectSymbols collector, SymbolChecks symcheck) { if ( g.implicitLexerOwner!=null ) { // copy vocab from combined to implicit lexer g.importVocab(g.implicitLexerOwner); diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index 13a0bd34b..ba6f4e81d 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -30,6 +30,7 @@ public class Grammar implements AttributeResolver { new HashMap() {{ put("lexer:RULE_LABEL", Rule.predefinedLexerRulePropertiesDict); put("lexer:LEXER_STRING_LABEL", Rule.predefinedLexerRulePropertiesDict); + put("lexer:TOKEN_LABEL", AttributeDict.predefinedTokenDict); put("parser:RULE_LABEL", Rule.predefinedRulePropertiesDict); put("parser:TOKEN_LABEL", AttributeDict.predefinedTokenDict); put("tree:RULE_LABEL", Rule.predefinedTreeRulePropertiesDict);