forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Lexer uses strictly-ordered alternatives within a rule. Simplifies code, increases performance when non-terminal (lexer rules) depth is limited, and actually fixes non-greedy behavior

This commit is contained in:
Sam Harwell 2012-10-21 18:50:24 -05:00
parent ed7d4b1dc1
commit 12b2c34946
12 changed files with 399 additions and 417 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java
View File

@ -1,30 +1,31 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.atn;
@ -71,22 +72,6 @@ public class ATNConfig {
@NotNull
public final SemanticContext semanticContext;
public boolean isGreedy() {
return true;
}
/** Lexer non-greedy implementations need to track information per
* ATNConfig. When the lexer reaches an accept state for a lexer
* rule, it needs to wipe out any configurations associated with
* that rule that are part of a non-greedy subrule. To do that it
* has to make sure that it tracks when a configuration was derived
* from an element within a non-greedy subrule. We use depth for
* that. We're greedy when the depth is 0.
*/
public int getNonGreedyDepth() {
return 0;
}
public ATNConfig(ATNConfig old) { // dup
this.state = old.state;
this.alt = old.alt;

77
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java
View File

@ -1,37 +1,37 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.Array2DHashSet;
import org.antlr.v4.runtime.misc.DoubleKeyMap;
import org.antlr.v4.runtime.misc.NotNull;
import java.util.ArrayList;
import java.util.BitSet;
@ -39,7 +39,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** Specialized OrderedHashSet that can track info about the set.
@ -248,7 +247,6 @@ public class ATNConfigSet implements Set<ATNConfig> {
int hashCode = 7;
hashCode = 31 * hashCode + o.state.stateNumber;
hashCode = 31 * hashCode + o.alt;
hashCode = 31 * hashCode + o.getNonGreedyDepth();
hashCode = 31 * hashCode + o.semanticContext.hashCode();
return hashCode;
}
@ -260,7 +258,6 @@ public class ATNConfigSet implements Set<ATNConfig> {
if ( hashCode(a) != hashCode(b) ) return false;
return a.state.stateNumber==b.state.stateNumber
&& a.alt==b.alt
&& a.getNonGreedyDepth() == b.getNonGreedyDepth()
&& b.semanticContext.equals(b.semanticContext);
}
}
@ -439,26 +436,6 @@ public class ATNConfigSet implements Set<ATNConfig> {
return configs.iterator();
}
public void removeNonGreedyConfigsInAlts(@NotNull BitSet alts) {
if ( readonly ) throw new IllegalStateException("This set is readonly");
if (this.configLookup != null) {
for (Iterator<ATNConfig> it = this.configLookup.iterator(); it.hasNext(); ) {
ATNConfig entry = it.next();
if (!entry.isGreedy() && alts.get(entry.alt)) {
it.remove();
}
}
}
for (Iterator<ATNConfig> it = this.configs.iterator(); it.hasNext(); ) {
ATNConfig value = it.next();
if (!value.isGreedy() && alts.get(value.alt)) {
it.remove();
}
}
}
@Override
public void clear() {
if ( readonly ) throw new IllegalStateException("This set is readonly");

55
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -1,30 +1,31 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.atn;
@ -276,11 +277,9 @@ public abstract class ATNSimulator {
int ndecisions = toInt(data[p++]);
for (int i=1; i<=ndecisions; i++) {
int s = toInt(data[p++]);
int nonGreedy = toInt(data[p++]);
DecisionState decState = (DecisionState)atn.states.get(s);
atn.decisionToState.add(decState);
decState.decision = i-1;
decState.nonGreedy = nonGreedy != 0;
}
verifyATN(atn);

5
runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java
View File

@ -32,9 +32,4 @@ package org.antlr.v4.runtime.atn;
/** Terminal node of a simple (a|b|c) block */
public class BlockEndState extends ATNState {
public BlockStartState startState;
@Override
public boolean isNonGreedyExitState() {
return startState != null && startState.nonGreedy;
}
}

1
runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java
View File

@ -31,5 +31,4 @@ package org.antlr.v4.runtime.atn;
public class DecisionState extends ATNState {
public int decision = -1;
public boolean nonGreedy;
}

67
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java
View File

@ -1,3 +1,33 @@
/*
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.misc.NotNull;
@ -7,14 +37,11 @@ public class LexerATNConfig extends ATNConfig {
/** Capture lexer action we traverse */
public int lexerActionIndex = -1;
private final int nonGreedyDepth;
public LexerATNConfig(@NotNull ATNState state,
int alt,
@Nullable PredictionContext context)
{
super(state, alt, context, SemanticContext.NONE);
this.nonGreedyDepth = 0;
}
public LexerATNConfig(@NotNull ATNState state,
@ -24,20 +51,17 @@ public class LexerATNConfig extends ATNConfig {
{
super(state, alt, context, SemanticContext.NONE);
this.lexerActionIndex = actionIndex;
this.nonGreedyDepth = 0;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state) {
super(c, state, c.context, c.semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
this.nonGreedyDepth = c.nonGreedyDepth;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@NotNull SemanticContext semanticContext) {
super(c, state, c.context, semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
this.nonGreedyDepth = c.nonGreedyDepth;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@ -45,42 +69,11 @@ public class LexerATNConfig extends ATNConfig {
{
super(c, state, c.context, c.semanticContext);
this.lexerActionIndex = actionIndex;
this.nonGreedyDepth = c.nonGreedyDepth;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@Nullable PredictionContext context) {
super(c, state, context, c.semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
this.nonGreedyDepth = c.nonGreedyDepth;
}
private LexerATNConfig(@NotNull LexerATNConfig c, int nonGreedyDepth) {
super(c, c.state, c.context, c.semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
this.nonGreedyDepth = nonGreedyDepth;
}
@Override
public boolean isGreedy() {
return nonGreedyDepth == 0;
}
@Override
public int getNonGreedyDepth() {
return nonGreedyDepth;
}
public LexerATNConfig enterNonGreedyBlock() {
return new LexerATNConfig(this, nonGreedyDepth + 1);
}
public LexerATNConfig exitNonGreedyBlock() {
if (isGreedy()) {
return this;
}
return new LexerATNConfig(this, nonGreedyDepth - 1);
}
}

190
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -1,30 +1,31 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.atn;
@ -42,7 +43,6 @@ import org.antlr.v4.runtime.misc.Nullable;
import java.io.IOException;
import java.io.OutputStream;
import java.util.BitSet;
/** "dup" of ParserInterpreter */
public class LexerATNSimulator extends ATNSimulator {
@ -315,7 +315,7 @@ public class LexerATNSimulator extends ATNSimulator {
}
if (target == null) {
reach = new ATNConfigSet();
reach = new OrderedATNConfigSet();
// if we don't find an existing DFA state
// Fill reach starting from closure, following t transitions
@ -387,7 +387,14 @@ public class LexerATNSimulator extends ATNSimulator {
* we can reach upon input t. Parameter reach is a return parameter.
*/
protected void getReachableConfigSet(ATNConfigSet closure, ATNConfigSet reach, int t) {
// this is used to skip processing for configs which have a lower priority
// than a config that already reached an accept state for the same rule
int skipAlt = ATN.INVALID_ALT_NUMBER;
for (ATNConfig c : closure) {
if (c.alt == skipAlt) {
continue;
}
if ( debug ) {
System.out.format("testing %s at %s\n", getTokenName(t), c.toString(recog, true));
}
@ -397,7 +404,12 @@ public class LexerATNSimulator extends ATNSimulator {
Transition trans = c.state.transition(ti);
ATNState target = getReachableTarget(trans, t);
if ( target!=null ) {
closure(new LexerATNConfig((LexerATNConfig)c, target), reach);
if (closure(new LexerATNConfig((LexerATNConfig)c, target), reach)) {
// any remaining configs for this alt have a lower priority than
// the one that just reached an accept state.
skipAlt = c.alt;
break;
}
}
}
}
@ -409,51 +421,12 @@ public class LexerATNSimulator extends ATNSimulator {
reach, prevAccept.config, prevAccept.index);
}
/* Non-greedy handling works by removing all non-greedy configurations
* from reach when an accept state is reached for the same token. For
* example, consider the following two tokens:
*
* BLOCK : '{' .* '}';
* OPTIONAL_BLOCK : '{' .* '}' '?';
*
* With the following input:
*
* {stuff}?
*
* After matching '}', an accept state at the end of BLOCK is reached,
* so any configurations inside the non-greedy .* loop in BLOCK will be
* removed from reach. The configuration(s) inside the non-greedy .*
* loop in OPTIONAL_BLOCK are unaffected by this because no
* configuration is in an accept state for OPTIONAL_BLOCK at this input
* symbol.
*/
BitSet altsAtAcceptState = new BitSet();
BitSet nonGreedyAlts = new BitSet();
LexerATNConfig acceptConfig = null;
for (ATNConfig config : reach) {
if (config.state instanceof RuleStopState) {
altsAtAcceptState.set(config.alt);
if ( debug ) {
System.out.format("processAcceptConfigs: hit accept config %s index %d\n",
config, input.index());
}
if (acceptConfig == null) {
acceptConfig = (LexerATNConfig)config;
}
acceptConfig = (LexerATNConfig)config;
break;
}
if ( !config.isGreedy() ) {
assert !(config.state instanceof RuleStopState);
nonGreedyAlts.set(config.alt);
}
}
nonGreedyAlts.and(altsAtAcceptState);
// this is now "alts with at least one non-greedy config and one accept config"
if (!nonGreedyAlts.isEmpty()) {
reach.removeNonGreedyConfigsInAlts(nonGreedyAlts);
}
// mark the new preferred accept state
@ -463,6 +436,7 @@ public class LexerATNSimulator extends ATNSimulator {
System.out.println("processAcceptConfigs: found longer token");
}
}
// condition > not >= will favor prev accept at same index.
// This way, "int" is keyword not ID if listed first.
traceAcceptState(acceptConfig.alt);
@ -558,7 +532,7 @@ public class LexerATNSimulator extends ATNSimulator {
@NotNull ATNState p)
{
PredictionContext initialContext = PredictionContext.EMPTY;
ATNConfigSet configs = new ATNConfigSet();
ATNConfigSet configs = new OrderedATNConfigSet();
for (int i=0; i<p.getNumberOfTransitions(); i++) {
ATNState target = p.transition(i).target;
LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
@ -567,7 +541,17 @@ public class LexerATNSimulator extends ATNSimulator {
return configs;
}
protected void closure(@NotNull LexerATNConfig config, @NotNull ATNConfigSet configs) {
/**
* Since the alternatives within any lexer decision are ordered by
* preference, this method stops pursuing the closure as soon as an accept
* state is reached. After the first accept state is reached by depth-first
* search from {@code config}, all other (potentially reachable) states for
* this rule would have a lower priority.
*
* @return {@code true} if an accept state is reached, otherwise
* {@code false}.
*/
protected boolean closure(@NotNull LexerATNConfig config, @NotNull ATNConfigSet configs) {
if ( debug ) {
System.out.println("closure("+config.toString(recog, true)+")");
}
@ -585,10 +569,12 @@ public class LexerATNSimulator extends ATNSimulator {
if ( config.context == null || config.context.hasEmptyPath() ) {
if (config.context == null || config.context.isEmpty()) {
configs.add(config);
return;
}
else {
configs.add(new LexerATNConfig(config, config.state, PredictionContext.EMPTY));
}
configs.add(new LexerATNConfig(config, config.state, PredictionContext.EMPTY));
return true;
}
if ( config.context!=null && !config.context.isEmpty() ) {
@ -609,11 +595,13 @@ public class LexerATNSimulator extends ATNSimulator {
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
LexerATNConfig c = new LexerATNConfig(retState, config.alt, newContext);
closure(c, configs);
if (closure(c, configs)) {
return true;
}
}
}
}
return;
return false;
}
// optimization
@ -622,19 +610,17 @@ public class LexerATNSimulator extends ATNSimulator {
}
ATNState p = config.state;
boolean nonGreedy = (p instanceof DecisionState && ((DecisionState)p).nonGreedy && !(p instanceof PlusLoopbackState))
|| (p instanceof PlusBlockStartState && ((PlusBlockStartState)p).loopBackState.nonGreedy);
for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i);
LexerATNConfig c = getEpsilonTarget(config, t, configs);
if ( c!=null ) {
if (nonGreedy) {
c = c.enterNonGreedyBlock();
if (closure(c, configs)) {
return true;
}
closure(c, configs);
}
}
return false;
}
// side-effect: can alter configs.hasSemanticContext
@ -644,46 +630,6 @@ public class LexerATNSimulator extends ATNSimulator {
@NotNull ATNConfigSet configs)
{
ATNState p = config.state;
switch (p.getStateType()) {
case ATNState.PLUS_LOOP_BACK:
if (((PlusLoopbackState)p).nonGreedy) {
config = config.exitNonGreedyBlock();
}
break;
case ATNState.STAR_LOOP_BACK:
if (((StarLoopbackState)p).getLoopEntryState().nonGreedy) {
config = config.exitNonGreedyBlock();
}
break;
case ATNState.LOOP_END:
ATNState loopBackState = ((LoopEndState)p).loopBackState;
if (loopBackState instanceof PlusLoopbackState) {
if (((PlusLoopbackState)loopBackState).nonGreedy) {
config = config.exitNonGreedyBlock();
}
}
else {
StarLoopbackState starLoopbackState = (StarLoopbackState)loopBackState;
if (starLoopbackState.getLoopEntryState().nonGreedy) {
config = config.exitNonGreedyBlock();
}
}
break;
case ATNState.BLOCK_END:
if (p.isNonGreedyExitState()) {
config = config.exitNonGreedyBlock();
}
break;
default:
break;
}
LexerATNConfig c = null;
switch (t.getSerializationType()) {
case Transition.RULE:

57
runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java Normal file
View File

@ -0,0 +1,57 @@
/*
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.atn;
/**
*
* @author Sam Harwell
*/
public class OrderedATNConfigSet extends ATNConfigSet {
public OrderedATNConfigSet() {
this.configLookup = new LexerConfigHashSet();
}
protected static class LexerConfigHashSet extends ConfigHashSet {
@Override
public int hashCode(ATNConfig o) {
return o.hashCode();
}
@Override
public boolean equals(ATNConfig a, ATNConfig b) {
return a.equals(b);
}
}
}

55
tool/src/org/antlr/v4/automata/ATNSerializer.java
View File

@ -1,30 +1,31 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.automata;
@ -226,7 +227,6 @@ public class ATNSerializer {
data.add(ndecisions);
for (DecisionState decStartState : atn.decisionToState) {
data.add(decStartState.stateNumber);
data.add(decStartState.nonGreedy ? 1 : 0);
}
return data;
}
@ -300,7 +300,6 @@ public class ATNSerializer {
int ndecisions = ATNSimulator.toInt(data[p++]);
for (int i=1; i<=ndecisions; i++) {
int s = ATNSimulator.toInt(data[p++]);
boolean nonGreedy = ATNSimulator.toInt(data[p++]) != 0;
buf.append(i-1).append(":").append(s).append("\n");
}
return buf.toString();

63
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -1,30 +1,31 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.automata;
@ -425,9 +426,7 @@ public class ParserATNFactory implements ATNFactory {
BlockStartState blkStart = (BlockStartState)blk.left;
BlockAST blkAST = (BlockAST)optAST.getChild(0);
blkStart.nonGreedy = !isGreedy(blkAST);
if (!blkStart.nonGreedy) {
if (isGreedy(blkAST)) {
epsilon(blkStart, blk.right);
} else {
Transition existing = blkStart.removeTransition(0);
@ -464,8 +463,7 @@ public class ParserATNFactory implements ATNFactory {
epsilon(blkEnd, loop); // blk can see loop back
BlockAST blkAST = (BlockAST)plusAST.getChild(0);
loop.nonGreedy = !isGreedy(blkAST);
if ( !loop.nonGreedy ) {
if ( isGreedy(blkAST) ) {
epsilon(loop, blkStart); // loop back to start
epsilon(loop, end); // or exit
}
@ -504,8 +502,7 @@ public class ParserATNFactory implements ATNFactory {
end.loopBackState = loop;
BlockAST blkAST = (BlockAST)starAST.getChild(0);
entry.nonGreedy = !isGreedy(blkAST);
if ( !entry.nonGreedy ) {
if ( isGreedy(blkAST) ) {
epsilon(entry, blkStart); // loop enter edge (alt 1)
epsilon(entry, end); // bypass loop edge (alt 2)
}

83
tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java
View File

@ -1,8 +1,37 @@
/*
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.Utils;
@ -39,10 +68,12 @@ public class TestATNLexerInterpreter extends BaseTest {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"A : 'xy'\n" +
" | 'xyz'\n" + // make sure nongreedy mech cut off doesn't kill this alt
" | 'xyz'\n" + // this alt shouldn't be reachable since the alts are ordered
" ;\n" +
"Z : 'z'\n" +
" ;\n");
checkLexerMatches(lg, "xy", "A, EOF");
checkLexerMatches(lg, "xyz", "A, EOF");
checkLexerMatches(lg, "xyz", "A, Z, EOF");
}
@Test public void testShortLongRule2() throws Exception {
@ -60,6 +91,8 @@ public class TestATNLexerInterpreter extends BaseTest {
"lexer grammar L;\n"+
"A : 'xy' .\n" + // should pursue '.' since xyz hits stop first, before 2nd alt
" | 'xy'\n" +
" ;\n" +
"Z : 'z'\n" +
" ;\n");
checkLexerMatches(lg, "xy", "A, EOF");
checkLexerMatches(lg, "xyz", "A, EOF");
@ -69,22 +102,12 @@ public class TestATNLexerInterpreter extends BaseTest {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"A : 'xy'\n" +
" | 'xy' .\n" + // should pursue '.' since A is greedy
" | 'xy' .\n" + // should not pursue '.' since alts are ordered
" ;\n" +
"Z : 'z'\n" +
" ;\n");
checkLexerMatches(lg, "xy", "A, EOF");
RecognitionException e = checkLexerMatches(lg, "xyz", "A, EOF");
assertNull(e);
}
@Test public void testWildcardQuirk() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"A : 'xy'\n" +
" | 'xy' . 'z'\n" + // will pursue '.' since A is greedy
" ;\n");
// checkLexerMatches(lg, "xy", "A, EOF");
RecognitionException e = checkLexerMatches(lg, "xyqz", "A, EOF");
assertNull(e);
checkLexerMatches(lg, "xyz", "A, Z, EOF");
}
@Test public void testWildcardNonQuirkWhenSplitBetweenTwoRules() throws Exception {
@ -93,7 +116,7 @@ public class TestATNLexerInterpreter extends BaseTest {
"A : 'xy' ;\n" +
"B : 'xy' . 'z' ;\n");
checkLexerMatches(lg, "xy", "A, EOF");
checkLexerMatches(lg, "xyz", "B, EOF");
checkLexerMatches(lg, "xyqz", "B, EOF");
}
@Test public void testLexerLoops() throws Exception {
@ -151,22 +174,15 @@ public class TestATNLexerInterpreter extends BaseTest {
@Test public void testRecursiveLexerRuleRefWithWildcard() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"CMT : '/*' (CMT | .)+ '*/' ;\n" +
"CMT : '/*' (CMT | .)* '*/' ;\n" +
"WS : (' '|'\n')+ ;");
String expecting = "CMT, WS, CMT, WS, CMT, WS, EOF";
// stuff on end of comment matches another rule
String expecting = "CMT, WS, CMT, WS, EOF";
checkLexerMatches(lg,
"/* ick */\n" +
"/* /* */\n" +
"/* /*nested*/ */\n",
expecting);
// stuff on end of comment doesn't match another rule
expecting = "CMT, WS, CMT, WS, CMT, WS, EOF";
checkLexerMatches(lg,
"/* ick */x\n" +
"/* /* */x\n" +
"/* /*nested*/ */x\n",
expecting);
}
@Test public void testLexerWildcardNonGreedyLoopByDefault() throws Exception {
@ -265,20 +281,14 @@ public class TestATNLexerInterpreter extends BaseTest {
checkLexerMatches(lg, "a", expecting);
}
protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
protected void checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
ATN atn = createATN(lg, true);
CharStream input = new ANTLRInputStream(inputString);
ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE");
DOTGenerator dot = new DOTGenerator(lg);
System.out.println(dot.getDOT(startState, true));
List<String> tokenTypes = null;
RecognitionException retException = null;
try {
tokenTypes = getTokenTypes(lg, atn, input, false);
}
catch (RecognitionException lre) { retException = lre; }
if ( retException!=null ) return retException;
List<String> tokenTypes = getTokenTypes(lg, atn, input, false);
String result = Utils.join(tokenTypes.iterator(), ", ");
System.out.println(tokenTypes);
@ -288,7 +298,6 @@ public class TestATNLexerInterpreter extends BaseTest {
input.seek(0);
List<String> tokenTypes2 = getTokenTypes(lg, atn, input, true);
assertEquals("interp vs adaptive types differ", tokenTypes, tokenTypes2);
return null;
}
}

94
tool/test/org/antlr/v4/test/TestLexerExec.java
View File

@ -1,3 +1,33 @@
/*
* [The "BSD license"]
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test;
import org.junit.Test;
@ -29,6 +59,20 @@ public class TestLexerExec extends BaseTest {
assertEquals(expecting, found);
}
@Test
public void testImplicitNonGreedyTermination() throws Exception {
String grammar =
"lexer grammar L;\n"
+ "STRING : '\"' ('\"\"' | .)* '\"';";
String found = execLexer("L.g4", grammar, "L", "\"hi\"\"mom\"");
assertEquals(
"[@0,0:3='\"hi\"',<1>,1:0]\n" +
"[@1,4:8='\"mom\"',<1>,1:4]\n" +
"[@2,9:8='<EOF>',<-1>,1:9]\n", found);
assertNull(stderrDuringParse);
}
@Test
public void testImplicitGreedyOptional() throws Exception {
String grammar =
@ -168,11 +212,9 @@ public class TestLexerExec extends BaseTest {
String expecting =
"[@0,0:8='/* ick */',<1>,1:0]\n" +
"[@1,9:9='\\n',<2>,1:9]\n" +
"[@2,10:17='/* /* */',<1>,2:0]\n" +
"[@3,18:18='\\n',<2>,2:8]\n" +
"[@4,19:31='/* /*nested*/',<1>,3:0]\n" +
"[@5,32:32=' ',<2>,3:13]\n" +
"[@6,36:35='<EOF>',<-1>,4:0]\n";
"[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,35:35='\\n',<2>,3:16]\n" +
"[@4,36:35='<EOF>',<-1>,4:17]\n";
// stuff on end of comment matches another rule
String found = execLexer("L.g4", grammar, "L",
@ -180,19 +222,14 @@ public class TestLexerExec extends BaseTest {
"/* /* */\n" +
"/* /*nested*/ */\n");
assertEquals(expecting, found);
assertEquals(
"line 3:14 token recognition error at: '*'\n" +
"line 3:15 token recognition error at: '/\n'\n", stderrDuringParse);
assertNull(stderrDuringParse);
// stuff on end of comment doesn't match another rule
expecting =
"[@0,0:8='/* ick */',<1>,1:0]\n" +
"[@1,10:10='\\n',<2>,1:10]\n" +
"[@2,11:18='/* /* */',<1>,2:0]\n" +
"[@3,20:20='\\n',<2>,2:9]\n" +
"[@4,21:33='/* /*nested*/',<1>,3:0]\n" +
"[@5,34:34=' ',<2>,3:13]\n" +
"[@6,38:38='\\n',<2>,3:17]\n" +
"[@7,39:38='<EOF>',<-1>,4:18]\n";
"[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,38:38='\\n',<2>,3:17]\n" +
"[@4,39:38='<EOF>',<-1>,4:18]\n";
found = execLexer("L.g4", grammar, "L",
"/* ick */x\n" +
"/* /* */x\n" +
@ -200,9 +237,7 @@ public class TestLexerExec extends BaseTest {
assertEquals(expecting, found);
assertEquals(
"line 1:9 token recognition error at: 'x'\n" +
"line 2:8 token recognition error at: 'x'\n" +
"line 3:14 token recognition error at: '*'\n" +
"line 3:15 token recognition error at: '/x'\n", stderrDuringParse);
"line 3:16 token recognition error at: 'x'\n", stderrDuringParse);
}
@Test public void testRecursiveLexerRuleRefWithWildcardPlus() throws Exception {
@ -215,11 +250,9 @@ public class TestLexerExec extends BaseTest {
String expecting =
"[@0,0:8='/* ick */',<1>,1:0]\n" +
"[@1,9:9='\\n',<2>,1:9]\n" +
"[@2,10:17='/* /* */',<1>,2:0]\n" +
"[@3,18:18='\\n',<2>,2:8]\n" +
"[@4,19:31='/* /*nested*/',<1>,3:0]\n" +
"[@5,32:32=' ',<2>,3:13]\n" +
"[@6,36:35='<EOF>',<-1>,4:0]\n";
"[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,35:35='\\n',<2>,3:16]\n" +
"[@4,36:35='<EOF>',<-1>,4:17]\n";
// stuff on end of comment matches another rule
String found = execLexer("L.g4", grammar, "L",
@ -227,19 +260,14 @@ public class TestLexerExec extends BaseTest {
"/* /* */\n" +
"/* /*nested*/ */\n");
assertEquals(expecting, found);
assertEquals(
"line 3:14 token recognition error at: '*'\n" +
"line 3:15 token recognition error at: '/\n'\n", stderrDuringParse);
assertNull(stderrDuringParse);
// stuff on end of comment doesn't match another rule
expecting =
"[@0,0:8='/* ick */',<1>,1:0]\n" +
"[@1,10:10='\\n',<2>,1:10]\n" +
"[@2,11:18='/* /* */',<1>,2:0]\n" +
"[@3,20:20='\\n',<2>,2:9]\n" +
"[@4,21:33='/* /*nested*/',<1>,3:0]\n" +
"[@5,34:34=' ',<2>,3:13]\n" +
"[@6,38:38='\\n',<2>,3:17]\n" +
"[@7,39:38='<EOF>',<-1>,4:18]\n";
"[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,38:38='\\n',<2>,3:17]\n" +
"[@4,39:38='<EOF>',<-1>,4:18]\n";
found = execLexer("L.g4", grammar, "L",
"/* ick */x\n" +
"/* /* */x\n" +
@ -247,9 +275,7 @@ public class TestLexerExec extends BaseTest {
assertEquals(expecting, found);
assertEquals(
"line 1:9 token recognition error at: 'x'\n" +
"line 2:8 token recognition error at: 'x'\n" +
"line 3:14 token recognition error at: '*'\n" +
"line 3:15 token recognition error at: '/x'\n", stderrDuringParse);
"line 3:16 token recognition error at: 'x'\n", stderrDuringParse);
}
@Test public void testActionExecutedInDFA() throws Exception {