jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

bug fixes in error handling 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9111]
This commit is contained in:
parrt 2011-10-06 10:40:07 -08:00
parent 1efa316de9
commit 251a42ddbb
10 changed files with 234 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java Normal file
View File

@ -0,0 +1,72 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
/** A kind of ReaderStream that pulls from an InputStream.
* Useful for reading from stdin and specifying file encodings etc...
*/
public class ANTLRInputStream extends ANTLRReaderStream {
public ANTLRInputStream() {
}
public ANTLRInputStream(InputStream input) throws IOException {
this(input, null);
}
public ANTLRInputStream(InputStream input, int size) throws IOException {
this(input, size, null);
}
public ANTLRInputStream(InputStream input, String encoding) throws IOException {
this(input, INITIAL_BUFFER_SIZE, encoding);
}
public ANTLRInputStream(InputStream input, int size, String encoding) throws IOException {
this(input, size, READ_BUFFER_SIZE, encoding);
}
public ANTLRInputStream(InputStream input,
int size,
int readBufferSize,
String encoding)
throws IOException
{
InputStreamReader isr;
if ( encoding!=null ) {
isr = new InputStreamReader(input, encoding);
}
else {
isr = new InputStreamReader(input);
}
load(isr, size, readBufferSize);
}
}

96
runtime/Java/src/org/antlr/v4/runtime/ANTLRReaderStream.java Normal file
View File

@ -0,0 +1,96 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import java.io.IOException;
import java.io.Reader;
/** Vacuum all input from a Reader and then treat it like a StringStream.
* Manage the buffer manually to avoid unnecessary data copying.
*
* If you need encoding, use ANTLRInputStream.
*/
public class ANTLRReaderStream extends ANTLRStringStream {
public static final int READ_BUFFER_SIZE = 1024;
public static final int INITIAL_BUFFER_SIZE = 1024;
public ANTLRReaderStream() {
}
public ANTLRReaderStream(Reader r) throws IOException {
this(r, INITIAL_BUFFER_SIZE, READ_BUFFER_SIZE);
}
public ANTLRReaderStream(Reader r, int size) throws IOException {
this(r, size, READ_BUFFER_SIZE);
}
public ANTLRReaderStream(Reader r, int size, int readChunkSize) throws IOException {
load(r, size, readChunkSize);
}
public void load(Reader r, int size, int readChunkSize)
throws IOException
{
if ( r==null ) {
return;
}
if ( size<=0 ) {
size = INITIAL_BUFFER_SIZE;
}
if ( readChunkSize<=0 ) {
readChunkSize = READ_BUFFER_SIZE;
}
// System.out.println("load "+size+" in chunks of "+readChunkSize);
try {
// alloc initial buffer size.
data = new char[size];
// read all the data in chunks of readChunkSize
int numRead=0;
int p = 0;
do {
if ( p+readChunkSize > data.length ) { // overflow?
// System.out.println("### overflow p="+p+", data.length="+data.length);
char[] newdata = new char[data.length*2]; // resize
System.arraycopy(data, 0, newdata, 0, data.length);
data = newdata;
}
numRead = r.read(data, p, readChunkSize);
// System.out.println("read "+numRead+" chars; p was "+p+" is now "+(p+numRead));
p += numRead;
} while (numRead!=-1); // while not EOF
// set the actual size of the data available;
// EOF subtracted one above in p+=numRead; add one back
super.n = p+1;
//System.out.println("n="+n);
}
finally {
r.close();
}
}
}

6
runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java
View File

@ -101,10 +101,8 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
// TODO: subclass this class for treeparsers
TokenStream tokens = (TokenStream)recognizer.getInputStream();
Token la = tokens.LT(1);
if ( expecting.contains(la.getType()) ) {
endErrorCondition(recognizer);
return;
}
// Return but don't end recovery. only do that upon valid token match
if ( expecting.contains(la.getType()) ) return;
reportUnwantedToken(recognizer);
consumeUntil(recognizer, expecting);
}

25
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -30,7 +30,8 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.dfa.*;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import org.antlr.v4.runtime.tree.ASTNodeStream;
import org.stringtemplate.v4.misc.MultiMap;
@ -138,6 +139,7 @@ public class ParserATNSimulator extends ATNSimulator {
// doesn't create DFA when matching
public int matchATN(TokenStream input, ATNState startState) {
DFA dfa = new DFA(startState);
if ( outerContext==null ) outerContext = RuleContext.EMPTY;
RuleContext ctx = RuleContext.EMPTY;
OrderedHashSet<ATNConfig> s0_closure = computeStartState(dfa.decision, startState, ctx);
return execATN(input, dfa, input.index(), s0_closure, false);
@ -200,22 +202,19 @@ public class ParserATNSimulator extends ATNSimulator {
s.edges[input.LA(1)+1] = ERROR; // IGNORE really not error
}
}
if ( dfa_debug ) {
System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa);
//dump(dfa);
}
// action already executed
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" predicts "+alt);
return alt; // we've updated DFA, exec'd action, and have our deepest answer
}
catch (NoViableAltException nvae) {
alt = -1;
}
if ( dfa_debug ) {
System.out.println("back from DFA update, alt="+alt+", dfa=\n"+dfa);
//dump(dfa);
}
if ( alt==-1 ) {
addDFAEdge(s, t, ERROR);
break loop; // dead end; no where to go, fall back on prev if any
throw nvae;
}
// action already executed
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" predicts "+alt);
return alt; // we've updated DFA, exec'd action, and have our deepest answer
}
DFAState target = s.edges[t+1];
if ( target == ERROR ) break;

14
tool/playground/TestAA.java Normal file
View File

@ -0,0 +1,14 @@
import org.antlr.v4.runtime.ANTLRFileStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.RuleContext;
public class TestAA {
public static void main(String[] args) throws Exception {
AALexer t = new AALexer(new ANTLRFileStream(args[0]));
CommonTokenStream tokens = new CommonTokenStream(t);
AAParser p = new AAParser(tokens);
p.setBuildParseTrees(true);
RuleContext ctx = p.prog();
//System.out.println("ctx="+ctx.toStringTree(p));
}
}

13
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -30,17 +30,22 @@
package org.antlr.v4.automata;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.*;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.Token;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.misc.CharSupport;
import org.antlr.v4.parse.*;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.ATNBuilder;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.semantics.UseDefAnalyzer;
import org.antlr.v4.tool.*;
import java.lang.reflect.Constructor;
import java.util.*;
import java.util.Collection;
import java.util.List;
/** ATN construction routines triggered by ATNBuilder.g.
*

8
tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java
View File

@ -29,10 +29,12 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.*;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.PlusBlockStartState;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarAST;
import java.util.List;
@ -53,6 +55,8 @@ public class LL1PlusBlock extends LL1Loop {
PlusBlockStartState blkStart = (PlusBlockStartState)plusRoot.atnState;
stateNumber = blkStart.loopBackState.stateNumber;
this.decision = blkStart.decision;
Grammar g = factory.getGrammar();
CodeGenerator gen = factory.getGenerator();

2
tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java
View File

@ -41,6 +41,8 @@ public class LL1PlusBlockSingleAlt extends LL1Loop {
public LL1PlusBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlockForAlt> alts) {
super(factory, blkAST, alts);
PlusBlockStartState blkStart = (PlusBlockStartState)blkAST.atnState;
stateNumber = blkStart.loopBackState.stateNumber;
PlusBlockStartState plus = (PlusBlockStartState)blkAST.atnState;
this.decision = plus.loopBackState.decision;
IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision);

4
tool/src/org/antlr/v4/codegen/model/PlusBlock.java
View File

@ -30,7 +30,8 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.PlusBlockStartState;
import org.antlr.v4.runtime.atn.PlusLoopbackState;
import org.antlr.v4.tool.GrammarAST;
import java.util.List;
@ -44,6 +45,7 @@ public class PlusBlock extends Loop {
{
super(factory, ebnfRootAST, alts);
PlusLoopbackState loop = ((PlusBlockStartState)ebnfRootAST.atnState).loopBackState;
stateNumber = loop.stateNumber;
this.error = new ThrowNoViableAlt(factory, ebnfRootAST, null);
decision = loop.decision;
exitAlt = alts.size()+1;

26
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -33,15 +33,24 @@ import org.antlr.v4.automata.*;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Rule;
import org.junit.*;
import org.stringtemplate.v4.*;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
import javax.tools.*;
import javax.tools.JavaCompiler;
import javax.tools.JavaFileObject;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;
import java.io.*;
import java.util.*;
@ -1308,9 +1317,10 @@ public abstract class BaseTest {
public Token LT(int i) {
CommonToken t;
if ( (p+i-1)>=types.size() ) t = new CommonToken(-1);
t = new CommonToken(types.get(p+i-1));
t.setTokenIndex(p+i-1);
int rawIndex = p + i - 1;
if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF);
else t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}