jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Rename/move file(s) from old antlr4 to antlr-rewrite; that's not antlr4 anymore 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 7912]
This commit is contained in:
parrt 2011-03-26 11:33:02 -08:00
parent 360bd17f21
commit 36fb229434
318 changed files with 0 additions and 72006 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
build.xml
View File

@ -1,5 +0,0 @@
<project name="ANTLRv4">
<property file="build.properties"/>
</project>

43
gunit/resources/org/antlr/v4/gunit/jUnit.stg
View File

@ -1,43 +0,0 @@
group jUnit;
jUnitClass(className, header, options, suites) ::= <<
<header>
import org.antlr.runtime.*;
import org.antlr.runtime.tree.*;
import org.junit.Test;
import org.junit.Before;
import static org.junit.Assert.*;
public class <className> extends org.antlr.v4.gunit.gUnitBase {
@Before public void setup() {
lexerClassName = "<options.lexer>";
parserClassName = "<options.parser>";
<if(options.adaptor)>
adaptorClassName = "<options.adaptor>";
<endif>
}
<suites>
}
>>
header(action) ::= "<action>"
testSuite(name,cases) ::= <<
<cases:{c | <c>}; separator="\n\n"> <! use {...} iterator to get <i> !>
>>
parserRuleTestSuccess(input,expecting) ::= <<
>>
parserRuleTestAST(ruleName,scriptLine,input,expecting) ::= <<
@Test public void test_<name><i>() throws Exception {
// gunit test on line <scriptLine>
RuleReturnScope rstruct = (RuleReturnScope)execParser("<ruleName>", "<input>", <scriptLine>);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "<expecting>";
assertEquals("testing rule <ruleName>", expecting, actual);
}
>>
string(s) ::= "<s>"

46
gunit/src/org/antlr/v4/gunit/ASTVerifier.g
View File

@ -1,46 +0,0 @@
tree grammar ASTVerifier;
options {
ASTLabelType=CommonTree;
tokenVocab = gUnit;
}
@header {
package org.antlr.v4.gunit;
}
gUnitDef
: ^('gunit' ID DOC_COMMENT? (optionsSpec|header)* testsuite+)
;
optionsSpec
: ^(OPTIONS option+)
;
option
: ^('=' ID ID)
| ^('=' ID STRING)
;
header : ^('@header' ACTION);
testsuite
: ^(SUITE ID ID DOC_COMMENT? testcase+)
| ^(SUITE ID DOC_COMMENT? testcase+)
;
testcase
: ^(TEST_OK DOC_COMMENT? input)
| ^(TEST_FAIL DOC_COMMENT? input)
| ^(TEST_RETVAL DOC_COMMENT? input RETVAL)
| ^(TEST_STDOUT DOC_COMMENT? input STRING)
| ^(TEST_STDOUT DOC_COMMENT? input ML_STRING)
| ^(TEST_TREE DOC_COMMENT? input TREE)
| ^(TEST_ACTION DOC_COMMENT? input ACTION)
;
input
: STRING
| ML_STRING
| FILENAME
;

981
gunit/src/org/antlr/v4/gunit/ASTVerifier.java
View File

@ -1,981 +0,0 @@
// $ANTLR 3.2.1-SNAPSHOT Jan 26, 2010 15:12:28 ASTVerifier.g 2010-01-27 17:03:31
package org.antlr.v4.gunit;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.*;import java.util.Stack;
import java.util.List;
import java.util.ArrayList;
public class ASTVerifier extends TreeParser {
public static final String[] tokenNames = new String[] {
"<invalid>", "<EOR>", "<DOWN>", "<UP>", "SUITE", "TEST_OK", "TEST_FAIL", "TEST_RETVAL", "TEST_STDOUT", "TEST_TREE", "TEST_ACTION", "DOC_COMMENT", "ID", "OPTIONS", "STRING", "ACTION", "RETVAL", "ML_STRING", "TREE", "FILENAME", "NESTED_RETVAL", "NESTED_AST", "STRING_", "WS", "ID_", "SL_COMMENT", "ML_COMMENT", "XDIGIT", "'gunit'", "';'", "'}'", "'='", "'@header'", "'walks'", "':'", "'OK'", "'FAIL'", "'returns'", "'->'"
};
public static final int T__29=29;
public static final int T__28=28;
public static final int RETVAL=16;
public static final int TEST_TREE=9;
public static final int STRING_=22;
public static final int NESTED_AST=21;
public static final int ML_STRING=17;
public static final int TEST_FAIL=6;
public static final int ID=12;
public static final int EOF=-1;
public static final int NESTED_RETVAL=20;
public static final int TEST_RETVAL=7;
public static final int TEST_STDOUT=8;
public static final int ACTION=15;
public static final int TEST_OK=5;
public static final int ML_COMMENT=26;
public static final int T__30=30;
public static final int T__31=31;
public static final int T__32=32;
public static final int T__33=33;
public static final int WS=23;
public static final int T__34=34;
public static final int T__35=35;
public static final int T__36=36;
public static final int TREE=18;
public static final int T__37=37;
public static final int T__38=38;
public static final int FILENAME=19;
public static final int ID_=24;
public static final int XDIGIT=27;
public static final int SL_COMMENT=25;
public static final int DOC_COMMENT=11;
public static final int TEST_ACTION=10;
public static final int SUITE=4;
public static final int OPTIONS=13;
public static final int STRING=14;
// delegates
// delegators
public ASTVerifier(TreeNodeStream input) {
this(input, new RecognizerSharedState());
}
public ASTVerifier(TreeNodeStream input, RecognizerSharedState state) {
super(input, state);
}
public String[] getTokenNames() { return ASTVerifier.tokenNames; }
public String getGrammarFileName() { return "ASTVerifier.g"; }
// $ANTLR start "gUnitDef"
// ASTVerifier.g:12:1: gUnitDef : ^( 'gunit' ID ( DOC_COMMENT )? ( optionsSpec | header )* ( testsuite )+ ) ;
public final void gUnitDef() throws RecognitionException {
try {
// ASTVerifier.g:13:2: ( ^( 'gunit' ID ( DOC_COMMENT )? ( optionsSpec | header )* ( testsuite )+ ) )
// ASTVerifier.g:13:4: ^( 'gunit' ID ( DOC_COMMENT )? ( optionsSpec | header )* ( testsuite )+ )
{
match(input,28,FOLLOW_28_in_gUnitDef39);
match(input, Token.DOWN, null);
match(input,ID,FOLLOW_ID_in_gUnitDef41);
// ASTVerifier.g:13:17: ( DOC_COMMENT )?
int alt1=2;
int LA1_0 = input.LA(1);
if ( (LA1_0==DOC_COMMENT) ) {
alt1=1;
}
switch (alt1) {
case 1 :
// ASTVerifier.g:13:17: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_gUnitDef43);
}
break;
}
// ASTVerifier.g:13:30: ( optionsSpec | header )*
loop2:
do {
int alt2=3;
int LA2_0 = input.LA(1);
if ( (LA2_0==OPTIONS) ) {
alt2=1;
}
else if ( (LA2_0==32) ) {
alt2=2;
}
switch (alt2) {
case 1 :
// ASTVerifier.g:13:31: optionsSpec
{
pushFollow(FOLLOW_optionsSpec_in_gUnitDef47);
optionsSpec();
state._fsp--;
}
break;
case 2 :
// ASTVerifier.g:13:43: header
{
pushFollow(FOLLOW_header_in_gUnitDef49);
header();
state._fsp--;
}
break;
default :
break loop2;
}
} while (true);
// ASTVerifier.g:13:52: ( testsuite )+
int cnt3=0;
loop3:
do {
int alt3=2;
int LA3_0 = input.LA(1);
if ( (LA3_0==SUITE) ) {
alt3=1;
}
switch (alt3) {
case 1 :
// ASTVerifier.g:13:52: testsuite
{
pushFollow(FOLLOW_testsuite_in_gUnitDef53);
testsuite();
state._fsp--;
}
break;
default :
if ( cnt3 >= 1 ) break loop3;
EarlyExitException eee =
new EarlyExitException(3, input);
throw eee;
}
cnt3++;
} while (true);
match(input, Token.UP, null);
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "gUnitDef"
// $ANTLR start "optionsSpec"
// ASTVerifier.g:16:1: optionsSpec : ^( OPTIONS ( option )+ ) ;
public final void optionsSpec() throws RecognitionException {
try {
// ASTVerifier.g:17:2: ( ^( OPTIONS ( option )+ ) )
// ASTVerifier.g:17:4: ^( OPTIONS ( option )+ )
{
match(input,OPTIONS,FOLLOW_OPTIONS_in_optionsSpec67);
match(input, Token.DOWN, null);
// ASTVerifier.g:17:14: ( option )+
int cnt4=0;
loop4:
do {
int alt4=2;
int LA4_0 = input.LA(1);
if ( (LA4_0==31) ) {
alt4=1;
}
switch (alt4) {
case 1 :
// ASTVerifier.g:17:14: option
{
pushFollow(FOLLOW_option_in_optionsSpec69);
option();
state._fsp--;
}
break;
default :
if ( cnt4 >= 1 ) break loop4;
EarlyExitException eee =
new EarlyExitException(4, input);
throw eee;
}
cnt4++;
} while (true);
match(input, Token.UP, null);
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "optionsSpec"
// $ANTLR start "option"
// ASTVerifier.g:20:1: option : ( ^( '=' ID ID ) | ^( '=' ID STRING ) );
public final void option() throws RecognitionException {
try {
// ASTVerifier.g:21:5: ( ^( '=' ID ID ) | ^( '=' ID STRING ) )
int alt5=2;
int LA5_0 = input.LA(1);
if ( (LA5_0==31) ) {
int LA5_1 = input.LA(2);
if ( (LA5_1==DOWN) ) {
int LA5_2 = input.LA(3);
if ( (LA5_2==ID) ) {
int LA5_3 = input.LA(4);
if ( (LA5_3==ID) ) {
alt5=1;
}
else if ( (LA5_3==STRING) ) {
alt5=2;
}
else {
NoViableAltException nvae =
new NoViableAltException("", 5, 3, input);
throw nvae;
}
}
else {
NoViableAltException nvae =
new NoViableAltException("", 5, 2, input);
throw nvae;
}
}
else {
NoViableAltException nvae =
new NoViableAltException("", 5, 1, input);
throw nvae;
}
}
else {
NoViableAltException nvae =
new NoViableAltException("", 5, 0, input);
throw nvae;
}
switch (alt5) {
case 1 :
// ASTVerifier.g:21:9: ^( '=' ID ID )
{
match(input,31,FOLLOW_31_in_option88);
match(input, Token.DOWN, null);
match(input,ID,FOLLOW_ID_in_option90);
match(input,ID,FOLLOW_ID_in_option92);
match(input, Token.UP, null);
}
break;
case 2 :
// ASTVerifier.g:22:9: ^( '=' ID STRING )
{
match(input,31,FOLLOW_31_in_option104);
match(input, Token.DOWN, null);
match(input,ID,FOLLOW_ID_in_option106);
match(input,STRING,FOLLOW_STRING_in_option108);
match(input, Token.UP, null);
}
break;
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "option"
// $ANTLR start "header"
// ASTVerifier.g:25:1: header : ^( '@header' ACTION ) ;
public final void header() throws RecognitionException {
try {
// ASTVerifier.g:25:8: ( ^( '@header' ACTION ) )
// ASTVerifier.g:25:10: ^( '@header' ACTION )
{
match(input,32,FOLLOW_32_in_header125);
match(input, Token.DOWN, null);
match(input,ACTION,FOLLOW_ACTION_in_header127);
match(input, Token.UP, null);
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "header"
// $ANTLR start "testsuite"
// ASTVerifier.g:27:1: testsuite : ( ^( SUITE ID ID ( DOC_COMMENT )? ( testcase )+ ) | ^( SUITE ID ( DOC_COMMENT )? ( testcase )+ ) );
public final void testsuite() throws RecognitionException {
try {
// ASTVerifier.g:28:2: ( ^( SUITE ID ID ( DOC_COMMENT )? ( testcase )+ ) | ^( SUITE ID ( DOC_COMMENT )? ( testcase )+ ) )
int alt10=2;
int LA10_0 = input.LA(1);
if ( (LA10_0==SUITE) ) {
int LA10_1 = input.LA(2);
if ( (LA10_1==DOWN) ) {
int LA10_2 = input.LA(3);
if ( (LA10_2==ID) ) {
int LA10_3 = input.LA(4);
if ( (LA10_3==ID) ) {
alt10=1;
}
else if ( ((LA10_3>=TEST_OK && LA10_3<=DOC_COMMENT)) ) {
alt10=2;
}
else {
NoViableAltException nvae =
new NoViableAltException("", 10, 3, input);
throw nvae;
}
}
else {
NoViableAltException nvae =
new NoViableAltException("", 10, 2, input);
throw nvae;
}
}
else {
NoViableAltException nvae =
new NoViableAltException("", 10, 1, input);
throw nvae;
}
}
else {
NoViableAltException nvae =
new NoViableAltException("", 10, 0, input);
throw nvae;
}
switch (alt10) {
case 1 :
// ASTVerifier.g:28:4: ^( SUITE ID ID ( DOC_COMMENT )? ( testcase )+ )
{
match(input,SUITE,FOLLOW_SUITE_in_testsuite138);
match(input, Token.DOWN, null);
match(input,ID,FOLLOW_ID_in_testsuite140);
match(input,ID,FOLLOW_ID_in_testsuite142);
// ASTVerifier.g:28:18: ( DOC_COMMENT )?
int alt6=2;
int LA6_0 = input.LA(1);
if ( (LA6_0==DOC_COMMENT) ) {
alt6=1;
}
switch (alt6) {
case 1 :
// ASTVerifier.g:28:18: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testsuite144);
}
break;
}
// ASTVerifier.g:28:31: ( testcase )+
int cnt7=0;
loop7:
do {
int alt7=2;
int LA7_0 = input.LA(1);
if ( ((LA7_0>=TEST_OK && LA7_0<=TEST_ACTION)) ) {
alt7=1;
}
switch (alt7) {
case 1 :
// ASTVerifier.g:28:31: testcase
{
pushFollow(FOLLOW_testcase_in_testsuite147);
testcase();
state._fsp--;
}
break;
default :
if ( cnt7 >= 1 ) break loop7;
EarlyExitException eee =
new EarlyExitException(7, input);
throw eee;
}
cnt7++;
} while (true);
match(input, Token.UP, null);
}
break;
case 2 :
// ASTVerifier.g:29:4: ^( SUITE ID ( DOC_COMMENT )? ( testcase )+ )
{
match(input,SUITE,FOLLOW_SUITE_in_testsuite155);
match(input, Token.DOWN, null);
match(input,ID,FOLLOW_ID_in_testsuite157);
// ASTVerifier.g:29:15: ( DOC_COMMENT )?
int alt8=2;
int LA8_0 = input.LA(1);
if ( (LA8_0==DOC_COMMENT) ) {
alt8=1;
}
switch (alt8) {
case 1 :
// ASTVerifier.g:29:15: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testsuite159);
}
break;
}
// ASTVerifier.g:29:28: ( testcase )+
int cnt9=0;
loop9:
do {
int alt9=2;
int LA9_0 = input.LA(1);
if ( ((LA9_0>=TEST_OK && LA9_0<=TEST_ACTION)) ) {
alt9=1;
}
switch (alt9) {
case 1 :
// ASTVerifier.g:29:28: testcase
{
pushFollow(FOLLOW_testcase_in_testsuite162);
testcase();
state._fsp--;
}
break;
default :
if ( cnt9 >= 1 ) break loop9;
EarlyExitException eee =
new EarlyExitException(9, input);
throw eee;
}
cnt9++;
} while (true);
match(input, Token.UP, null);
}
break;
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "testsuite"
// $ANTLR start "testcase"
// ASTVerifier.g:32:1: testcase : ( ^( TEST_OK ( DOC_COMMENT )? input ) | ^( TEST_FAIL ( DOC_COMMENT )? input ) | ^( TEST_RETVAL ( DOC_COMMENT )? input RETVAL ) | ^( TEST_STDOUT ( DOC_COMMENT )? input STRING ) | ^( TEST_STDOUT ( DOC_COMMENT )? input ML_STRING ) | ^( TEST_TREE ( DOC_COMMENT )? input TREE ) | ^( TEST_ACTION ( DOC_COMMENT )? input ACTION ) );
public final void testcase() throws RecognitionException {
try {
// ASTVerifier.g:33:2: ( ^( TEST_OK ( DOC_COMMENT )? input ) | ^( TEST_FAIL ( DOC_COMMENT )? input ) | ^( TEST_RETVAL ( DOC_COMMENT )? input RETVAL ) | ^( TEST_STDOUT ( DOC_COMMENT )? input STRING ) | ^( TEST_STDOUT ( DOC_COMMENT )? input ML_STRING ) | ^( TEST_TREE ( DOC_COMMENT )? input TREE ) | ^( TEST_ACTION ( DOC_COMMENT )? input ACTION ) )
int alt18=7;
alt18 = dfa18.predict(input);
switch (alt18) {
case 1 :
// ASTVerifier.g:33:4: ^( TEST_OK ( DOC_COMMENT )? input )
{
match(input,TEST_OK,FOLLOW_TEST_OK_in_testcase176);
match(input, Token.DOWN, null);
// ASTVerifier.g:33:14: ( DOC_COMMENT )?
int alt11=2;
int LA11_0 = input.LA(1);
if ( (LA11_0==DOC_COMMENT) ) {
alt11=1;
}
switch (alt11) {
case 1 :
// ASTVerifier.g:33:14: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testcase178);
}
break;
}
pushFollow(FOLLOW_input_in_testcase181);
input();
state._fsp--;
match(input, Token.UP, null);
}
break;
case 2 :
// ASTVerifier.g:34:4: ^( TEST_FAIL ( DOC_COMMENT )? input )
{
match(input,TEST_FAIL,FOLLOW_TEST_FAIL_in_testcase188);
match(input, Token.DOWN, null);
// ASTVerifier.g:34:16: ( DOC_COMMENT )?
int alt12=2;
int LA12_0 = input.LA(1);
if ( (LA12_0==DOC_COMMENT) ) {
alt12=1;
}
switch (alt12) {
case 1 :
// ASTVerifier.g:34:16: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testcase190);
}
break;
}
pushFollow(FOLLOW_input_in_testcase193);
input();
state._fsp--;
match(input, Token.UP, null);
}
break;
case 3 :
// ASTVerifier.g:35:4: ^( TEST_RETVAL ( DOC_COMMENT )? input RETVAL )
{
match(input,TEST_RETVAL,FOLLOW_TEST_RETVAL_in_testcase200);
match(input, Token.DOWN, null);
// ASTVerifier.g:35:18: ( DOC_COMMENT )?
int alt13=2;
int LA13_0 = input.LA(1);
if ( (LA13_0==DOC_COMMENT) ) {
alt13=1;
}
switch (alt13) {
case 1 :
// ASTVerifier.g:35:18: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testcase202);
}
break;
}
pushFollow(FOLLOW_input_in_testcase205);
input();
state._fsp--;
match(input,RETVAL,FOLLOW_RETVAL_in_testcase207);
match(input, Token.UP, null);
}
break;
case 4 :
// ASTVerifier.g:36:4: ^( TEST_STDOUT ( DOC_COMMENT )? input STRING )
{
match(input,TEST_STDOUT,FOLLOW_TEST_STDOUT_in_testcase214);
match(input, Token.DOWN, null);
// ASTVerifier.g:36:18: ( DOC_COMMENT )?
int alt14=2;
int LA14_0 = input.LA(1);
if ( (LA14_0==DOC_COMMENT) ) {
alt14=1;
}
switch (alt14) {
case 1 :
// ASTVerifier.g:36:18: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testcase216);
}
break;
}
pushFollow(FOLLOW_input_in_testcase219);
input();
state._fsp--;
match(input,STRING,FOLLOW_STRING_in_testcase221);
match(input, Token.UP, null);
}
break;
case 5 :
// ASTVerifier.g:37:4: ^( TEST_STDOUT ( DOC_COMMENT )? input ML_STRING )
{
match(input,TEST_STDOUT,FOLLOW_TEST_STDOUT_in_testcase228);
match(input, Token.DOWN, null);
// ASTVerifier.g:37:18: ( DOC_COMMENT )?
int alt15=2;
int LA15_0 = input.LA(1);
if ( (LA15_0==DOC_COMMENT) ) {
alt15=1;
}
switch (alt15) {
case 1 :
// ASTVerifier.g:37:18: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testcase230);
}
break;
}
pushFollow(FOLLOW_input_in_testcase233);
input();
state._fsp--;
match(input,ML_STRING,FOLLOW_ML_STRING_in_testcase235);
match(input, Token.UP, null);
}
break;
case 6 :
// ASTVerifier.g:38:4: ^( TEST_TREE ( DOC_COMMENT )? input TREE )
{
match(input,TEST_TREE,FOLLOW_TEST_TREE_in_testcase242);
match(input, Token.DOWN, null);
// ASTVerifier.g:38:16: ( DOC_COMMENT )?
int alt16=2;
int LA16_0 = input.LA(1);
if ( (LA16_0==DOC_COMMENT) ) {
alt16=1;
}
switch (alt16) {
case 1 :
// ASTVerifier.g:38:16: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testcase244);
}
break;
}
pushFollow(FOLLOW_input_in_testcase247);
input();
state._fsp--;
match(input,TREE,FOLLOW_TREE_in_testcase249);
match(input, Token.UP, null);
}
break;
case 7 :
// ASTVerifier.g:39:4: ^( TEST_ACTION ( DOC_COMMENT )? input ACTION )
{
match(input,TEST_ACTION,FOLLOW_TEST_ACTION_in_testcase256);
match(input, Token.DOWN, null);
// ASTVerifier.g:39:18: ( DOC_COMMENT )?
int alt17=2;
int LA17_0 = input.LA(1);
if ( (LA17_0==DOC_COMMENT) ) {
alt17=1;
}
switch (alt17) {
case 1 :
// ASTVerifier.g:39:18: DOC_COMMENT
{
match(input,DOC_COMMENT,FOLLOW_DOC_COMMENT_in_testcase258);
}
break;
}
pushFollow(FOLLOW_input_in_testcase261);
input();
state._fsp--;
match(input,ACTION,FOLLOW_ACTION_in_testcase263);
match(input, Token.UP, null);
}
break;
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "testcase"
// $ANTLR start "input"
// ASTVerifier.g:42:1: input : ( STRING | ML_STRING | FILENAME );
public final void input() throws RecognitionException {
try {
// ASTVerifier.g:43:2: ( STRING | ML_STRING | FILENAME )
// ASTVerifier.g:
{
if ( input.LA(1)==STRING||input.LA(1)==ML_STRING||input.LA(1)==FILENAME ) {
input.consume();
state.errorRecovery=false;
}
else {
MismatchedSetException mse = new MismatchedSetException(null,input);
throw mse;
}
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "input"
// Delegated rules
protected DFA18 dfa18 = new DFA18(this);
static final String DFA18_eotS =
"\14\uffff";
static final String DFA18_eofS =
"\14\uffff";
static final String DFA18_minS =
"\1\5\3\uffff\1\2\2\uffff\1\13\2\16\2\uffff";
static final String DFA18_maxS =
"\1\12\3\uffff\1\2\2\uffff\2\23\1\21\2\uffff";
static final String DFA18_acceptS =
"\1\uffff\1\1\1\2\1\3\1\uffff\1\6\1\7\3\uffff\1\4\1\5";
static final String DFA18_specialS =
"\14\uffff}>";
static final String[] DFA18_transitionS = {
"\1\1\1\2\1\3\1\4\1\5\1\6",
"",
"",
"",
"\1\7",
"",
"",
"\1\10\2\uffff\1\11\2\uffff\1\11\1\uffff\1\11",
"\1\11\2\uffff\1\11\1\uffff\1\11",
"\1\12\2\uffff\1\13",
"",
""
};
static final short[] DFA18_eot = DFA.unpackEncodedString(DFA18_eotS);
static final short[] DFA18_eof = DFA.unpackEncodedString(DFA18_eofS);
static final char[] DFA18_min = DFA.unpackEncodedStringToUnsignedChars(DFA18_minS);
static final char[] DFA18_max = DFA.unpackEncodedStringToUnsignedChars(DFA18_maxS);
static final short[] DFA18_accept = DFA.unpackEncodedString(DFA18_acceptS);
static final short[] DFA18_special = DFA.unpackEncodedString(DFA18_specialS);
static final short[][] DFA18_transition;
static {
int numStates = DFA18_transitionS.length;
DFA18_transition = new short[numStates][];
for (int i=0; i<numStates; i++) {
DFA18_transition[i] = DFA.unpackEncodedString(DFA18_transitionS[i]);
}
}
class DFA18 extends DFA {
public DFA18(BaseRecognizer recognizer) {
this.recognizer = recognizer;
this.decisionNumber = 18;
this.eot = DFA18_eot;
this.eof = DFA18_eof;
this.min = DFA18_min;
this.max = DFA18_max;
this.accept = DFA18_accept;
this.special = DFA18_special;
this.transition = DFA18_transition;
}
public String getDescription() {
return "32:1: testcase : ( ^( TEST_OK ( DOC_COMMENT )? input ) | ^( TEST_FAIL ( DOC_COMMENT )? input ) | ^( TEST_RETVAL ( DOC_COMMENT )? input RETVAL ) | ^( TEST_STDOUT ( DOC_COMMENT )? input STRING ) | ^( TEST_STDOUT ( DOC_COMMENT )? input ML_STRING ) | ^( TEST_TREE ( DOC_COMMENT )? input TREE ) | ^( TEST_ACTION ( DOC_COMMENT )? input ACTION ) );";
}
}
public static final BitSet FOLLOW_28_in_gUnitDef39 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_gUnitDef41 = new BitSet(new long[]{0x0000000100002810L});
public static final BitSet FOLLOW_DOC_COMMENT_in_gUnitDef43 = new BitSet(new long[]{0x0000000100002810L});
public static final BitSet FOLLOW_optionsSpec_in_gUnitDef47 = new BitSet(new long[]{0x0000000100002810L});
public static final BitSet FOLLOW_header_in_gUnitDef49 = new BitSet(new long[]{0x0000000100002810L});
public static final BitSet FOLLOW_testsuite_in_gUnitDef53 = new BitSet(new long[]{0x0000000100002818L});
public static final BitSet FOLLOW_OPTIONS_in_optionsSpec67 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_option_in_optionsSpec69 = new BitSet(new long[]{0x0000000080000008L});
public static final BitSet FOLLOW_31_in_option88 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_option90 = new BitSet(new long[]{0x0000000000001000L});
public static final BitSet FOLLOW_ID_in_option92 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_31_in_option104 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_option106 = new BitSet(new long[]{0x0000000000004000L});
public static final BitSet FOLLOW_STRING_in_option108 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_32_in_header125 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ACTION_in_header127 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_SUITE_in_testsuite138 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_testsuite140 = new BitSet(new long[]{0x0000000000001000L});
public static final BitSet FOLLOW_ID_in_testsuite142 = new BitSet(new long[]{0x0000000000000FE0L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testsuite144 = new BitSet(new long[]{0x0000000000000FE0L});
public static final BitSet FOLLOW_testcase_in_testsuite147 = new BitSet(new long[]{0x0000000000000FE8L});
public static final BitSet FOLLOW_SUITE_in_testsuite155 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_testsuite157 = new BitSet(new long[]{0x0000000000000FE0L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testsuite159 = new BitSet(new long[]{0x0000000000000FE0L});
public static final BitSet FOLLOW_testcase_in_testsuite162 = new BitSet(new long[]{0x0000000000000FE8L});
public static final BitSet FOLLOW_TEST_OK_in_testcase176 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testcase178 = new BitSet(new long[]{0x00000000000A4000L});
public static final BitSet FOLLOW_input_in_testcase181 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_TEST_FAIL_in_testcase188 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testcase190 = new BitSet(new long[]{0x00000000000A4000L});
public static final BitSet FOLLOW_input_in_testcase193 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_TEST_RETVAL_in_testcase200 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testcase202 = new BitSet(new long[]{0x00000000000A4000L});
public static final BitSet FOLLOW_input_in_testcase205 = new BitSet(new long[]{0x0000000000010000L});
public static final BitSet FOLLOW_RETVAL_in_testcase207 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_TEST_STDOUT_in_testcase214 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testcase216 = new BitSet(new long[]{0x00000000000A4000L});
public static final BitSet FOLLOW_input_in_testcase219 = new BitSet(new long[]{0x0000000000004000L});
public static final BitSet FOLLOW_STRING_in_testcase221 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_TEST_STDOUT_in_testcase228 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testcase230 = new BitSet(new long[]{0x00000000000A4000L});
public static final BitSet FOLLOW_input_in_testcase233 = new BitSet(new long[]{0x0000000000020000L});
public static final BitSet FOLLOW_ML_STRING_in_testcase235 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_TEST_TREE_in_testcase242 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testcase244 = new BitSet(new long[]{0x00000000000A4000L});
public static final BitSet FOLLOW_input_in_testcase247 = new BitSet(new long[]{0x0000000000040000L});
public static final BitSet FOLLOW_TREE_in_testcase249 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_TEST_ACTION_in_testcase256 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_DOC_COMMENT_in_testcase258 = new BitSet(new long[]{0x00000000000A4000L});
public static final BitSet FOLLOW_input_in_testcase261 = new BitSet(new long[]{0x0000000000008000L});
public static final BitSet FOLLOW_ACTION_in_testcase263 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_set_in_input0 = new BitSet(new long[]{0x0000000000000002L});
}

151
gunit/src/org/antlr/v4/gunit/Gen.java
View File

@ -1,151 +0,0 @@
package org.antlr.v4.gunit;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.BufferedTreeNodeStream;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.stringtemplate.AutoIndentWriter;
import org.antlr.stringtemplate.StringTemplate;
import org.antlr.stringtemplate.StringTemplateGroup;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
public class Gen {
// TODO: don't hardcode
public static final String TEMPLATE_FILE =
"/Users/parrt/antlr/code/antlr4/main/gunit/resources/org/antlr/v4/gunit/jUnit.stg";
public static void main(String[] args) throws Exception {
if ( args.length==0 ) System.exit(0);
String outputDirName = ".";
String fileName = args[0];
if ( args[0].equals("-o") ) {
if ( args.length<3 ) {
help();
System.exit(0);
}
outputDirName = args[1];
fileName = args[2];
}
new Gen().process(fileName, outputDirName);
}
public void process(String fileName, String outputDirName) throws Exception {
// PARSE SCRIPT
ANTLRFileStream fs = new ANTLRFileStream(fileName);
gUnitLexer lexer = new gUnitLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
gUnitParser parser = new gUnitParser(tokens);
RuleReturnScope r = parser.gUnitDef();
CommonTree scriptAST = (CommonTree)r.getTree();
System.out.println(scriptAST.toStringTree());
// ANALYZE
CommonTreeNodeStream nodes = new CommonTreeNodeStream(r.getTree());
Semantics sem = new Semantics(nodes);
sem.downup(scriptAST);
System.out.println("options="+sem.options);
// GENERATE CODE
FileReader fr = new FileReader(TEMPLATE_FILE);
StringTemplateGroup templates =
new StringTemplateGroup(fr);
fr.close();
BufferedTreeNodeStream bnodes = new BufferedTreeNodeStream(scriptAST);
jUnitGen gen = new jUnitGen(bnodes);
gen.setTemplateLib(templates);
RuleReturnScope r2 = gen.gUnitDef();
StringTemplate st = (StringTemplate)r2.getTemplate();
st.setAttribute("options", sem.options);
FileWriter fw = new FileWriter(outputDirName+"/"+sem.name+".java");
BufferedWriter bw = new BufferedWriter(fw);
st.write(new AutoIndentWriter(bw));
bw.close();
}
/** Borrowed from Leon Su in gunit v3 */
public static String escapeForJava(String inputString) {
// Gotta escape literal backslash before putting in specials that use escape.
inputString = inputString.replace("\\", "\\\\");
// Then double quotes need escaping (singles are OK of course).
inputString = inputString.replace("\"", "\\\"");
// note: replace newline to String ".\n", replace tab to String ".\t"
inputString = inputString.replace("\n", "\\n").replace("\t", "\\t").replace("\r", "\\r").replace("\b", "\\b").replace("\f", "\\f");
return inputString;
}
public static String normalizeTreeSpec(String t) {
List<String> words = new ArrayList<String>();
int i = 0;
StringBuilder word = new StringBuilder();
while ( i<t.length() ) {
if ( t.charAt(i)=='(' || t.charAt(i)==')' ) {
if ( word.length()>0 ) {
words.add(word.toString());
word.setLength(0);
}
words.add(String.valueOf(t.charAt(i)));
i++;
continue;
}
if ( Character.isWhitespace(t.charAt(i)) ) {
// upon WS, save word
if ( word.length()>0 ) {
words.add(word.toString());
word.setLength(0);
}
i++;
continue;
}
// ... "x" or ...("x"
if ( t.charAt(i)=='"' && (i-1)>=0 &&
(t.charAt(i-1)=='(' || Character.isWhitespace(t.charAt(i-1))) )
{
i++;
while ( i<t.length() && t.charAt(i)!='"' ) {
if ( t.charAt(i)=='\\' &&
(i+1)<t.length() && t.charAt(i+1)=='"' ) // handle \"
{
word.append('"');
i+=2;
continue;
}
word.append(t.charAt(i));
i++;
}
i++; // skip final "
words.add(word.toString());
word.setLength(0);
continue;
}
word.append(t.charAt(i));
i++;
}
if ( word.length()>0 ) {
words.add(word.toString());
}
//System.out.println("words="+words);
StringBuilder buf = new StringBuilder();
for (int j=0; j<words.size(); j++) {
if ( j>0 && !words.get(j).equals(")") &&
!words.get(j-1).equals("(") ) {
buf.append(' ');
}
buf.append(words.get(j));
}
return buf.toString();
}
public static void help() {
System.err.println("org.antlr.v4.gunit.Gen [-o output-dir] gunit-file");
}
}

21
gunit/src/org/antlr/v4/gunit/Interp.java
View File

@ -1,21 +0,0 @@
package org.antlr.v4.gunit;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.BufferedTreeNodeStream;
import org.antlr.runtime.tree.Tree;
public class Interp {
public static void main(String[] args) throws Exception {
String fileName = args[0];
ANTLRFileStream fs = new ANTLRFileStream(fileName);
gUnitLexer lexer = new gUnitLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
gUnitParser parser = new gUnitParser(tokens);
RuleReturnScope r = parser.gUnitDef();
System.out.println(((Tree)r.getTree()).toStringTree());
BufferedTreeNodeStream nodes = new BufferedTreeNodeStream(r.getTree());
ASTVerifier verifier = new ASTVerifier(nodes);
verifier.gUnitDef();
}
}

36
gunit/src/org/antlr/v4/gunit/Semantics.g
View File

@ -1,36 +0,0 @@
tree grammar Semantics;
options {
filter=true;
ASTLabelType=CommonTree;
tokenVocab = gUnit;
}
@header {
package org.antlr.v4.gunit;
import java.util.Map;
import java.util.HashMap;
}
@members {
public String name;
public Map<String,String> options = new HashMap<String,String>();
}
topdown
: optionsSpec
| gUnitDef
;
gUnitDef
: ^('gunit' ID .*) {name = $ID.text;}
;
optionsSpec
: ^(OPTIONS option+)
;
option
: ^('=' o=ID v=ID) {options.put($o.text, $v.text);}
| ^('=' o=ID v=STRING) {options.put($o.text, $v.text);}
;

379
gunit/src/org/antlr/v4/gunit/Semantics.java
View File

@ -1,379 +0,0 @@
// $ANTLR 3.2.1-SNAPSHOT Jan 26, 2010 15:12:28 Semantics.g 2010-01-27 17:03:31
package org.antlr.v4.gunit;
import java.util.Map;
import java.util.HashMap;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.*;import java.util.Stack;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
public class Semantics extends TreeFilter {
public static final String[] tokenNames = new String[] {
"<invalid>", "<EOR>", "<DOWN>", "<UP>", "SUITE", "TEST_OK", "TEST_FAIL", "TEST_RETVAL", "TEST_STDOUT", "TEST_TREE", "TEST_ACTION", "DOC_COMMENT", "ID", "OPTIONS", "STRING", "ACTION", "RETVAL", "ML_STRING", "TREE", "FILENAME", "NESTED_RETVAL", "NESTED_AST", "STRING_", "WS", "ID_", "SL_COMMENT", "ML_COMMENT", "XDIGIT", "'gunit'", "';'", "'}'", "'='", "'@header'", "'walks'", "':'", "'OK'", "'FAIL'", "'returns'", "'->'"
};
public static final int T__29=29;
public static final int T__28=28;
public static final int RETVAL=16;
public static final int TEST_TREE=9;
public static final int STRING_=22;
public static final int NESTED_AST=21;
public static final int ML_STRING=17;
public static final int TEST_FAIL=6;
public static final int ID=12;
public static final int EOF=-1;
public static final int NESTED_RETVAL=20;
public static final int TEST_RETVAL=7;
public static final int TEST_STDOUT=8;
public static final int ACTION=15;
public static final int TEST_OK=5;
public static final int ML_COMMENT=26;
public static final int T__30=30;
public static final int T__31=31;
public static final int T__32=32;
public static final int T__33=33;
public static final int WS=23;
public static final int T__34=34;
public static final int T__35=35;
public static final int T__36=36;
public static final int TREE=18;
public static final int T__37=37;
public static final int T__38=38;
public static final int FILENAME=19;
public static final int ID_=24;
public static final int XDIGIT=27;
public static final int SL_COMMENT=25;
public static final int DOC_COMMENT=11;
public static final int TEST_ACTION=10;
public static final int SUITE=4;
public static final int OPTIONS=13;
public static final int STRING=14;
// delegates
// delegators
public Semantics(TreeNodeStream input) {
this(input, new RecognizerSharedState());
}
public Semantics(TreeNodeStream input, RecognizerSharedState state) {
super(input, state);
}
public String[] getTokenNames() { return Semantics.tokenNames; }
public String getGrammarFileName() { return "Semantics.g"; }
public String name;
public Map<String,String> options = new HashMap<String,String>();
// $ANTLR start "topdown"
// Semantics.g:20:1: topdown : ( optionsSpec | gUnitDef );
public final void topdown() throws RecognitionException {
try {
// Semantics.g:21:2: ( optionsSpec | gUnitDef )
int alt1=2;
int LA1_0 = input.LA(1);
if ( (LA1_0==OPTIONS) ) {
alt1=1;
}
else if ( (LA1_0==28) ) {
alt1=2;
}
else {
if (state.backtracking>0) {state.failed=true; return ;}
NoViableAltException nvae =
new NoViableAltException("", 1, 0, input);
throw nvae;
}
switch (alt1) {
case 1 :
// Semantics.g:21:4: optionsSpec
{
pushFollow(FOLLOW_optionsSpec_in_topdown50);
optionsSpec();
state._fsp--;
if (state.failed) return ;
}
break;
case 2 :
// Semantics.g:22:4: gUnitDef
{
pushFollow(FOLLOW_gUnitDef_in_topdown55);
gUnitDef();
state._fsp--;
if (state.failed) return ;
}
break;
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "topdown"
// $ANTLR start "gUnitDef"
// Semantics.g:25:1: gUnitDef : ^( 'gunit' ID ( . )* ) ;
public final void gUnitDef() throws RecognitionException {
CommonTree ID1=null;
try {
// Semantics.g:26:2: ( ^( 'gunit' ID ( . )* ) )
// Semantics.g:26:4: ^( 'gunit' ID ( . )* )
{
match(input,28,FOLLOW_28_in_gUnitDef67); if (state.failed) return ;
match(input, Token.DOWN, null); if (state.failed) return ;
ID1=(CommonTree)match(input,ID,FOLLOW_ID_in_gUnitDef69); if (state.failed) return ;
// Semantics.g:26:17: ( . )*
loop2:
do {
int alt2=2;
int LA2_0 = input.LA(1);
if ( ((LA2_0>=SUITE && LA2_0<=38)) ) {
alt2=1;
}
else if ( (LA2_0==UP) ) {
alt2=2;
}
switch (alt2) {
case 1 :
// Semantics.g:26:17: .
{
matchAny(input); if (state.failed) return ;
}
break;
default :
break loop2;
}
} while (true);
match(input, Token.UP, null); if (state.failed) return ;
if ( state.backtracking==1 ) {
name = (ID1!=null?ID1.getText():null);
}
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "gUnitDef"
// $ANTLR start "optionsSpec"
// Semantics.g:29:1: optionsSpec : ^( OPTIONS ( option )+ ) ;
public final void optionsSpec() throws RecognitionException {
try {
// Semantics.g:30:2: ( ^( OPTIONS ( option )+ ) )
// Semantics.g:30:4: ^( OPTIONS ( option )+ )
{
match(input,OPTIONS,FOLLOW_OPTIONS_in_optionsSpec88); if (state.failed) return ;
match(input, Token.DOWN, null); if (state.failed) return ;
// Semantics.g:30:14: ( option )+
int cnt3=0;
loop3:
do {
int alt3=2;
int LA3_0 = input.LA(1);
if ( (LA3_0==31) ) {
alt3=1;
}
switch (alt3) {
case 1 :
// Semantics.g:30:14: option
{
pushFollow(FOLLOW_option_in_optionsSpec90);
option();
state._fsp--;
if (state.failed) return ;
}
break;
default :
if ( cnt3 >= 1 ) break loop3;
if (state.backtracking>0) {state.failed=true; return ;}
EarlyExitException eee =
new EarlyExitException(3, input);
throw eee;
}
cnt3++;
} while (true);
match(input, Token.UP, null); if (state.failed) return ;
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "optionsSpec"
// $ANTLR start "option"
// Semantics.g:33:1: option : ( ^( '=' o= ID v= ID ) | ^( '=' o= ID v= STRING ) );
public final void option() throws RecognitionException {
CommonTree o=null;
CommonTree v=null;
try {
// Semantics.g:34:5: ( ^( '=' o= ID v= ID ) | ^( '=' o= ID v= STRING ) )
int alt4=2;
int LA4_0 = input.LA(1);
if ( (LA4_0==31) ) {
int LA4_1 = input.LA(2);
if ( (LA4_1==DOWN) ) {
int LA4_2 = input.LA(3);
if ( (LA4_2==ID) ) {
int LA4_3 = input.LA(4);
if ( (LA4_3==ID) ) {
alt4=1;
}
else if ( (LA4_3==STRING) ) {
alt4=2;
}
else {
if (state.backtracking>0) {state.failed=true; return ;}
NoViableAltException nvae =
new NoViableAltException("", 4, 3, input);
throw nvae;
}
}
else {
if (state.backtracking>0) {state.failed=true; return ;}
NoViableAltException nvae =
new NoViableAltException("", 4, 2, input);
throw nvae;
}
}
else {
if (state.backtracking>0) {state.failed=true; return ;}
NoViableAltException nvae =
new NoViableAltException("", 4, 1, input);
throw nvae;
}
}
else {
if (state.backtracking>0) {state.failed=true; return ;}
NoViableAltException nvae =
new NoViableAltException("", 4, 0, input);
throw nvae;
}
switch (alt4) {
case 1 :
// Semantics.g:34:9: ^( '=' o= ID v= ID )
{
match(input,31,FOLLOW_31_in_option109); if (state.failed) return ;
match(input, Token.DOWN, null); if (state.failed) return ;
o=(CommonTree)match(input,ID,FOLLOW_ID_in_option113); if (state.failed) return ;
v=(CommonTree)match(input,ID,FOLLOW_ID_in_option117); if (state.failed) return ;
match(input, Token.UP, null); if (state.failed) return ;
if ( state.backtracking==1 ) {
options.put((o!=null?o.getText():null), (v!=null?v.getText():null));
}
}
break;
case 2 :
// Semantics.g:35:9: ^( '=' o= ID v= STRING )
{
match(input,31,FOLLOW_31_in_option132); if (state.failed) return ;
match(input, Token.DOWN, null); if (state.failed) return ;
o=(CommonTree)match(input,ID,FOLLOW_ID_in_option136); if (state.failed) return ;
v=(CommonTree)match(input,STRING,FOLLOW_STRING_in_option140); if (state.failed) return ;
match(input, Token.UP, null); if (state.failed) return ;
if ( state.backtracking==1 ) {
options.put((o!=null?o.getText():null), (v!=null?v.getText():null));
}
}
break;
}
}
catch (RecognitionException re) {
reportError(re);
recover(input,re);
}
finally {
}
return ;
}
// $ANTLR end "option"
// Delegated rules
public static final BitSet FOLLOW_optionsSpec_in_topdown50 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_gUnitDef_in_topdown55 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_28_in_gUnitDef67 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_gUnitDef69 = new BitSet(new long[]{0x0000007FFFFFFFF8L});
public static final BitSet FOLLOW_OPTIONS_in_optionsSpec88 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_option_in_optionsSpec90 = new BitSet(new long[]{0x0000000080000008L});
public static final BitSet FOLLOW_31_in_option109 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_option113 = new BitSet(new long[]{0x0000000000001000L});
public static final BitSet FOLLOW_ID_in_option117 = new BitSet(new long[]{0x0000000000000008L});
public static final BitSet FOLLOW_31_in_option132 = new BitSet(new long[]{0x0000000000000004L});
public static final BitSet FOLLOW_ID_in_option136 = new BitSet(new long[]{0x0000000000004000L});
public static final BitSet FOLLOW_STRING_in_option140 = new BitSet(new long[]{0x0000000000000008L});
}

155
gunit/src/org/antlr/v4/gunit/gUnit.g
View File

@ -1,155 +0,0 @@
grammar gUnit;
options {
output=AST;
ASTLabelType=CommonTree;
}
tokens { SUITE; TEST_OK; TEST_FAIL; TEST_RETVAL; TEST_STDOUT; TEST_TREE; TEST_ACTION; }
@header {
package org.antlr.v4.gunit;
}
@lexer::header {
package org.antlr.v4.gunit;
}
gUnitDef
: DOC_COMMENT? 'gunit' ID ';' (optionsSpec|header)* testsuite+
-> ^('gunit' ID DOC_COMMENT? optionsSpec? header? testsuite+)
;
optionsSpec
: OPTIONS (option ';')+ '}' -> ^(OPTIONS option+)
;
option
: ID '=' optionValue -> ^('=' ID optionValue)
;
optionValue
: ID
| STRING
;
header : '@header' ACTION -> ^('@header' ACTION);
testsuite
: DOC_COMMENT? treeRule=ID 'walks' parserRule=ID ':' testcase+
-> ^(SUITE $treeRule $parserRule DOC_COMMENT? testcase+)
| DOC_COMMENT? ID ':' testcase+ -> ^(SUITE ID DOC_COMMENT? testcase+)
;
testcase
: DOC_COMMENT? input 'OK' -> ^(TEST_OK DOC_COMMENT? input)
| DOC_COMMENT? input 'FAIL' -> ^(TEST_FAIL DOC_COMMENT? input)
| DOC_COMMENT? input 'returns' RETVAL -> ^(TEST_RETVAL DOC_COMMENT? input RETVAL)
| DOC_COMMENT? input '->' STRING -> ^(TEST_STDOUT DOC_COMMENT? input STRING)
| DOC_COMMENT? input '->' ML_STRING -> ^(TEST_STDOUT DOC_COMMENT? input ML_STRING)
| DOC_COMMENT? input '->' TREE -> ^(TEST_TREE DOC_COMMENT? input TREE)
| DOC_COMMENT? input '->' ACTION -> ^(TEST_ACTION DOC_COMMENT? input ACTION)
;
input
: STRING
| ML_STRING
| FILENAME
;
ACTION
: '{' ('\\}'|'\\' ~'}'|~('\\'|'}'))* '}' {setText(getText().substring(1, getText().length()-1));}
;
RETVAL
: NESTED_RETVAL {setText(getText().substring(1, getText().length()-1));}
;
fragment
NESTED_RETVAL :
'['
( options {greedy=false;}
: NESTED_RETVAL
| .
)*
']'
;
TREE : NESTED_AST (' '? NESTED_AST)*;
fragment
NESTED_AST
: '('
( NESTED_AST
| STRING_
| ~('('|')'|'"')
)*
')'
;
OPTIONS : 'options' WS* '{' ;
ID : ID_ ('.' ID_)* ;
fragment
ID_ : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
;
WS : ( ' '
| '\t'
| '\r'
| '\n'
) {$channel=HIDDEN;}
;
SL_COMMENT
: '//' ~('\r'|'\n')* '\r'? '\n' {$channel=HIDDEN;}
;
DOC_COMMENT
: '/**' (options {greedy=false;}:.)* '*/'
;
ML_COMMENT
: '/*' ~'*' (options {greedy=false;}:.)* '*/' {$channel=HIDDEN;}
;
STRING : STRING_ {setText(getText().substring(1, getText().length()-1));} ;
fragment
STRING_
: '"' ('\\"'|'\\' ~'"'|~('\\'|'"'))+ '"'
;
ML_STRING
: '<<' .* '>>' {setText(getText().substring(2, getText().length()-2));}
;
FILENAME
: '/' ID ('/' ID)*
| ID ('/' ID)+
;
/*
fragment
ESC : '\\'
( 'n'
| 'r'
| 't'
| 'b'
| 'f'
| '"'
| '\''
| '\\'
| '>'
| 'u' XDIGIT XDIGIT XDIGIT XDIGIT
| . // unknown, leave as it is
)
;
*/
fragment
XDIGIT :
'0' .. '9'
| 'a' .. 'f'
| 'A' .. 'F'
;

49
gunit/src/org/antlr/v4/gunit/gUnitBase.java
View File

@ -1,49 +0,0 @@
package org.antlr.v4.gunit;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.TreeAdaptor;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
public class gUnitBase {
public String lexerClassName;
public String parserClassName;
public String adaptorClassName;
public Object execParser(
String ruleName,
String input,
int scriptLine)
throws Exception
{
ANTLRStringStream is = new ANTLRStringStream(input);
Class lexerClass = Class.forName(lexerClassName);
Class[] lexArgTypes = new Class[]{CharStream.class};
Constructor lexConstructor = lexerClass.getConstructor(lexArgTypes);
Object[] lexArgs = new Object[]{is};
TokenSource lexer = (TokenSource)lexConstructor.newInstance(lexArgs);
is.setLine(scriptLine);
CommonTokenStream tokens = new CommonTokenStream(lexer);
Class parserClass = Class.forName(parserClassName);
Class[] parArgTypes = new Class[]{TokenStream.class};
Constructor parConstructor = parserClass.getConstructor(parArgTypes);
Object[] parArgs = new Object[]{tokens};
Parser parser = (Parser)parConstructor.newInstance(parArgs);
// set up customized tree adaptor if necessary
if ( adaptorClassName!=null ) {
parArgTypes = new Class[]{TreeAdaptor.class};
Method m = parserClass.getMethod("setTreeAdaptor", parArgTypes);
Class adaptorClass = Class.forName(adaptorClassName);
m.invoke(parser, adaptorClass.newInstance());
}
Method ruleMethod = parserClass.getMethod(ruleName);
// INVOKE RULE
return ruleMethod.invoke(parser);
}
}

1615
gunit/src/org/antlr/v4/gunit/gUnitLexer.java
View File

File diff suppressed because it is too large Load Diff

1782
gunit/src/org/antlr/v4/gunit/gUnitParser.java
View File

File diff suppressed because it is too large Load Diff

53
gunit/src/org/antlr/v4/gunit/jUnitGen.g
View File

@ -1,53 +0,0 @@
tree grammar jUnitGen;
options {
output=template;
ASTLabelType=CommonTree;
tokenVocab = gUnit;
}
@header {
package org.antlr.v4.gunit;
}
gUnitDef
: ^('gunit' ID DOC_COMMENT? (optionsSpec|header)* suites+=testsuite+)
-> jUnitClass(className={$ID.text}, header={$header.st}, suites={$suites})
;
optionsSpec
: ^(OPTIONS option+)
;
option
: ^('=' ID ID)
| ^('=' ID STRING)
;
header : ^('@header' ACTION) -> header(action={$ACTION.text});
testsuite
: ^(SUITE rule=ID ID DOC_COMMENT? cases+=testcase[$rule.text]+)
| ^(SUITE rule=ID DOC_COMMENT? cases+=testcase[$rule.text]+)
-> testSuite(name={$rule.text}, cases={$cases})
;
testcase[String ruleName]
: ^(TEST_OK DOC_COMMENT? input)
| ^(TEST_FAIL DOC_COMMENT? input)
| ^(TEST_RETVAL DOC_COMMENT? input RETVAL)
| ^(TEST_STDOUT DOC_COMMENT? input STRING)
| ^(TEST_STDOUT DOC_COMMENT? input ML_STRING)
| ^(TEST_TREE DOC_COMMENT? input TREE)
-> parserRuleTestAST(ruleName={$ruleName},
input={$input.st},
expecting={Gen.normalizeTreeSpec($TREE.text)},
scriptLine={$input.start.getLine()})
| ^(TEST_ACTION DOC_COMMENT? input ACTION)
;
input
: STRING -> string(s={Gen.escapeForJava($STRING.text)})
| ML_STRING -> string(s={Gen.escapeForJava($ML_STRING.text)})
| FILENAME
;

1180
gunit/src/org/antlr/v4/gunit/jUnitGen.java
View File

File diff suppressed because it is too large Load Diff

6
runtime/Java/src/org/antlr/v4/runtime/ANTLRParserListener.java
View File

@ -1,6 +0,0 @@
package org.antlr.v4.runtime;
/** */
public interface ANTLRParserListener {
public void error(RecognitionException msg);
}

901
runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java
View File

@ -1,901 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.IntStream;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenStream;
import org.antlr.v4.runtime.misc.LABitSet;
import java.util.*;
/** A generic recognizer that can handle recognizers generated from
* parser and tree grammars. This is all the parsing
* support code essentially; most of it is error recovery stuff and
* backtracking.
*
* TODO: rename since lexer not under. or reorg parser/treeparser; treeparser under parser?
*/
public abstract class BaseRecognizer {
public static final int EOF=-1;
public static final int MEMO_RULE_FAILED = -2;
public static final int MEMO_RULE_UNKNOWN = -1;
public static final String NEXT_TOKEN_RULE_NAME = "nextToken";
/** State of a lexer, parser, or tree parser are collected into a state
* object so the state can be shared. This sharing is needed to
* have one grammar import others and share same error variables
* and other state variables. It's a kind of explicit multiple
* inheritance via delegation of methods and shared state.
*/
public ParserSharedState state;
public BaseRecognizer(IntStream input) {
this(input, new ParserSharedState());
}
public BaseRecognizer(IntStream input, ParserSharedState state) {
if ( state==null ) {
state = new ParserSharedState();
}
this.state = state;
state.input = input;
}
/** reset the parser's state; subclasses must rewinds the input stream */
public void reset() {
// wack everything related to error recovery
if ( state==null ) {
return; // no shared state work to do
}
state.ctx.clear();
state.errorRecovery = false;
state.lastErrorIndex = -1;
state.failed = false;
state.syntaxErrors = 0;
// wack everything related to backtracking and memoization
state.backtracking = 0;
for (int i = 0; state.ruleMemo!=null && i < state.ruleMemo.length; i++) { // wipe cache
state.ruleMemo[i] = null;
}
}
/** Match current input symbol against ttype. Attempt
* single token insertion or deletion error recovery. If
* that fails, throw MismatchedTokenException.
*
* To turn off single token insertion or deletion error
* recovery, override recoverFromMismatchedToken() and have it
* throw an exception. See TreeParser.recoverFromMismatchedToken().
* This way any error in a rule will cause an exception and
* immediate exit from rule. Rule would recover by resynchronizing
* to the set of symbols that can follow rule ref.
*/
public Object match(int ttype, LABitSet follow)
throws RecognitionException
{
System.out.println("match "+((TokenStream)state.input).LT(1)+" vs expected "+ttype);
Object matchedSymbol = getCurrentInputSymbol();
if ( state.input.LA(1)==ttype ) {
state.input.consume();
state.errorRecovery = false;
state.failed = false;
return matchedSymbol;
}
// if ( state.backtracking>0 ) {
// state.failed = true;
// return matchedSymbol;
// }
matchedSymbol = recoverFromMismatchedToken(ttype, follow);
System.out.println("rsync'd to "+matchedSymbol);
return matchedSymbol;
}
// like matchSet but w/o consume; error checking routine.
public void sync(LABitSet expecting) {
if ( expecting.member(state.input.LA(1)) ) return;
System.out.println("failed sync to "+expecting);
LABitSet followSet = computeErrorRecoverySet();
followSet.orInPlace(expecting);
NoViableAltException e = new NoViableAltException(this, followSet);
recoverFromMismatchedSet(e, followSet);
}
/** Match the wildcard: in a symbol */
public void matchAny() {
state.errorRecovery = false;
state.failed = false;
state.input.consume();
}
public boolean mismatchIsUnwantedToken(int ttype) {
return state.input.LA(2)==ttype;
}
public boolean mismatchIsMissingToken(LABitSet follow) {
if ( follow==null ) {
// we have no information about the follow; we can only consume
// a single token and hope for the best
return false;
}
// compute what can follow this grammar element reference
if ( follow.member(Token.EOR_TOKEN_TYPE) ) {
LABitSet viableTokensFollowingThisRule = computeNextViableTokenSet();
follow = follow.or(viableTokensFollowingThisRule);
if ( state.ctx.sp>=0 ) { // remove EOR if we're not the start symbol
follow.remove(Token.EOR_TOKEN_TYPE);
}
}
// if current token is consistent with what could come after set
// then we know we're missing a token; error recovery is free to
// "insert" the missing token
//System.out.println("viable tokens="+follow.toString(getTokenNames()));
//System.out.println("LT(1)="+((TokenStream)state.input).LT(1));
// LABitSet cannot handle negative numbers like -1 (EOF) so I leave EOR
// in follow set to indicate that the fall of the start symbol is
// in the set (EOF can follow).
if ( follow.member(state.input.LA(1)) || follow.member(Token.EOR_TOKEN_TYPE) ) {
//System.out.println("LT(1)=="+((TokenStream)state.input).LT(1)+" is consistent with what follows; inserting...");
return true;
}
return false;
}
/** Report a recognition problem.
*
* This method sets errorRecovery to indicate the parser is recovering
* not parsing. Once in recovery mode, no errors are generated.
* To get out of recovery mode, the parser must successfully match
* a token (after a resync). So it will go:
*
* 1. error occurs
* 2. enter recovery mode, report error
* 3. consume until token found in resynch set
* 4. try to resume parsing
* 5. next match() will reset errorRecovery mode
*/
public void reportError(RecognitionException e) {
// if we've already reported an error and have not matched a token
// yet successfully, don't report any errors.
if ( state.errorRecovery ) {
//System.err.print("[SPURIOUS] ");
return;
}
state.syntaxErrors++; // don't count spurious
state.errorRecovery = true;
notifyListeners(e);
}
/** What error message should be generated for the various
* exception types?
*
* Not very object-oriented code, but I like having all error message
* generation within one method rather than spread among all of the
* exception classes. This also makes it much easier for the exception
* handling because the exception classes do not have to have pointers back
* to this object to access utility routines and so on. Also, changing
* the message for an exception type would be difficult because you
* would have to subclassing exception, but then somehow get ANTLR
* to make those kinds of exception objects instead of the default.
* This looks weird, but trust me--it makes the most sense in terms
* of flexibility.
*
* For grammar debugging, you will want to override this to add
* more information such as the stack frame with
* getRuleInvocationStack(e, this.getClass().getName()) and,
* for no viable alts, the decision description and state etc...
*
* Override this to change the message generated for one or more
* exception types.
*/
public String getErrorMessage(RecognitionException e) {
String[] tokenNames = getTokenNames();
String msg = e.getMessage();
if ( e instanceof UnwantedTokenException ) {
UnwantedTokenException ute = (UnwantedTokenException)e;
String tokenName="<unknown>";
if ( ute.expecting.member(Token.EOF) ) {
tokenName = "EOF";
}
else {
tokenName = tokenNames[ute.expecting.getSingleElement()];
}
msg = "extraneous input "+getTokenErrorDisplay(ute.getUnexpectedToken())+
" expecting "+tokenName;
}
else if ( e instanceof MissingTokenException ) {
MissingTokenException mte = (MissingTokenException)e;
String tokenName="<unknown>";
if ( mte.expecting.member(Token.EOF) ) {
tokenName = "EOF";
}
else {
tokenName = tokenNames[mte.expecting.getSingleElement()];
}
msg = "missing "+tokenName+" at "+getTokenErrorDisplay(e.token);
}
else if ( e instanceof MismatchedTokenException ) {
MismatchedTokenException mte = (MismatchedTokenException)e;
String tokenName="<unknown>";
if ( mte.expecting.member(Token.EOF) ) {
tokenName = "EOF";
}
else {
tokenName = tokenNames[mte.expecting.getSingleElement()];
}
msg = "mismatched input "+getTokenErrorDisplay(e.token)+
" expecting "+tokenName;
}
else if ( e instanceof MismatchedTreeNodeException ) {
MismatchedTreeNodeException mtne = (MismatchedTreeNodeException)e;
String tokenName="<unknown>";
if ( mtne.expecting.member(Token.EOF) ) {
tokenName = "EOF";
}
else {
tokenName = tokenNames[mtne.expecting.getSingleElement()];
}
msg = "mismatched tree node: "+mtne.node+
" expecting "+tokenName;
}
else if ( e instanceof NoViableAltException ) {
//NoViableAltException nvae = (NoViableAltException)e;
// for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
// and "(decision="+nvae.decisionNumber+") and
// "state "+nvae.stateNumber
msg = "no viable alternative at input "+getTokenErrorDisplay(e.token);
}
else if ( e instanceof EarlyExitException ) {
//EarlyExitException eee = (EarlyExitException)e;
// for development, can add "(decision="+eee.decisionNumber+")"
msg = "required (...)+ loop did not match anything at input "+
getTokenErrorDisplay(e.token);
}
else if ( e instanceof MismatchedSetException ) {
MismatchedSetException mse = (MismatchedSetException)e;
msg = "mismatched input "+getTokenErrorDisplay(e.token)+
" expecting set "+mse.expecting;
}
else if ( e instanceof MismatchedNotSetException ) {
MismatchedNotSetException mse = (MismatchedNotSetException)e;
msg = "mismatched input "+getTokenErrorDisplay(e.token)+
" expecting set "+mse.expecting;
}
else if ( e instanceof FailedPredicateException ) {
FailedPredicateException fpe = (FailedPredicateException)e;
msg = "rule "+fpe.ruleName+" failed predicate: {"+
fpe.predicateText+"}?";
}
return msg;
}
/** Get number of recognition errors (lexer, parser, tree parser). Each
* recognizer tracks its own number. So parser and lexer each have
* separate count. Does not count the spurious errors found between
* an error and next valid token match
*
* See also reportError()
*/
public int getNumberOfSyntaxErrors() {
return state.syntaxErrors;
}
/** What is the error header, normally line/character position information? */
public String getErrorHeader(RecognitionException e) {
return "line "+e.line+":"+e.charPositionInLine;
}
/** How should a token be displayed in an error message? The default
* is to display just the text, but during development you might
* want to have a lot of information spit out. Override in that case
* to use t.toString() (which, for CommonToken, dumps everything about
* the token). This is better than forcing you to override a method in
* your token objects because you don't have to go modify your lexer
* so that it creates a new Java type.
*/
public String getTokenErrorDisplay(Token t) {
String s = t.getText();
if ( s==null ) {
if ( t.getType()==Token.EOF ) {
s = "<EOF>";
}
else {
s = "<"+t.getType()+">";
}
}
s = s.replaceAll("\n","\\\\n");
s = s.replaceAll("\r","\\\\r");
s = s.replaceAll("\t","\\\\t");
return "'"+s+"'";
}
/** Recover from an error found on the input stream. This is
* for NoViableAlt and mismatched symbol exceptions. If you enable
* single token insertion and deletion, this will usually not
* handle mismatched symbol exceptions but there could be a mismatched
* token that the match() routine could not recover from.
*/
public void recover() {
if ( state.lastErrorIndex==state.input.index() ) {
// uh oh, another error at same token index; must be a case
// where LT(1) is in the recovery token set so nothing is
// consumed; consume a single token so at least to prevent
// an infinite loop; this is a failsafe.
state.input.consume();
}
state.lastErrorIndex = state.input.index();
LABitSet followSet = computeErrorRecoverySet();
beginResync();
consumeUntil(followSet);
endResync();
}
/** A hook to listen in on the token consumption during error recovery.
* The DebugParser subclasses this to fire events to the listenter.
*/
public void beginResync() {
}
public void endResync() {
}
/* Compute the error recovery set for the current rule. During
* rule invocation, the parser pushes the set of tokens that can
* follow that rule reference on the stack; this amounts to
* computing FIRST of what follows the rule reference in the
* enclosing rule. See LinearApproximator.FIRST().
* This local follow set only includes tokens
* from within the rule; i.e., the FIRST computation done by
* ANTLR stops at the end of a rule.
*
* EXAMPLE
*
* When you find a "no viable alt exception", the input is not
* consistent with any of the alternatives for rule r. The best
* thing to do is to consume tokens until you see something that
* can legally follow a call to r *or* any rule that called r.
* You don't want the exact set of viable next tokens because the
* input might just be missing a token--you might consume the
* rest of the input looking for one of the missing tokens.
*
* Consider grammar:
*
* a : '[' b ']'
* | '(' b ')'
* ;
* b : c '^' INT ;
* c : ID
* | INT
* ;
*
* At each rule invocation, the set of tokens that could follow
* that rule is pushed on a stack. Here are the various
* context-sensitive follow sets:
*
* FOLLOW(b1_in_a) = FIRST(']') = ']'
* FOLLOW(b2_in_a) = FIRST(')') = ')'
* FOLLOW(c_in_b) = FIRST('^') = '^'
*
* Upon erroneous input "[]", the call chain is
*
* a -> b -> c
*
* and, hence, the follow context stack is:
*
* depth follow set start of rule execution
* 0 <EOF> a (from main())
* 1 ']' b
* 2 '^' c
*
* Notice that ')' is not included, because b would have to have
* been called from a different context in rule a for ')' to be
* included.
*
* For error recovery, we cannot consider FOLLOW(c)
* (context-sensitive or otherwise). We need the combined set of
* all context-sensitive FOLLOW sets--the set of all tokens that
* could follow any reference in the call chain. We need to
* resync to one of those tokens. Note that FOLLOW(c)='^' and if
* we resync'd to that token, we'd consume until EOF. We need to
* sync to context-sensitive FOLLOWs for a, b, and c: {']','^'}.
* In this case, for input "[]", LA(1) is ']' and in the set, so we would
* not consume anything. After printing an error, rule c would
* return normally. Rule b would not find the required '^' though.
* At this point, it gets a mismatched token error and throws an
* exception (since LA(1) is not in the viable following token
* set). The rule exception handler tries to recover, but finds
* the same recovery set and doesn't consume anything. Rule b
* exits normally returning to rule a. Now it finds the ']' (and
* with the successful match exits errorRecovery mode).
*
* So, you can see that the parser walks up the call chain looking
* for the token that was a member of the recovery set.
*
* Errors are not generated in errorRecovery mode.
*
* ANTLR's error recovery mechanism is based upon original ideas:
*
* "Algorithms + Data Structures = Programs" by Niklaus Wirth
*
* and
*
* "A note on error recovery in recursive descent parsers":
* http://portal.acm.org/citation.cfm?id=947902.947905
*
* Later, Josef Grosch had some good ideas:
*
* "Efficient and Comfortable Error Recovery in Recursive Descent
* Parsers":
* ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip
*
* Like Grosch I implement context-sensitive FOLLOW sets that are combined
* at run-time upon error to avoid overhead during parsing.
*/
protected LABitSet computeErrorRecoverySet() {
int top = state.ctx.sp;
LABitSet followSet = new LABitSet();
for (int i=top; i>=0; i--) { // i==0 is EOF context for start rule invocation
LABitSet f = (LABitSet)state.ctx.get(i).follow;
followSet.orInPlace(f);
}
return followSet;
}
/** Compute the context-sensitive FOLLOW set for current rule.
* This is set of token types that can follow a specific rule
* reference given a specific call chain. You get the set of
* viable tokens that can possibly come next (lookahead depth 1)
* given the current call chain. Contrast this with the
* definition of plain FOLLOW for rule r:
*
* FOLLOW(r)={x | S=>*alpha r beta in G and x in FIRST(beta)}
*
* where x in T* and alpha, beta in V*; T is set of terminals and
* V is the set of terminals and nonterminals. In other words,
* FOLLOW(r) is the set of all tokens that can possibly follow
* references to r in *any* sentential form (context). At
* runtime, however, we know precisely which context applies as
* we have the call chain. We may compute the exact (rather
* than covering superset) set of following tokens.
*
* For example, consider grammar:
*
* stat : ID '=' expr ';' // FOLLOW(stat)=={EOF}
* | "return" expr '.'
* ;
* expr : atom ('+' atom)* ; // FOLLOW(expr)=={';','.',')'}
* atom : INT // FOLLOW(atom)=={'+',')',';','.'}
* | '(' expr ')'
* ;
*
* The FOLLOW sets are all inclusive whereas context-sensitive
* FOLLOW sets are precisely what could follow a rule reference.
* For input input "i=(3);", here is the derivation:
*
* stat => ID '=' expr ';'
* => ID '=' atom ('+' atom)* ';'
* => ID '=' '(' expr ')' ('+' atom)* ';'
* => ID '=' '(' atom ')' ('+' atom)* ';'
* => ID '=' '(' INT ')' ('+' atom)* ';'
* => ID '=' '(' INT ')' ';'
*
* At the "3" token, you'd have a call chain of
*
* stat -> expr -> atom -> expr -> atom
*
* What can follow that specific nested ref to atom? Exactly ')'
* as you can see by looking at the derivation of this specific
* input. Contrast this with the FOLLOW(atom)={'+',')',';','.'}.
*
* You want the exact viable token set when recovering from a
* token mismatch. Upon token mismatch, if LA(1) is member of
* the viable next token set, then you know there is most likely
* a missing token in the input stream. "Insert" one by just not
* throwing an exception.
*/
public LABitSet computeNextViableTokenSet() {
int top = state.ctx.sp;
LABitSet followSet = new LABitSet();
for (int i=top; i>=0; i--) { // i==0 is EOF context for start rule invocation
LABitSet f = (LABitSet)state.ctx.get(i).follow;
followSet.orInPlace(f);
// can we see end of rule? if not, don't include follow of this rule
if ( !f.member(Token.EOR_TOKEN_TYPE) ) break;
// else combine with tokens that can follow this rule (rm EOR also)
// EOR indicates we have to include follow(start rule); i.e., EOF
followSet.remove(Token.EOR_TOKEN_TYPE);
}
return followSet;
}
/** Attempt to recover from a single missing or extra token.
*
* EXTRA TOKEN
*
* LA(1) is not what we are looking for. If LA(2) has the right token,
* however, then assume LA(1) is some extra spurious token. Delete it
* and LA(2) as if we were doing a normal match(), which advances the
* input.
*
* MISSING TOKEN
*
* If current token is consistent with what could come after
* ttype then it is ok to "insert" the missing token, else throw
* exception For example, Input "i=(3;" is clearly missing the
* ')'. When the parser returns from the nested call to expr, it
* will have call chain:
*
* stat -> expr -> atom
*
* and it will be trying to match the ')' at this point in the
* derivation:
*
* => ID '=' '(' INT ')' ('+' atom)* ';'
* ^
* match() will see that ';' doesn't match ')' and report a
* mismatched token error. To recover, it sees that LA(1)==';'
* is in the set of tokens that can follow the ')' token
* reference in rule atom. It can assume that you forgot the ')'.
*/
protected Object recoverFromMismatchedToken(int ttype, LABitSet follow)
throws RecognitionException
{
RecognitionException e = null;
// if next token is what we are looking for then "delete" this token
if ( mismatchIsUnwantedToken(ttype) ) {
e = new UnwantedTokenException(this, ttype);
/*
System.err.println("recoverFromMismatchedToken deleting "+
((TokenStream)state.input).LT(1)+
" since "+((TokenStream)state.input).LT(2)+" is what we want");
*/
beginResync();
state.input.consume(); // simply delete extra token
endResync();
reportError(e); // report after consuming so AW sees the token in the exception
// we want to return the token we're actually matching
Object matchedSymbol = getCurrentInputSymbol();
state.input.consume(); // move past ttype token as if all were ok
return matchedSymbol;
}
// can't recover with single token deletion, try insertion
if ( mismatchIsMissingToken(follow) ) {
Object inserted = getMissingSymbol(e, ttype, follow);
e = new MissingTokenException(this, ttype, inserted);
reportError(e); // report after inserting so AW sees the token in the exception
return inserted;
}
// even that didn't work; must throw the exception
e = new MismatchedTokenException(this, ttype);
throw e;
}
public Object recoverFromMismatchedSet(RecognitionException e,
LABitSet follow)
throws RecognitionException
{
if ( mismatchIsMissingToken(follow) ) {
// System.out.println("missing token");
reportError(e);
// we don't know how to conjure up a token for sets yet
return getMissingSymbol(e, Token.INVALID_TOKEN_TYPE, follow);
}
// TODO do single token deletion like above for Token mismatch
throw e;
}
/** Match needs to return the current input symbol, which gets put
* into the label for the associated token ref; e.g., x=ID. Token
* and tree parsers need to return different objects. Rather than test
* for input stream type or change the IntStream interface, I use
* a simple method to ask the recognizer to tell me what the current
* input symbol is.
*/
protected Object getCurrentInputSymbol() { return null; }
/** Conjure up a missing token during error recovery.
*
* The recognizer attempts to recover from single missing
* symbols. But, actions might refer to that missing symbol.
* For example, x=ID {f($x);}. The action clearly assumes
* that there has been an identifier matched previously and that
* $x points at that token. If that token is missing, but
* the next token in the stream is what we want we assume that
* this token is missing and we keep going. Because we
* have to return some token to replace the missing token,
* we have to conjure one up. This method gives the user control
* over the tokens returned for missing tokens. Mostly,
* you will want to create something special for identifier
* tokens. For literals such as '{' and ',', the default
* action in the parser or tree parser works. It simply creates
* a CommonToken of the appropriate type. The text will be the token.
* If you change what tokens must be created by the lexer,
* override this method to create the appropriate tokens.
*/
protected Object getMissingSymbol(RecognitionException e,
int expectedTokenType,
LABitSet follow)
{
return null;
}
public void consumeUntil(int tokenType) {
//System.out.println("consumeUntil "+tokenType);
int ttype = state.input.LA(1);
while (ttype != Token.EOF && ttype != tokenType) {
state.input.consume();
ttype = state.input.LA(1);
}
}
/** Consume tokens until one matches the given token set */
public void consumeUntil(LABitSet set) {
//System.out.println("consumeUntil("+set.toString(getTokenNames())+")");
int ttype = state.input.LA(1);
while (ttype != Token.EOF && !set.member(ttype) ) {
//System.out.println("consume during recover LA(1)="+getTokenNames()[state.input.LA(1)]);
state.input.consume();
ttype = state.input.LA(1);
}
}
/** Return List<String> of the rules in your parser instance
* leading up to a call to this method. You could override if
* you want more details such as the file/line info of where
* in the parser java code a rule is invoked.
*
* This is very useful for error messages and for context-sensitive
* error recovery.
*/
public List getRuleInvocationStack() {
String parserClassName = getClass().getName();
return getRuleInvocationStack(new Throwable(), parserClassName);
}
/** A more general version of getRuleInvocationStack where you can
* pass in, for example, a RecognitionException to get it's rule
* stack trace. This routine is shared with all recognizers, hence,
* static.
*
* TODO: move to a utility class or something; weird having lexer call this
*/
public static List getRuleInvocationStack(Throwable e,
String recognizerClassName)
{
List rules = new ArrayList();
StackTraceElement[] stack = e.getStackTrace();
int i = 0;
for (i=stack.length-1; i>=0; i--) {
StackTraceElement t = stack[i];
if ( t.getClassName().startsWith("org.antlr.v4.runtime.") ) {
continue; // skip support code such as this method
}
if ( t.getMethodName().equals(NEXT_TOKEN_RULE_NAME) ) {
continue;
}
if ( !t.getClassName().equals(recognizerClassName) ) {
continue; // must not be part of this parser
}
rules.add(t.getMethodName());
}
return rules;
}
public int getBacktrackingLevel() { return state.backtracking; }
public void setBacktrackingLevel(int n) { state.backtracking = n; }
/** Return whether or not a backtracking attempt failed. */
public boolean failed() { return state.failed; }
/** Used to print out token names like ID during debugging and
* error reporting. The generated parsers implement a method
* that overrides this to point to their String[] tokenNames.
*/
public String[] getTokenNames() {
return null;
}
/** For debugging and other purposes, might want the grammar name.
* Have ANTLR generate an implementation for this method.
*/
public String getGrammarFileName() {
return null;
}
public abstract String getSourceName();
/** A convenience method for use most often with template rewrites.
* Convert a List<Token> to List<String>
*/
public List toStrings(List tokens) {
if ( tokens==null ) return null;
List strings = new ArrayList(tokens.size());
for (int i=0; i<tokens.size(); i++) {
strings.add(((Token)tokens.get(i)).getText());
}
return strings;
}
/** Given a rule number and a start token index number, return
* MEMO_RULE_UNKNOWN if the rule has not parsed input starting from
* start index. If this rule has parsed input starting from the
* start index before, then return where the rule stopped parsing.
* It returns the index of the last token matched by the rule.
*
* For now we use a hashtable and just the slow Object-based one.
* Later, we can make a special one for ints and also one that
* tosses out data after we commit past input position i.
*/
public int getRuleMemoization(int ruleIndex, int ruleStartIndex) {
if ( state.ruleMemo[ruleIndex]==null ) {
state.ruleMemo[ruleIndex] = new HashMap();
}
Integer stopIndexI =
(Integer)state.ruleMemo[ruleIndex].get(new Integer(ruleStartIndex));
if ( stopIndexI==null ) {
return MEMO_RULE_UNKNOWN;
}
return stopIndexI.intValue();
}
/** Has this rule already parsed input at the current index in the
* input stream? Return the stop token index or MEMO_RULE_UNKNOWN.
* If we attempted but failed to parse properly before, return
* MEMO_RULE_FAILED.
*
* This method has a side-effect: if we have seen this input for
* this rule and successfully parsed before, then seek ahead to
* 1 past the stop token matched for this rule last time.
*/
public boolean alreadyParsedRule(int ruleIndex) {
int stopIndex = getRuleMemoization(ruleIndex, state.input.index());
if ( stopIndex==MEMO_RULE_UNKNOWN ) {
return false;
}
if ( stopIndex==MEMO_RULE_FAILED ) {
//System.out.println("rule "+ruleIndex+" will never succeed");
state.failed=true;
}
else {
//System.out.println("seen rule "+ruleIndex+" before; skipping ahead to @"+(stopIndex+1)+" failed="+state.failed);
state.input.seek(stopIndex+1); // jump to one past stop token
}
return true;
}
/** Record whether or not this rule parsed the input at this position
* successfully. Use a standard java hashtable for now.
*/
public void memoize(IntStream input,
int ruleIndex,
int ruleStartIndex)
{
int stopTokenIndex = state.failed?MEMO_RULE_FAILED:input.index()-1;
if ( state.ruleMemo==null ) {
System.err.println("!!!!!!!!! memo array is null for "+ getGrammarFileName());
}
if ( ruleIndex >= state.ruleMemo.length ) {
System.err.println("!!!!!!!!! memo size is "+state.ruleMemo.length+", but rule index is "+ruleIndex);
}
if ( state.ruleMemo[ruleIndex]!=null ) {
state.ruleMemo[ruleIndex].put(
new Integer(ruleStartIndex), new Integer(stopTokenIndex)
);
}
}
/** return how many rule/input-index pairs there are in total.
* TODO: this includes synpreds. :(
*/
public int getRuleMemoizationCacheSize() {
int n = 0;
for (int i = 0; state.ruleMemo!=null && i < state.ruleMemo.length; i++) {
Map ruleMap = state.ruleMemo[i];
if ( ruleMap!=null ) {
n += ruleMap.size(); // how many input indexes are recorded?
}
}
return n;
}
public void traceIn(String ruleName, int ruleIndex, Object inputSymbol) {
System.out.print("enter "+ruleName+" "+inputSymbol);
if ( state.backtracking>0 ) {
System.out.print(" backtracking="+state.backtracking);
}
System.out.println();
}
public void traceOut(String ruleName,
int ruleIndex,
Object inputSymbol)
{
System.out.print("exit "+ruleName+" "+inputSymbol);
if ( state.backtracking>0 ) {
System.out.print(" backtracking="+state.backtracking);
if ( state.failed ) System.out.print(" failed");
else System.out.print(" succeeded");
}
System.out.println();
}
/* In v3, programmers altered error messages by overriding
displayRecognitionError() and possibly getTokenErrorDisplay().
They overrode emitErrorMessage(String) to change where the output goes.
Now, in v4, we're going to use a listener mechanism. This makes it
easier for language applications to have parsers notify them
upon error without having to override the parsers. If you don't specify
a listener, ANTLR calls the v3 legacy displayRecognitionError()
method. All that does is format a message and call emitErrorMessage().
Otherwise, your listener will receive RecognitionException
exceptions and you can do what ever you want with them including
reproducing the same behavior by calling the legacy methods.
(In v4, RecognitionException includes the recognizer object).
Grammar tools can have a listeners without having to worry about
messing up the programmers' error handling.
*/
public void displayRecognitionError(RecognitionException e) {
String hdr = getErrorHeader(e);
String msg = getErrorMessage(e);
emitErrorMessage(hdr+" "+msg);
}
/** Override this method to change where error messages go */
public void emitErrorMessage(String msg) {
System.err.println(msg);
}
public void addListener(ANTLRParserListener pl) {
if ( state.listeners==null ) {
state.listeners =
Collections.synchronizedList(new ArrayList<ANTLRParserListener>(2));
}
if ( pl!=null ) state.listeners.add(pl);
}
public void removeListener(ANTLRParserListener pl) { state.listeners.remove(pl); }
public void removeListeners() { state.listeners.clear(); }
public List<ANTLRParserListener> getListeners() { return state.listeners; }
public void notifyListeners(RecognitionException re) {
if ( state.listeners==null || state.listeners.size()==0 ) {
// call legacy v3 func; this calls emitErrorMessage(String msg)
displayRecognitionError(re);
return;
}
for (ANTLRParserListener pl : state.listeners) pl.error(re);
}
}

45
runtime/Java/src/org/antlr/v4/runtime/CharStreamState.java
View File

@ -1,45 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
/** When walking ahead with cyclic DFA or for syntactic predicates,
* we need to record the state of the input stream (char index,
* line, etc...) so that we can rewind the state after scanning ahead.
*
* This is the complete state of a stream.
*/
public class CharStreamState {
/** Index into the char stream of next lookahead char */
public int p;
/** What line number is the scanner at before processing buffer[p]? */
public int line;
/** What char position 0..n-1 in line is scanner before processing buffer[p]? */
public int charPositionInLine;
}

196
runtime/Java/src/org/antlr/v4/runtime/CommonToken.java
View File

@ -1,196 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.Token;
import java.io.Serializable;
public class CommonToken implements Token, Serializable {
protected int type;
protected int line;
protected int charPositionInLine = -1; // set to invalid position
protected int channel=DEFAULT_CHANNEL;
protected transient CharStream input;
/** We need to be able to change the text once in a while. If
* this is non-null, then getText should return this. Note that
* start/stop are not affected by changing this.
*/
protected String text;
/** What token number is this from 0..n-1 tokens; < 0 implies invalid index */
protected int index = -1;
/** The char position into the input buffer where this token starts */
protected int start;
/** The char position into the input buffer where this token stops */
protected int stop;
public CommonToken(int type) {
this.type = type;
}
public CommonToken(CharStream input, int type, int channel, int start, int stop) {
this.input = input;
this.type = type;
this.channel = channel;
this.start = start;
this.stop = stop;
this.line = input.getLine();
this.charPositionInLine = input.getCharPositionInLine();
}
public CommonToken(int type, String text) {
this.type = type;
this.channel = DEFAULT_CHANNEL;
this.text = text;
}
public CommonToken(Token oldToken) {
text = oldToken.getText();
type = oldToken.getType();
line = oldToken.getLine();
index = oldToken.getTokenIndex();
charPositionInLine = oldToken.getCharPositionInLine();
channel = oldToken.getChannel();
input = oldToken.getInputStream();
if ( oldToken instanceof CommonToken ) {
start = ((CommonToken)oldToken).start;
stop = ((CommonToken)oldToken).stop;
}
}
public int getType() {
return type;
}
public void setLine(int line) {
this.line = line;
}
public String getText() {
if ( text!=null ) {
return text;
}
if ( input==null ) {
return null;
}
if ( start<input.size() && stop<input.size() ) {
text = input.substring(start,stop);
}
else {
text = "<EOF>";
}
return text;
}
/** Override the text for this token. getText() will return this text
* rather than pulling from the buffer. Note that this does not mean
* that start/stop indexes are not valid. It means that that input
* was converted to a new string in the token object.
*/
public void setText(String text) {
this.text = text;
}
public int getLine() {
return line;
}
public int getCharPositionInLine() {
return charPositionInLine;
}
public void setCharPositionInLine(int charPositionInLine) {
this.charPositionInLine = charPositionInLine;
}
public int getChannel() {
return channel;
}
public void setChannel(int channel) {
this.channel = channel;
}
public void setType(int type) {
this.type = type;
}
public int getStartIndex() {
return start;
}
public void setStartIndex(int start) {
this.start = start;
}
public int getStopIndex() {
return stop;
}
public void setStopIndex(int stop) {
this.stop = stop;
}
public int getTokenIndex() {
return index;
}
public void setTokenIndex(int index) {
this.index = index;
}
public CharStream getInputStream() {
return input;
}
public void setInputStream(CharStream input) {
this.input = input;
}
public String toString() {
String channelStr = "";
if ( channel>0 ) {
channelStr=",channel="+channel;
}
String txt = getText();
if ( txt!=null ) {
txt = txt.replaceAll("\n","\\\\n");
txt = txt.replaceAll("\r","\\\\r");
txt = txt.replaceAll("\t","\\\\t");
}
else {
txt = "<no text>";
}
return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
}
}

40
runtime/Java/src/org/antlr/v4/runtime/EarlyExitException.java
View File

@ -1,40 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.LABitSet;
/** The recognizer did not match anything for a (..)+ loop. */
public class EarlyExitException extends RecognitionException {
/** Used for remote debugger deserialization */
public EarlyExitException() {;}
public EarlyExitException(BaseRecognizer recognizer, LABitSet firstSet) {
super(recognizer, firstSet);
}
}

54
runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java
View File

@ -1,54 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
/** A semantic predicate failed during validation. Validation of predicates
* occurs when normally parsing the alternative just like matching a token.
* Disambiguating predicate evaluation occurs when we hoist a predicate into
* a prediction decision.
*/
public class FailedPredicateException extends RecognitionException {
public String ruleName;
public String predicateText;
/** Used for remote debugger deserialization */
public FailedPredicateException() {;}
public FailedPredicateException(BaseRecognizer recognizer,
String ruleName,
String predicateText)
{
super(recognizer);
this.ruleName = ruleName;
this.predicateText = predicateText;
}
public String toString() {
return "FailedPredicateException("+ruleName+",{"+predicateText+"}?)";
}
}

359
runtime/Java/src/org/antlr/v4/runtime/Lexer.java
View File

@ -1,359 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenSource;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.QStack;
import org.antlr.v4.runtime.pda.Bytecode;
import org.antlr.v4.runtime.pda.PDA;
import java.util.EmptyStackException;
/** A lexer is recognizer that draws input symbols from a character stream.
* lexer grammars result in a subclass of this object. A Lexer object
* uses simplified match() and error recovery mechanisms in the interest
* of speed.
*/
public abstract class Lexer implements TokenSource {
public static final int DEFAULT_MODE = 0;
public static final int MORE = -2;
public static final int SKIP = -3;
public static final int DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL;
public static final int HIDDEN = Token.HIDDEN_CHANNEL;
public LexerSharedState state;
public static PDA[] modeToPDA;
public static DFA[] modeToDFA;
public Lexer(CharStream input) {
this(input, new LexerSharedState());
}
public Lexer(CharStream input, LexerSharedState state) {
if ( state==null ) {
state = new LexerSharedState();
}
this.state = state;
state.input = input;
}
public void reset() {
// wack Lexer state variables
if ( state.input!=null ) {
state.input.seek(0); // rewind the input
}
if ( state==null ) {
return; // no shared state work to do
}
state.token = null;
state.type = Token.INVALID_TOKEN_TYPE;
state.channel = Token.DEFAULT_CHANNEL;
state.tokenStartCharIndex = -1;
state.tokenStartCharPositionInLine = -1;
state.tokenStartLine = -1;
state.text = null;
}
/** Return a token from this source; i.e., match a token on the char
* stream.
*/
public Token nextToken() {
outer:
while (true) {
state.token = null;
state.channel = Token.DEFAULT_CHANNEL;
state.tokenStartCharIndex = state.input.index();
state.tokenStartCharPositionInLine = state.input.getCharPositionInLine();
state.tokenStartLine = state.input.getLine();
state.text = null;
do {
state.type = Token.INVALID_TOKEN_TYPE;
if ( state.input.LA(1)==CharStream.EOF ) {
Token eof = new org.antlr.runtime.CommonToken(state.input,Token.EOF,
Token.DEFAULT_CHANNEL,
state.input.index(),state.input.index());
eof.setLine(getLine());
eof.setCharPositionInLine(getCharPositionInLine());
return eof;
}
System.err.println("predict mode "+state.mode+" at index "+state.input.index());
int ttype = modeToDFA[state.mode].predict(state.input);
System.err.println("returns "+ttype);
if ( state.type == Token.INVALID_TOKEN_TYPE ) state.type = ttype;
if ( state.type==SKIP ) {
continue outer;
}
} while ( state.type==MORE );
if ( state.token==null ) emit();
return state.token;
}
}
public Token nextToken_PDA() {
outer:
while (true) {
state.token = null;
state.channel = Token.DEFAULT_CHANNEL;
state.tokenStartCharIndex = state.input.index();
state.tokenStartCharPositionInLine = ((CharStream)state.input).getCharPositionInLine();
state.tokenStartLine = ((CharStream)state.input).getLine();
state.text = null;
do {
state.type = Token.INVALID_TOKEN_TYPE;
if ( state.input.LA(1)==CharStream.EOF ) {
Token eof = new CommonToken((CharStream)state.input,Token.EOF,
Token.DEFAULT_CHANNEL,
state.input.index(),state.input.index());
eof.setLine(getLine());
eof.setCharPositionInLine(getCharPositionInLine());
return eof;
}
int ttype = 0;
try {
ttype = modeToPDA[state.mode].execThompson(state.input);
}
catch (PDA.InvalidElement re) {
CharStream cs = (CharStream)state.input;
System.err.println("!!!!! no match for char "+
Bytecode.quotedCharLiteral(state.input.LA(1))+
" at "+state.input.index()+
" line "+cs.getLine()+":"+cs.getCharPositionInLine());
state.input.consume();
continue;
}
if ( state.type == Token.INVALID_TOKEN_TYPE ) state.type = ttype;
if ( state.type==SKIP ) {
continue outer;
}
} while ( state.type==MORE );
if ( state.token==null ) emit();
return state.token;
}
}
/** Instruct the lexer to skip creating a token for current lexer rule
* and look for another token. nextToken() knows to keep looking when
* a lexer rule finishes with token set to SKIP_TOKEN. Recall that
* if token==null at end of any token rule, it creates one for you
* and emits it.
*/
public void skip() {
state.type = SKIP;
}
public void more() {
state.type = MORE;
}
public void mode(int m) {
state.mode = m;
}
public void pushMode(int m) {
if ( state.modeStack==null ) state.modeStack = new QStack<Integer>();
state.modeStack.push(state.mode);
mode(m);
}
public int popMode() {
if ( state.modeStack==null ) throw new EmptyStackException();
mode( state.modeStack.pop() );
return state.mode;
}
/** Set the char stream and reset the lexer */
public void setCharStream(CharStream input) {
this.state.input = null;
reset();
this.state.input = input;
}
public CharStream getCharStream() {
return ((CharStream)state.input);
}
public String getSourceName() {
return state.input.getSourceName();
}
/** Currently does not support multiple emits per nextToken invocation
* for efficiency reasons. Subclass and override this method and
* nextToken (to push tokens into a list and pull from that list rather
* than a single variable as this implementation does).
*/
public void emit(Token token) {
//System.err.println("emit "+token);
state.token = token;
}
/** The standard method called to automatically emit a token at the
* outermost lexical rule. The token object should point into the
* char buffer start..stop. If there is a text override in 'text',
* use that to set the token's text. Override this method to emit
* custom Token objects.
*
* If you are building trees, then you should also override
* Parser or TreeParser.getMissingSymbol().
*/
public Token emit() {
Token t = new CommonToken(((CharStream)state.input), state.type,
state.channel, state.tokenStartCharIndex,
getCharIndex()-1);
t.setLine(state.tokenStartLine);
t.setText(state.text);
t.setCharPositionInLine(state.tokenStartCharPositionInLine);
emit(t);
return t;
}
public int getLine() {
return ((CharStream)state.input).getLine();
}
public int getCharPositionInLine() {
return ((CharStream)state.input).getCharPositionInLine();
}
/** What is the index of the current character of lookahead? */
public int getCharIndex() {
return state.input.index();
}
/** Return the text matched so far for the current token or any
* text override.
*/
public String getText() {
if ( state.text!=null ) {
return state.text;
}
return ((CharStream)state.input).substring(state.tokenStartCharIndex,getCharIndex()-1);
}
/** Set the complete text of this token; it wipes any previous
* changes to the text.
*/
public void setText(String text) {
state.text = text;
}
public void reportError(RecognitionException e) {
/** TODO: not thought about recovery in lexer yet.
*
// if we've already reported an error and have not matched a token
// yet successfully, don't report any errors.
if ( errorRecovery ) {
//System.err.print("[SPURIOUS] ");
return;
}
errorRecovery = true;
*/
//displayRecognitionError(this.getTokenNames(), e);
}
/** Used to print out token names like ID during debugging and
* error reporting. The generated parsers implement a method
* that overrides this to point to their String[] tokenNames.
*/
public String[] getTokenNames() {
return null;
}
public String getErrorMessage(RecognitionException e) {
String msg = null;
if ( e instanceof MismatchedTokenException ) {
MismatchedTokenException mte = (MismatchedTokenException)e;
msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting "+
getCharErrorDisplay(mte.expecting.getSingleElement());
}
else if ( e instanceof NoViableAltException ) {
NoViableAltException nvae = (NoViableAltException)e;
// for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
// and "(decision="+nvae.decisionNumber+") and
// "state "+nvae.stateNumber
msg = "no viable alternative at character "+getCharErrorDisplay(e.c);
}
else if ( e instanceof EarlyExitException ) {
EarlyExitException eee = (EarlyExitException)e;
// for development, can add "(decision="+eee.decisionNumber+")"
msg = "required (...)+ loop did not match anything at character "+getCharErrorDisplay(e.c);
}
else if ( e instanceof MismatchedNotSetException ) {
MismatchedNotSetException mse = (MismatchedNotSetException)e;
msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
}
else if ( e instanceof MismatchedSetException ) {
MismatchedSetException mse = (MismatchedSetException)e;
msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
}
else if ( e instanceof MismatchedRangeException ) {
MismatchedRangeException mre = (MismatchedRangeException)e;
msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+
getCharErrorDisplay(mre.a)+".."+getCharErrorDisplay(mre.b);
}
else {
//msg = super.getErrorMessage(e, tokenNames);
}
return msg;
}
public String getCharErrorDisplay(int c) {
String s = String.valueOf((char)c);
switch ( c ) {
case Token.EOF :
s = "<EOF>";
break;
case '\n' :
s = "\\n";
break;
case '\t' :
s = "\\t";
break;
case '\r' :
s = "\\r";
break;
}
return "'"+s+"'";
}
/** Lexers can normally match any char in it's vocabulary after matching
* a token, so do the easy thing and just kill a character and hope
* it all works out. You can instead use the rule invocation stack
* to do sophisticated error recovery if you are in a fragment rule.
*/
public void recover(RecognitionException re) {
//System.out.println("consuming char "+(char)state.input.LA(1)+" during recovery");
//re.printStackTrace();
state.input.consume();
}
}

58
runtime/Java/src/org/antlr/v4/runtime/LexerSharedState.java
View File

@ -1,58 +0,0 @@
package org.antlr.v4.runtime;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.Token;
import org.antlr.v4.runtime.misc.QStack;
public class LexerSharedState {
public CharStream input;
/** The goal of all lexer rules/methods is to create a token object.
* This is an instance variable as multiple rules may collaborate to
* create a single token. nextToken will return this object after
* matching lexer rule(s). If you subclass to allow multiple token
* emissions, then set this to the last token to be matched or
* something nonnull so that the auto token emit mechanism will not
* emit another token.
*/
public Token token;
/** What character index in the stream did the current token start at?
* Needed, for example, to get the text for current token. Set at
* the start of nextToken.
*/
public int tokenStartCharIndex = -1;
/** The line on which the first character of the token resides */
public int tokenStartLine;
/** The character position of first character within the line */
public int tokenStartCharPositionInLine;
/** The channel number for the current token */
public int channel;
/** The token type for the current token */
public int type;
public QStack<Integer> modeStack;
public int mode = Lexer.DEFAULT_MODE;
/** You can set the text for the current token to override what is in
* the input char buffer. Use setText() or can set this instance var.
*/
public String text;
public LexerSharedState() {
}
public LexerSharedState(LexerSharedState state) {
this.token = state.token;
this.tokenStartCharIndex = state.tokenStartCharIndex;
this.tokenStartLine = state.tokenStartLine;
this.tokenStartCharPositionInLine = state.tokenStartCharPositionInLine;
this.channel = state.channel;
this.type = state.type;
this.text = state.text;
}
}

43
runtime/Java/src/org/antlr/v4/runtime/MismatchedNotSetException.java
View File

@ -1,43 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.LABitSet;
public class MismatchedNotSetException extends MismatchedSetException {
/** Used for remote debugger deserialization */
public MismatchedNotSetException() {;}
public MismatchedNotSetException(BaseRecognizer recognizer, LABitSet expecting) {
super(recognizer, expecting);
}
public String toString() {
return "MismatchedNotSetException("+getUnexpectedType()+"!="+expecting+")";
}
}

45
runtime/Java/src/org/antlr/v4/runtime/MismatchedRangeException.java
View File

@ -1,45 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
public class MismatchedRangeException extends RecognitionException {
public int a,b;
/** Used for remote debugger deserialization */
public MismatchedRangeException() {;}
public MismatchedRangeException(BaseRecognizer recognizer, int a, int b) {
super(recognizer);
this.a = a;
this.b = b;
}
public String toString() {
return "MismatchedNotSetException("+getUnexpectedType()+" not in ["+a+","+b+"])";
}
}

43
runtime/Java/src/org/antlr/v4/runtime/MismatchedSetException.java
View File

@ -1,43 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.LABitSet;
public class MismatchedSetException extends RecognitionException {
/** Used for remote debugger deserialization */
public MismatchedSetException() {;}
public MismatchedSetException(BaseRecognizer recognizer, LABitSet firstSet) {
super(recognizer, firstSet);
}
public String toString() {
return "MismatchedSetException("+getUnexpectedType()+"!="+expecting+")";
}
}

44
runtime/Java/src/org/antlr/v4/runtime/MismatchedTokenException.java
View File

@ -1,44 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.LABitSet;
/** A mismatched char or Token or tree node */
public class MismatchedTokenException extends RecognitionException {
/** Used for remote debugger deserialization */
public MismatchedTokenException() {;}
public MismatchedTokenException(BaseRecognizer recognizer, int firstSet) {
super(recognizer, LABitSet.of(firstSet));
}
public String toString() {
return "MismatchedTokenException("+getUnexpectedType()+"!="+expecting+")";
}
}

47
runtime/Java/src/org/antlr/v4/runtime/MismatchedTreeNodeException.java
View File

@ -1,47 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.LABitSet;
/**
*/
public class MismatchedTreeNodeException extends RecognitionException {
public MismatchedTreeNodeException() {
}
public MismatchedTreeNodeException(BaseRecognizer recognizer,
int firstSet)
{
super(recognizer, LABitSet.of(firstSet));
}
public String toString() {
return "MismatchedTreeNodeException("+getUnexpectedType()+"!="+expecting+")";
}
}

56
runtime/Java/src/org/antlr/v4/runtime/MissingTokenException.java
View File

@ -1,56 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
/** We were expecting a token but it's not found. The current token
* is actually what we wanted next. Used for tree node errors too.
*/
public class MissingTokenException extends MismatchedTokenException {
public Object inserted;
/** Used for remote debugger deserialization */
public MissingTokenException() {;}
public MissingTokenException(BaseRecognizer recognizer, int expecting, Object inserted) {
super(recognizer,expecting);
this.inserted = inserted;
}
public int getMissingType() {
return expecting.getSingleElement();
}
public String toString() {
if ( inserted!=null && token!=null ) {
return "MissingTokenException(inserted "+inserted+" at "+token.getText()+")";
}
if ( token!=null ) {
return "MissingTokenException(at "+token.getText()+")";
}
return "MissingTokenException";
}
}

51
runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java
View File

@ -1,51 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.CharStream;
import org.antlr.v4.runtime.misc.LABitSet;
public class NoViableAltException extends RecognitionException {
/** Used for remote debugger deserialization */
public NoViableAltException() {;}
public NoViableAltException(BaseRecognizer recognizer,
LABitSet firstSet)
{
super(recognizer, firstSet);
}
public String toString() {
if ( recognizer.state.input instanceof CharStream) {
return "NoViableAltException('"+(char)getUnexpectedType()+", expecting "+expecting+")";
}
else {
return "NoViableAltException('"+getUnexpectedType()+", expecting "+expecting+")";
}
}
}

98
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -1,98 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenStream;
import org.antlr.v4.runtime.misc.LABitSet;
/** A parser for TokenStreams. "parser grammars" result in a subclass
* of this.
*/
public class Parser extends BaseRecognizer {
public Parser(TokenStream input) {
super(input);
}
public Parser(TokenStream input, ParserSharedState state) {
super(input, state); // share the state object with another parser
}
public void reset() {
super.reset(); // reset all recognizer state variables
if ( state.input!=null ) {
state.input.seek(0); // rewind the input
}
}
protected Object getCurrentInputSymbol() {
return ((TokenStream)state.input).LT(1);
}
protected Object getMissingSymbol(RecognitionException e,
int expectedTokenType,
LABitSet follow)
{
String tokenText = null;
if ( expectedTokenType== Token.EOF ) tokenText = "<missing EOF>";
else tokenText = "<missing "+getTokenNames()[expectedTokenType]+">";
CommonToken t = new CommonToken(expectedTokenType, tokenText);
Token current = ((TokenStream)state.input).LT(1);
if ( current.getType() == Token.EOF ) {
current = ((TokenStream)state.input).LT(-1);
}
t.line = current.getLine();
t.charPositionInLine = current.getCharPositionInLine();
t.channel = Token.DEFAULT_CHANNEL;
return t;
}
/** Set the token stream and reset the parser */
public void setTokenStream(TokenStream input) {
this.state.input = null;
reset();
this.state.input = input;
}
public TokenStream getTokenStream() {
return (TokenStream)state.input;
}
public String getSourceName() {
return state.input.getSourceName();
}
public void traceIn(String ruleName, int ruleIndex) {
super.traceIn(ruleName, ruleIndex, ((TokenStream)state.input).LT(1));
}
public void traceOut(String ruleName, int ruleIndex) {
super.traceOut(ruleName, ruleIndex, ((TokenStream)state.input).LT(1));
}
}

57
runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
View File

@ -1,57 +0,0 @@
/*
[BSD]
Copyright (c) 2010 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.Token;
import org.antlr.v4.runtime.misc.LABitSet;
/** Rules return values in an object containing all the values.
* Besides the properties defined in
* RuleLabelScope.predefinedRulePropertiesScope there may be user-defined
* return values. This class simply defines the minimum properties that
* are always defined and methods to access the others that might be
* available depending on output option such as template and tree.
*
* Note text is not an actual property of the return value, it is computed
* from start and stop using the input stream's toString() method. I
* could add a ctor to this so that we can pass in and store the input
* stream, but I'm not sure we want to do that. It would seem to be undefined
* to get the .text property anyway if the rule matches tokens from multiple
* input streams.
*
* I do not use getters for fields of objects that are used simply to
* group values such as this aggregate. The getters/setters are there to
* satisfy the superclass interface.
*/
public class ParserRuleContext extends RuleContext {
public Token start, stop;
public Object getStart() { return start; }
public Object getStop() { return stop; }
public ParserRuleContext() { super(); }
public ParserRuleContext(LABitSet follow) { super(follow); }
}

102
runtime/Java/src/org/antlr/v4/runtime/ParserSharedState.java
View File

@ -1,102 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.IntStream;
import org.antlr.v4.runtime.misc.QStack;
import java.util.List;
import java.util.Map;
/** The set of fields needed by an abstract recognizer to recognize input
* and recover from errors etc... As a separate state object, it can be
* shared among multiple grammars; e.g., when one grammar imports another.
*/
public class ParserSharedState {
public IntStream input;
/** First on stack is fake a call to start rule from S' : S EOF ;
* Generated start rule does this.
*/
public QStack<RuleContext> ctx;
/** This is true when we see an error and before having successfully
* matched a token. Prevents generation of more than one error message
* per error.
*/
public boolean errorRecovery = false;
/** The index into the input stream where the last error occurred.
* This is used to prevent infinite loops where an error is found
* but no token is consumed during recovery...another error is found,
* ad naseum. This is a failsafe mechanism to guarantee that at least
* one token/tree node is consumed for two errors.
*/
public int lastErrorIndex = -1;
/** In lieu of a return value, this indicates that a rule or token
* has failed to match. Reset to false upon valid token match.
*/
public boolean failed = false;
/** Did the recognizer encounter a syntax error? Track how many. */
public int syntaxErrors = 0;
/** If 0, no backtracking is going on. Safe to exec actions etc...
* If >0 then it's the level of backtracking.
*/
public int backtracking = 0;
/** An array[size num rules] of Map<Integer,Integer> that tracks
* the stop token index for each rule. ruleMemo[ruleIndex] is
* the memoization table for ruleIndex. For key ruleStartIndex, you
* get back the stop token for associated rule or MEMO_RULE_FAILED.
*
* This is only used if rule memoization is on (which it is by default).
*/
public Map[] ruleMemo;
List<ANTLRParserListener> listeners;
public ParserSharedState() {
ctx = new QStack<RuleContext>();
}
// public RecognizerSharedState(RecognizerSharedState state) {
// this.ctx = state.ctx;
// this.errorRecovery = state.errorRecovery;
// this.lastErrorIndex = state.lastErrorIndex;
// this.failed = state.failed;
// this.syntaxErrors = state.syntaxErrors;
// this.backtracking = state.backtracking;
// if ( state.ruleMemo!=null ) {
// this.ruleMemo = new Map[state.ruleMemo.length];
// System.arraycopy(state.ruleMemo, 0, this.ruleMemo, 0, state.ruleMemo.length);
// }
// }
}

199
runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java
View File

@ -1,199 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.IntStream;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenStream;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
import org.antlr.runtime.tree.TreeAdaptor;
import org.antlr.runtime.tree.TreeNodeStream;
import org.antlr.v4.runtime.misc.LABitSet;
/** The root of the ANTLR exception hierarchy.
*
* To avoid English-only error messages and to generally make things
* as flexible as possible, these exceptions are not created with strings,
* but rather the information necessary to generate an error. Then
* the various reporting methods in Parser and Lexer can be overridden
* to generate a localized error message. For example, MismatchedToken
* exceptions are built with the expected token type.
* So, don't expect getMessage() to return anything.
*
* Note that as of Java 1.4, you can access the stack trace, which means
* that you can compute the complete trace of rules from the start symbol.
* This gives you considerable context information with which to generate
* useful error messages.
*
* ANTLR generates code that throws exceptions upon recognition error and
* also generates code to catch these exceptions in each rule. If you
* want to quit upon first error, you can turn off the automatic error
* handling mechanism using rulecatch action, but you still need to
* override methods mismatch and recoverFromMismatchSet.
*
* In general, the recognition exceptions can track where in a grammar a
* problem occurred and/or what was the expected input. While the parser
* knows its state (such as current input symbol and line info) that
* state can change before the exception is reported so current token index
* is computed and stored at exception time. From this info, you can
* perhaps print an entire line of input not just a single token, for example.
* Better to just say the recognizer had a problem and then let the parser
* figure out a fancy report.
*/
public class RecognitionException extends RuntimeException {
/** Who threw the exception? */
public BaseRecognizer recognizer;
public LABitSet expecting;
/** What is index of token/char were we looking at when the error occurred? */
public int index;
/** The current Token when an error occurred. Since not all streams
* can retrieve the ith Token, we have to track the Token object.
* For parsers. Even when it's a tree parser, token might be set.
*/
public Token token;
/** If this is a tree parser exception, node is set to the node with
* the problem.
*/
public Object node;
/** The current char when an error occurred. For lexers. */
public int c;
/** Track the line at which the error occurred in case this is
* generated from a lexer. We need to track this since the
* unexpected char doesn't carry the line info.
*/
public int line;
public int charPositionInLine;
/** If you are parsing a tree node stream, you will encounter som
* imaginary nodes w/o line/col info. We now search backwards looking
* for most recent token with line/col info, but notify getErrorHeader()
* that info is approximate.
*/
public boolean approximateLineInfo;
/** Used for remote debugger deserialization */
public RecognitionException() {
}
public RecognitionException(BaseRecognizer recognizer) {
this(recognizer, null);
}
public RecognitionException(BaseRecognizer recognizer, LABitSet firstSet) {
this.recognizer = recognizer;
// firstSet is what can we're expecting within rule that calls this ctor.
// must combine with context-sensitive FOLLOW of that rule.
LABitSet viableTokensFollowingThisRule = recognizer.computeNextViableTokenSet();
this.expecting = viableTokensFollowingThisRule.or(firstSet);
IntStream input = recognizer.state.input;
this.index = input.index();
if ( input instanceof TokenStream ) {
this.token = ((TokenStream)input).LT(1);
this.line = token.getLine();
this.charPositionInLine = token.getCharPositionInLine();
}
if ( input instanceof TreeNodeStream ) {
extractInformationFromTreeNodeStream(input);
}
else if ( input instanceof CharStream) {
this.c = input.LA(1);
this.line = ((CharStream)input).getLine();
this.charPositionInLine = ((CharStream)input).getCharPositionInLine();
}
else {
this.c = input.LA(1);
}
}
protected void extractInformationFromTreeNodeStream(IntStream input) {
TreeNodeStream nodes = (TreeNodeStream)input;
this.node = nodes.LT(1);
TreeAdaptor adaptor = nodes.getTreeAdaptor();
Token payload = adaptor.getToken(node);
if ( payload!=null ) {
this.token = payload;
if ( payload.getLine()<= 0 ) {
// imaginary node; no line/pos info; scan backwards
int i = -1;
Object priorNode = nodes.LT(i);
while ( priorNode!=null ) {
Token priorPayload = adaptor.getToken(priorNode);
if ( priorPayload!=null && priorPayload.getLine()>0 ) {
// we found the most recent real line / pos info
this.line = priorPayload.getLine();
this.charPositionInLine = priorPayload.getCharPositionInLine();
this.approximateLineInfo = true;
break;
}
--i;
priorNode = nodes.LT(i);
}
}
else { // node created from real token
this.line = payload.getLine();
this.charPositionInLine = payload.getCharPositionInLine();
}
}
else if ( this.node instanceof Tree) {
this.line = ((Tree)this.node).getLine();
this.charPositionInLine = ((Tree)this.node).getCharPositionInLine();
if ( this.node instanceof CommonTree) {
this.token = ((CommonTree)this.node).token;
}
}
else {
int type = adaptor.getType(this.node);
String text = adaptor.getText(this.node);
this.token = new CommonToken(type, text);
}
}
/** Return the token type or char of the unexpected input element */
public int getUnexpectedType() {
if ( recognizer.state.input instanceof TokenStream) {
return token.getType();
}
else if ( recognizer.state.input instanceof TreeNodeStream ) {
TreeNodeStream nodes = (TreeNodeStream)recognizer.state.input;
TreeAdaptor adaptor = nodes.getTreeAdaptor();
return adaptor.getType(node);
}
else {
return c;
}
}
}

55
runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
View File

@ -1,55 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.LABitSet;
/** Rules can return start/stop info as well as possible trees and templates.
* Each context must have a FOLLOW context. It's EOF if none is specified.
*/
public class RuleContext {
/** Track the set of token types that can follow any rule invocation. */
public LABitSet follow;
/** Return the start token or tree */
public Object getStart() { return null; }
/** Return the stop token or tree */
public Object getStop() { return null; }
/** Has a value potentially if output=AST; */
public Object getTree() { return null; }
/** Has a value potentially if output=template; Don't use StringTemplate
* type as it then causes a dependency with ST lib.
*/
public Object getTemplate() { return null; }
public RuleContext() { this(LABitSet.EOF_SET); }
public RuleContext(LABitSet follow) { this.follow = follow; }
}

52
runtime/Java/src/org/antlr/v4/runtime/UnwantedTokenException.java
View File

@ -1,52 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime;
import org.antlr.runtime.Token;
/** An extra token while parsing a TokenStream */
public class UnwantedTokenException extends MismatchedTokenException {
/** Used for remote debugger deserialization */
public UnwantedTokenException() {;}
public UnwantedTokenException(BaseRecognizer recognizer, int expecting) {
super(recognizer, expecting);
}
public Token getUnexpectedToken() {
return token;
}
public String toString() {
String exp = ", expected "+expecting;
if ( token==null ) {
return "UnwantedTokenException(found="+null+exp+")";
}
return "UnwantedTokenException(found="+token.getText()+exp+")";
}
}

228
runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java
View File

@ -1,228 +0,0 @@
/*
[BSD]
Copyright (c) 2010 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.dfa;
import org.antlr.runtime.IntStream;
import org.antlr.runtime.Token;
import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.RecognitionException;
/** A DFA implemented as a set of transition tables.
*
* Any state that has a semantic predicate edge is special; those states
* are generated with if-then-else structures in a specialStateTransition()
* which is generated by cyclicDFA template.
*
* There are at most 32767 states (16-bit signed short).
* Could get away with byte sometimes but would have to generate different
* types and the simulation code too. For a point of reference, the Java
* lexer's Tokens rule DFA has 326 states roughly.
*/
public class DFA {
public short[] eof;
public char[] max;
public short[] accept;
/** { target1, npairs1, range-pairs1,
target2, npairs2, range-pairs2, ... }
*/
public int[][] set_edges;
public int[][] pred_edges; // 'a'&&{p1}?
public short[][] transition;
public short[] action_index;
public int decisionNumber;
/** Which recognizer encloses this DFA? Needed to check backtracking */
//public BaseRecognizer recognizer;
public static final boolean debug = false;
/** From the input stream, predict what alternative will succeed
* using this DFA (representing the covering regular approximation
* to the underlying CFL). Return an alternative number 1..n. Throw
* an exception upon error.
*/
public int predict(IntStream input)
throws RecognitionException
{
if ( debug ) {
System.err.println("Enter DFA.predict for decision "+decisionNumber);
}
//int mark = input.mark(); // remember where decision started in input
int prevAcceptMarker = -1;
int prevAcceptState = -1;
int s = 0; // we always start at s0
try {
while ( true ) {
if ( debug ) System.err.println("DFA "+decisionNumber+" state "+s+" LA(1)="+(char)input.LA(1)+"("+input.LA(1)+
"), index="+input.index());
if ( accept[s] >= 1 ) {
// TODO: have to keep going and then backtrack if we fail!!!!
if ( debug ) System.err.println("accept; predict "+accept[s]+" from state "+s);
prevAcceptMarker = input.mark();
prevAcceptState = s;
// keep going
}
// look for a normal char transition
char c = (char)input.LA(1); // -1 == \uFFFF, all types fit in 64k space
if ( c<=max[s] ) {
int snext = transition[s][c]; // move to next state
if ( snext < 0 ) {
// was in range but not valid transition
// TODO: check if eof[s]>=0, indicating that EOF goes to another
// state.
// TODO: refactor this common fail code
if ( prevAcceptMarker<0 ) noViableAlt(s,input);
input.rewind(prevAcceptMarker);
s = prevAcceptState;
if ( action_index[s]>=0 ) action(action_index[s]);
System.err.println("accept state "+s+" with ttype "+accept[s]+" at index "+input.index());
return accept[s];
}
s = snext;
input.consume();
continue;
}
if ( set_edges[s]!=null ) {
// TODO: unicode
}
if ( pred_edges[s]!=null ) {
// TODO: gated or disambiguating sem
}
if ( c==(char)Token.EOF && eof[s]>=0 ) { // EOF Transition to accept state?
if ( debug ) System.err.println("accept via EOF; predict "+accept[eof[s]]+" from "+eof[s]);
// TODO: have to keep going and then backtrack if we fail??
return accept[eof[s]];
}
// not in range and not EOF/EOT, must be invalid symbol
if ( debug ) {
System.err.println("max["+s+"]="+max[s]);
System.err.println("eof["+s+"]="+eof[s]);
if ( transition[s]!=null ) {
System.err.print("transitions=");
for (int p=0; p<transition[s].length; p++) {
System.err.print(transition[s][p]+" ");
}
System.err.println();
}
}
if ( prevAcceptMarker<0 ) noViableAlt(s,input);
input.rewind(prevAcceptMarker);
s = prevAcceptState;
if ( action_index[s]>=0 ) action(action_index[s]);
System.err.println("accept state "+s+" with ttype "+accept[s]+" at index "+input.index());
return accept[s];
}
}
finally {
// input.rewind(mark);
}
}
// subclass needs to override these if there are sempreds or actions in lexer rules
public boolean sempred(int predIndex) {
return true;
}
public void action(int actionIndex) {
}
public void noViableAlt(int s, IntStream input) throws NoViableAltException {
NoViableAltException nvae = new NoViableAltException();
// new NoViableAltException(getDescription(),
// decisionNumber,
// s,
// input);
error(nvae);
throw nvae;
}
/** A hook for debugging interface */
public void error(NoViableAltException nvae) { ; }
public int specialStateTransition(int s, IntStream input)
throws NoViableAltException
{
return -1;
}
public String getDescription() {
return "n/a";
}
/** Given a String that has a run-length-encoding of some unsigned shorts
* like "\1\2\3\9", convert to short[] {2,9,9,9}. We do this to avoid
* static short[] which generates so much init code that the class won't
* compile. :(
*/
public static short[] unpackEncodedString(String encodedString) {
// walk first to find how big it is.
int size = 0;
for (int i=0; i<encodedString.length(); i+=2) {
size += encodedString.charAt(i);
}
short[] data = new short[size];
int di = 0;
for (int i=0; i<encodedString.length(); i+=2) {
char n = encodedString.charAt(i);
char v = encodedString.charAt(i+1);
// add v n times to data
for (int j=1; j<=n; j++) {
data[di++] = (short)v;
}
}
return data;
}
/** Hideous duplication of code, but I need different typed arrays out :( */
public static char[] unpackEncodedStringToUnsignedChars(String encodedString) {
// walk first to find how big it is.
int size = 0;
for (int i=0; i<encodedString.length(); i+=2) {
size += encodedString.charAt(i);
}
char[] data = new char[size];
int di = 0;
for (int i=0; i<encodedString.length(); i+=2) {
char n = encodedString.charAt(i);
char v = encodedString.charAt(i+1);
// add v n times to data
for (int j=1; j<=n; j++) {
data[di++] = v;
}
}
return data;
}
/*
public int specialTransition(int state, int symbol) {
return 0;
}
*/
}

219
runtime/Java/src/org/antlr/v4/runtime/misc/LABitSet.java
View File

@ -1,219 +0,0 @@
package org.antlr.v4.runtime.misc;
import org.antlr.runtime.Token;
/** */
public class LABitSet implements Cloneable {
public final static int BITS = 64; // number of bits / long
public final static int LOG_BITS = 6; // 2^6 == 64
/* We will often need to do a mod operator (i mod nbits). Its
* turns out that, for powers of two, this mod operation is
* same as (i & (nbits-1)). Since mod is slow, we use a
* precomputed mod mask to do the mod instead.
*/
public final static int MOD_MASK = BITS - 1;
public static final LABitSet EOF_SET = LABitSet.of(Token.EOF);
/** The actual data bits */
public long bits[];
public boolean EOF; // is EOF in set (-1)?
/** Construct a bitset of size one word (64 bits) */
public LABitSet() {
this(BITS);
}
/** Construct a bitset given the size
* @param nbits The size of the bitset in bits
*/
public LABitSet(int nbits) {
bits = new long[((nbits - 1) >> LOG_BITS) + 1];
}
/** Construction from a static array of longs */
public LABitSet(long[] bits_) {
if ( bits_==null || bits_.length==0 ) bits = new long[1];
else bits = bits_;
}
/** Construction from a static array of longs */
public LABitSet(long[] bits_, boolean EOF) {
this(bits_);
this.EOF = EOF;
}
public static LABitSet of(int el) {
LABitSet s = new LABitSet(el + 1);
s.add(el);
return s;
}
/** or this element into this set (grow as necessary to accommodate) */
public void add(int el) {
//System.out.println("add("+el+")");
if ( el==Token.EOF ) { EOF = true; return; }
int n = wordNumber(el);
//System.out.println("word number is "+n);
//System.out.println("bits.length "+bits.length);
if (n >= bits.length) {
growToInclude(el);
}
bits[n] |= bitMask(el);
}
public boolean member(int el) {
if ( el == Token.EOF ) return EOF;
int n = wordNumber(el);
if (n >= bits.length) return false;
return (bits[n] & bitMask(el)) != 0;
}
/** return this | a in a new set */
public LABitSet or(LABitSet a) {
if ( a==null ) {
return this;
}
LABitSet s = (LABitSet)this.clone();
s.orInPlace((LABitSet)a);
return s;
}
public void orInPlace(LABitSet a) {
if ( a==null ) {
return;
}
// If this is smaller than a, grow this first
if (a.bits.length > bits.length) {
setSize(a.bits.length);
}
int min = Math.min(bits.length, a.bits.length);
for (int i = min - 1; i >= 0; i--) {
bits[i] |= a.bits[i];
}
EOF = EOF | a.EOF;
}
// remove this element from this set
public void remove(int el) {
if ( el==Token.EOF ) { EOF = false; return; }
int n = wordNumber(el);
if (n >= bits.length) {
throw new IllegalArgumentException(el+" is outside set range of "+bits.length+" words");
}
bits[n] &= ~bitMask(el);
}
public Object clone() {
LABitSet s;
try {
s = (LABitSet)super.clone();
s.bits = new long[bits.length];
System.arraycopy(bits, 0, s.bits, 0, bits.length);
s.EOF = EOF;
return s;
}
catch (CloneNotSupportedException e) {
e.printStackTrace(System.err);
}
return null;
}
/**
* Sets the size of a set.
* @param nwords how many words the new set should be
*/
void setSize(int nwords) {
long newbits[] = new long[nwords];
int n = Math.min(nwords, bits.length);
System.arraycopy(bits, 0, newbits, 0, n);
bits = newbits;
}
/** Get the first element you find and return it. */
public int getSingleElement() {
for (int i = 0; i < (bits.length << LOG_BITS); i++) {
if (member(i)) {
return i;
}
}
return Token.INVALID_TOKEN_TYPE;
}
/** Transform a bit set into a string by formatting each element as an integer
* separator The string to put in between elements
* @return A commma-separated list of values
*/
public String toString() {
StringBuffer buf = new StringBuffer();
String separator = ",";
boolean havePrintedAnElement = false;
buf.append('{');
if ( EOF ) { buf.append("EOF"); havePrintedAnElement=true; }
for (int i = 0; i < (bits.length << LOG_BITS); i++) {
if (member(i)) {
if ( havePrintedAnElement ) {
buf.append(separator);
}
buf.append(i);
havePrintedAnElement = true;
}
}
buf.append('}');
return buf.toString();
}
// /**Create a string representation where instead of integer elements, the
// * ith element of vocabulary is displayed instead. Vocabulary is a Vector
// * of Strings.
// * separator The string to put in between elements
// * @return A commma-separated list of character constants.
// */
// public String toString(String separator, List vocabulary) {
// String str = "";
// for (int i = 0; i < (bits.length << LOG_BITS); i++) {
// if (member(i)) {
// if (str.length() > 0) {
// str += separator;
// }
// if (i >= vocabulary.size()) {
// str += "'" + (char)i + "'";
// }
// else if (vocabulary.get(i) == null) {
// str += "'" + (char)i + "'";
// }
// else {
// str += (String)vocabulary.get(i);
// }
// }
// }
// return str;
// }
/**
* Grows the set to a larger number of bits.
* @param bit element that must fit in set
*/
public void growToInclude(int bit) {
int newSize = Math.max(bits.length << 1, numWordsToHold(bit));
long newbits[] = new long[newSize];
System.arraycopy(bits, 0, newbits, 0, bits.length);
bits = newbits;
}
static long bitMask(int bitNumber) {
int bitPosition = bitNumber & MOD_MASK; // bitNumber mod BITS
return 1L << bitPosition;
}
static int numWordsToHold(int el) {
return (el >> LOG_BITS) + 1;
}
static int wordNumber(int bit) {
return bit >> LOG_BITS; // bit / BITS
}
}

46
runtime/Java/src/org/antlr/v4/runtime/misc/QStack.java
View File

@ -1,46 +0,0 @@
package org.antlr.v4.runtime.misc;
import java.util.EmptyStackException;
/** A quicker stack than Stack */
public class QStack<T> {
Object[] elements;
public int sp = -1;
public QStack() {
elements = new Object[10];
}
public QStack(QStack s) {
elements = new Object[s.elements.length];
System.arraycopy(s.elements, 0, elements, 0, s.elements.length);
this.sp = s.sp;
}
public void push(T fset) {
if ( (sp+1)>=elements.length ) {
Object[] f = new Object[elements.length*2];
System.arraycopy(elements, 0, f, 0, elements.length);
elements = f;
}
elements[++sp] = fset;
}
public T peek() {
if ( sp<0 ) throw new EmptyStackException();
return (T)elements[sp];
}
public T get(int i) {
if ( i<0 ) throw new IllegalArgumentException("i<0");
if ( i>sp ) throw new IllegalArgumentException("i>"+sp);
return (T)elements[sp];
}
public T pop() {
if ( sp<0 ) throw new EmptyStackException();
return (T)elements[sp--];
}
public void clear() { sp = -1; }
}

222
runtime/Java/src/org/antlr/v4/runtime/pda/Bytecode.java
View File

@ -1,222 +0,0 @@
package org.antlr.v4.runtime.pda;
import org.antlr.runtime.Token;
import java.util.ArrayList;
import java.util.List;
/** */
public class Bytecode {
public static final int MAX_OPNDS = 3; // Or single opnd indicating variable number
public static final int ADDR_SIZE = 2;
public enum OperandType {
NONE(0), BYTE(1), CHAR(2), ADDR(ADDR_SIZE), SHORT(2), INT(4), VARARGS(0);
public int sizeInBytes;
OperandType(int sizeInBytes) { this.sizeInBytes = sizeInBytes; }
}
public static class Instruction {
String name; // E.g., "load_str", "new"
OperandType[] type = new OperandType[MAX_OPNDS];
int n = 0;
public Instruction(String name) {
this(name,OperandType.NONE,OperandType.NONE,OperandType.NONE); n=0;
}
public Instruction(String name, OperandType a) {
this(name,a,OperandType.NONE,OperandType.NONE); n=1;
}
public Instruction(String name, OperandType a, OperandType b) {
this(name,a,b,OperandType.NONE); n=2;
}
public Instruction(String name, OperandType a, OperandType b, OperandType c) {
this.name = name;
type[0] = a;
type[1] = b;
type[2] = c;
n = MAX_OPNDS;
}
}
// don't use enum for efficiency; don't want code block to
// be an array of objects (Bytecode[]). We want it to be byte[].
// INSTRUCTION BYTECODES (byte is signed; use a short to keep 0..255)
public static final short ACCEPT = 1;
public static final short JMP = 2;
public static final short SPLIT = 3;
public static final short MATCH8 = 4;
public static final short MATCH16 = 5;
public static final short RANGE8 = 6;
public static final short RANGE16 = 7;
public static final short WILDCARD = 8;
public static final short SET = 9;
public static final short CALL = 10; // JMP with a push
public static final short RET = 11; // an accept instr for fragment rules
public static final short LABEL = 12;
public static final short SAVE = 13;
public static final short SEMPRED = 14;
public static final short ACTION = 15;
public static final short NOT = 16; // not next match instr
public static final short SWITCH = 17;
/** Used for disassembly; describes instruction set */
public static Instruction[] instructions = new Instruction[] {
null, // <INVALID>
new Instruction("accept", OperandType.SHORT), // index is the opcode
new Instruction("jmp", OperandType.ADDR),
new Instruction("split", OperandType.VARARGS),
new Instruction("match8", OperandType.BYTE),
new Instruction("match16", OperandType.CHAR),
new Instruction("range8", OperandType.BYTE, OperandType.BYTE),
new Instruction("range16", OperandType.CHAR, OperandType.CHAR),
new Instruction("wildcard"),
new Instruction("set", OperandType.SHORT),
new Instruction("call", OperandType.ADDR),
new Instruction("ret"),
new Instruction("label", OperandType.SHORT),
new Instruction("save", OperandType.SHORT),
new Instruction("sempred", OperandType.SHORT, OperandType.SHORT), // sempred ruleIndex, predIndex
new Instruction("action", OperandType.SHORT, OperandType.SHORT), // action ruleIndex, actionIndex
new Instruction("not"),
new Instruction("switch", OperandType.SHORT),
};
public static String disassemble(byte[] code, int start, boolean operandsAreChars) {
StringBuilder buf = new StringBuilder();
int i=start;
while (i<code.length) {
i = disassembleInstruction(buf, code, i, operandsAreChars);
buf.append('\n');
}
return buf.toString();
}
public static String disassemble(byte[] code) { return disassemble(code, 0, true); }
public static String disassemble(byte[] code, boolean operandsAreChars) {
return disassemble(code, 0, operandsAreChars);
}
public static String disassembleInstruction(byte[] code, int ip, boolean operandsAreChars) {
StringBuilder buf = new StringBuilder();
disassembleInstruction(buf, code, ip, operandsAreChars);
return buf.toString();
}
public static int disassembleInstruction(StringBuilder buf, byte[] code, int ip, boolean operandsAreChars) {
int opcode = code[ip];
if ( ip>=code.length ) {
throw new IllegalArgumentException("ip out of range: "+ip);
}
Bytecode.Instruction I =
Bytecode.instructions[opcode];
if ( I==null ) {
throw new IllegalArgumentException("no such instruction "+opcode+
" at address "+ip);
}
String instrName = I.name;
buf.append( String.format("%04d:\t%-14s", ip, instrName) );
ip++;
if ( I.n==0 ) {
buf.append(" ");
return ip;
}
List<String> operands = new ArrayList<String>();
if ( I.n==1 && I.type[0]==OperandType.VARARGS) { // get n (opnd) operands
int n = getShort(code, ip);
ip += 2;
// operands.add(String.valueOf(n)); don't show n in varargs
for (int j=1; j<=n; j++) {
operands.add(String.valueOf(getShort(code, ip)));
ip += ADDR_SIZE; // VARARGS only works on address for now
}
}
else {
for (int i=0; i<I.n; i++) {
switch ( I.type[i] ) {
case NONE:
break;
case BYTE:
if ( operandsAreChars ) operands.add(quotedCharLiteral((char)code[ip]));
else operands.add(String.valueOf(code[ip]));
break;
case CHAR :
if ( operandsAreChars ) operands.add(quotedCharLiteral(getShort(code, ip)));
else operands.add(String.valueOf(getShort(code, ip)));
break;
case INT :
if ( operandsAreChars ) operands.add(quotedCharLiteral(getInt(code, ip)));
else operands.add(String.valueOf(getInt(code, ip)));
case SHORT :
case ADDR :
operands.add(String.valueOf(getShort(code, ip)));
break;
default :
System.err.println("invalid opnd type: "+I.type[i]);
}
ip += I.type[i].sizeInBytes;
}
}
for (int i = 0; i < operands.size(); i++) {
String s = operands.get(i);
if ( i>0 ) buf.append(", ");
buf.append( s );
}
return ip;
}
public static int getInt(byte[] memory, int index) {
int b1 = memory[index++]&0xFF; // high byte
int b2 = memory[index++]&0xFF;
int b3 = memory[index++]&0xFF;
int b4 = memory[index++]&0xFF; // low byte
return b1<<(8*3) | b2<<(8*2) | b3<<(8*1) | b4;
}
public static int getShort(byte[] memory, int index) {
int b1 = memory[index++]&0xFF; // mask off sign-extended bits
int b2 = memory[index++]&0xFF;
return b1<<(8*1) | b2;
}
public static String LiteralCharValueEscape[] = new String[255];
static {
LiteralCharValueEscape['\n'] = "\\n";
LiteralCharValueEscape['\r'] = "\\r";
LiteralCharValueEscape['\t'] = "\\t";
LiteralCharValueEscape['\b'] = "\\b";
LiteralCharValueEscape['\f'] = "\\f";
LiteralCharValueEscape['\\'] = "\\\\";
LiteralCharValueEscape['\''] = "\\'";
}
/** Return a string representing the escaped char for code c. E.g., If c
* has value 0x100, you will get "\u0100". ASCII gets the usual
* char (non-hex) representation. Control characters are spit out
* as unicode.
*/
public static String quotedCharLiteral(int c) {
if ( c== Token.EOF ) return "'<EOF>'";
if ( c<LiteralCharValueEscape.length && LiteralCharValueEscape[c]!=null ) {
return '\''+LiteralCharValueEscape[c]+'\'';
}
if ( Character.UnicodeBlock.of((char)c)==Character.UnicodeBlock.BASIC_LATIN &&
!Character.isISOControl((char)c) ) {
if ( c=='\\' ) {
return "'\\\\'";
}
if ( c=='\'') {
return "'\\''";
}
return '\''+Character.toString((char)c)+'\'';
}
// turn on the bit above max "\uFFFF" value so that we pad with zeros
// then only take last 4 digits
String hex = Integer.toHexString(c|0x10000).toUpperCase().substring(1,5);
String unicodeStr = "'\\u"+hex+"'";
return unicodeStr;
}
}

75
runtime/Java/src/org/antlr/v4/runtime/pda/NFAStack.java
View File

@ -1,75 +0,0 @@
package org.antlr.v4.runtime.pda;
/** Identical to ANTLR's static grammar analysis NFAContext object */
public class NFAStack {
public static final NFAStack EMPTY = new NFAStack(null, -1);
public NFAStack parent;
/** The NFA state following state that invoked another rule's start state
* is recorded on the rule invocation context stack.
*/
public int returnAddr;
/** Computing the hashCode is very expensive and NFA.addToClosure()
* uses it to track when it's seen a state|ctx before to avoid
* infinite loops. As we add new contexts, record the hash code
* as this + parent.cachedHashCode. Avoids walking
* up the tree for every hashCode(). Note that this caching works
* because a context is a monotonically growing tree of context nodes
* and nothing on the stack is ever modified...ctx just grows
* or shrinks.
*/
protected int cachedHashCode;
public NFAStack(NFAStack parent, int returnAddr) {
this.parent = parent;
this.returnAddr = returnAddr;
if ( returnAddr >= 0 ) {
this.cachedHashCode = returnAddr;
}
if ( parent!=null ) {
this.cachedHashCode += parent.cachedHashCode;
}
}
public int hashCode() { return cachedHashCode; }
/** Two contexts are equals() if both have
* same call stack; walk upwards to the root.
* Recall that the root sentinel node has no parent.
* Note that you may be comparing contextsv in different alt trees.
*/
public boolean equals(Object o) {
NFAStack other = ((NFAStack)o);
if ( this.cachedHashCode != other.cachedHashCode ) {
return false; // can't be same if hash is different
}
if ( this==other ) return true;
// System.out.println("comparing "+this+" with "+other);
NFAStack sp = this;
while ( sp.parent!=null && other.parent!=null ) {
if ( sp.returnAddr != other.returnAddr) return false;
sp = sp.parent;
other = other.parent;
}
if ( !(sp.parent==null && other.parent==null) ) {
return false; // both pointers must be at their roots after walk
}
return true;
}
public String toString() {
StringBuffer buf = new StringBuffer();
NFAStack sp = this;
buf.append("[");
while ( sp.parent!=null ) {
buf.append(sp.returnAddr);
buf.append(" ");
sp = sp.parent;
}
buf.append("$]");
return buf.toString();
}
}

654
runtime/Java/src/org/antlr/v4/runtime/pda/PDA.java
View File

@ -1,654 +0,0 @@
package org.antlr.v4.runtime.pda;
import org.antlr.runtime.*;
import org.antlr.v4.runtime.CommonToken;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/** A (nondeterministic) pushdown bytecode machine for lexing and LL prediction.
* Derived partially from Cox' description of Thompson's 1960s work:
* http://swtch.com/~rsc/regexp/regexp2.html
*
* Primary difference is that I've extended to have actions, semantic predicates
* and a stack for rule invocation.
*/
public class PDA {
public static class InvalidElement extends RuntimeException {}
public static final InvalidElement INVALID_ELEMENT = new InvalidElement();
public interface action_fptr { void exec(int action); }
public interface sempred_fptr { boolean eval(int predIndex); }
public byte[] code;
//public Map<String, Integer> ruleToAddr;
public int[] altToAddr; // either token type (in lexer) or alt num for DFA in parser
public CommonToken[] labelValues;
public int nLabels;
public int[][] charToAddr;
/** If we hit an action, we'll have to rewind and do the winning rule again */
boolean bypassedAction;
boolean notNextMatch;
List<ThreadState> s0_closure;
List<ThreadState>[] closure_cache;
public PDA(byte[] code, int[] altToAddr, int nLabels) {
//System.out.println("code="+Arrays.toString(code));
this.code = code;
this.altToAddr = altToAddr;
this.nLabels = nLabels;
labelValues = new CommonToken[nLabels];
closure_cache = new ArrayList[255+1];
}
public int execThompson(IntStream input) {
int m = input.mark();
Arrays.fill(labelValues, null);
int ttype = execThompson(input, 0, false);
// System.out.println("first attempt ttype="+ttype);
if ( bypassedAction ) {
input.rewind(m);
//System.out.println("Bypassed action; rewinding to "+input.index()+" doing with feeling");
bypassedAction = false;
Arrays.fill(labelValues, null);
int ttype2 = execThompson(input, altToAddr[ttype], true);
if ( ttype!=ttype2 ) {
System.err.println("eh? token diff with action(s)");
}
//else System.out.println("types are same");
}
else input.release(m);
return ttype;
}
public int execThompson(IntStream input, int ip, boolean doActions) {
int c = input.LA(1);
if ( c==Token.EOF ) return Token.EOF;
// List<ThreadState> closure = null;
// int[] x = charToAddr[c];
// //System.out.println("list for "+Bytecode.quotedCharLiteral(c)+" is "+Arrays.toString(x));
// if ( closure_cache[c] != null ) {
// closure = new ArrayList<ThreadState>();
// closure.addAll(closure_cache[c]);
// }
// else {
// if ( x!=null ) {
// closure = new ArrayList<ThreadState>();
// int i = 1;
// for (int v : x) {
// //ThreadState t = new ThreadState(v, i, NFAStack.EMPTY);
// addToClosure(closure, v, i, NFAStack.EMPTY);
// //closure.add(t);
// i++;
// }
// closure_cache[c] = new ArrayList<ThreadState>();
// closure_cache[c].addAll(closure);
// //System.out.println("caching "+closure);
// }
// else {
// System.err.println("invalid char: "+Bytecode.quotedCharLiteral(c));
// }
// }
List<ThreadState> closure = null;
if ( s0_closure == null ) {
s0_closure = computeStartState(ip);
}
closure = new ArrayList<ThreadState>();
closure.addAll(s0_closure);
List<ThreadState> reach = new ArrayList<ThreadState>();
ThreadState prevAccept = new ThreadState(Integer.MAX_VALUE, -1, NFAStack.EMPTY);
ThreadState firstAccept = null;
int firstCharIndex = input.index(); // use when creating Token
do { // while more work
c = input.LA(1);
int i = 0;
boolean accepted = false;
// System.out.println("input["+input.index()+"]=="+Bytecode.quotedCharLiteral(c)+
// " closure="+closure);
processOneChar:
while ( i<closure.size() ) {
ThreadState t = closure.get(i);
ip = t.addr;
NFAStack context = t.context;
int alt = t.alt;
short opcode = code[ip];
boolean matched;
ip++; // move to next instruction or first byte of operand
switch (opcode) {
case Bytecode.NOT :
notNextMatch = true;
break;
case Bytecode.MATCH8 :
if ( (!notNextMatch && c == code[ip]) || (notNextMatch && c != code[ip] && c != Token.EOF) ) {
addToClosure(reach, ip+1, alt, context);
}
notNextMatch = false;
break;
case Bytecode.MATCH16 :
matched = c == getShort(code, ip);
if ( (!notNextMatch && matched) || (notNextMatch && matched && c != Token.EOF) ) {
addToClosure(reach, ip+2, alt, context);
}
notNextMatch = false;
break;
case Bytecode.RANGE8 :
matched = c >= code[ip] && c <= code[ip + 1];
if ( (!notNextMatch && matched) || (notNextMatch && matched && c != Token.EOF) ) {
addToClosure(reach, ip+2, alt, context);
}
notNextMatch = false;
break;
case Bytecode.RANGE16 :
matched = c >= getShort(code, ip) && c <= getShort(code, ip + 2);
if ( (!notNextMatch && matched) || (notNextMatch && matched && c != Token.EOF) ) {
addToClosure(reach, ip+4, alt, context);
}
notNextMatch = false;
break;
case Bytecode.WILDCARD :
if ( c!=Token.EOF ) {
addToClosure(reach, ip, alt, context);
}
break;
case Bytecode.LABEL : // lexers only
int labelIndex = getShort(code, ip);
labelValues[labelIndex] =
new CommonToken(((CharStream)input), 0, 0, input.index(), -1);
break;
case Bytecode.SAVE :
labelIndex = getShort(code, ip);
labelValues[labelIndex].setStopIndex(input.index()-1);
break;
case Bytecode.ACTION :
bypassedAction = true;
if ( doActions ) {
int ruleIndex = getShort(code, ip);
int actionIndex = getShort(code, ip+2);
action(ruleIndex, actionIndex);
}
break;
case Bytecode.ACCEPT :
if ( context != NFAStack.EMPTY ) break; // only do accept for outermost rule
accepted = true;
int tokenLastCharIndex = input.index() - 1;
int ttype = getShort(code, ip);
ANTLRStringStream is = (ANTLRStringStream)input;
// System.out.println("ACCEPT "+is.substring(firstCharIndex,tokenLastCharIndex)+" as type "+ttype);
if ( tokenLastCharIndex > prevAccept.inputIndex ) {
prevAccept.inputIndex = tokenLastCharIndex;
// choose longest match so far regardless of rule priority
// System.out.println("replacing old best match @ "+prevAccept.addr);
prevAccept.addr = ip-1;
prevAccept.inputMarker = input.mark();
if ( firstAccept==null ) firstAccept = prevAccept;
}
else if ( tokenLastCharIndex == prevAccept.inputIndex ) {
// choose first rule matched if match is of same length
if ( ip-1 < prevAccept.addr ) { // it will see both accepts for ambig rules
// System.out.println("replacing old best match @ "+prevAccept.addr);
prevAccept.addr = ip-1;
prevAccept.inputMarker = input.mark();
}
}
// if we reach accept state, toss out any addresses in rest
// of work list associated with accept's rule; that rule is done
int j=i+1;
while ( j<closure.size() ) {
ThreadState cl = closure.get(j);
//System.out.println("remaining "+ cl);
if ( cl.alt==alt ) closure.remove(j);
else j++;
}
// then, move to next char, looking for longer match
// (we continue processing if there are states in reach)
break;
// case Bytecode.JMP : // ignore
// case Bytecode.SPLIT :
// case Bytecode.CALL :
// case Bytecode.RET :
// case Bytecode.SEMPRED :
// break;
default :
throw new RuntimeException("invalid instruction @ "+ip+": "+opcode);
}
i++;
// trace(t,reach);
}
// if reach is empty, we didn't match anything but might have accepted
if ( reach.size()>0 ) { // if we reached other states, consume and process them
input.consume();
}
else if ( !accepted && c!=Token.EOF ) {
throw INVALID_ELEMENT;
}
// else reach.size==0 && matched, don't consume: accepted
// swap to avoid reallocating space
List<ThreadState> tmp = reach;
reach = closure;
closure = tmp;
reach.clear();
} while ( closure.size()>0 );
if ( prevAccept.addr >= code.length ) return Token.INVALID_TOKEN_TYPE;
int ttype = getShort(code, prevAccept.addr+1);
input.rewind(prevAccept.inputMarker); // does nothing if we accept'd at input.index() but might need to rewind
if ( firstAccept.inputMarker < prevAccept.inputMarker ) {
System.out.println("done at index "+input.index());
System.out.println("accept marker="+prevAccept.inputMarker);
input.release(firstAccept.inputMarker); // kill any other markers in stream we made
System.out.println("leaving with index "+input.index());
}
return ttype;
}
void addToClosure(List<ThreadState> closure, int ip, int alt, NFAStack context) {
ThreadState t = new ThreadState(ip, alt, context);
//System.out.println("add to closure "+ip+" "+closure);
if ( closure.contains(t) ) return;
short opcode = code[ip];
ip++; // move to next instruction or first byte of operand
switch (opcode) {
case Bytecode.NOT : // see thru NOT but include in closure so we exec during reach
closure.add(t); // add to closure; need to execute during reach
// add NOT and next instruction since reach only looks at
// what's in closure (it doesn't jump to ip after NOT)
addToClosure(closure, ip, alt, context);
break;
case Bytecode.JMP :
addToClosure(closure, getShort(code, ip), alt, context);
break;
case Bytecode.ACTION :
ip += 2; // has 2 more bytes than LABEL/SAVE
case Bytecode.LABEL :
case Bytecode.SAVE :
// see through them for closure ops
closure.add(t); // add to closure; need to execute during reach
ip += 2;
addToClosure(closure, ip, alt, context); // do closure past SAVE
break;
case Bytecode.SPLIT :
int nopnds = getShort(code, ip);
ip += 2;
// add split addresses to work queue in reverse order ('cept first one)
for (int i=0; i<nopnds; i++) {
addToClosure(closure, getShort(code, ip+i*2), alt, context);
}
break;
case Bytecode.CALL :
int target = getShort(code, ip);
int retaddr = ip+2;
addToClosure(closure, target, alt, new NFAStack(context, retaddr));
break;
case Bytecode.ACCEPT :
// accept is just a ret if we have a stack;
// i.e., don't stop; someone called us and we need to use their
// accept, not this one
closure.add(t); // add to closure; need to execute during reach
case Bytecode.RET :
if ( context != NFAStack.EMPTY ) {
addToClosure(closure, context.returnAddr, alt, context.parent);
}
break;
case Bytecode.SEMPRED :
// add next instruction only if sempred succeeds
int ruleIndex = getShort(code, ip);
int predIndex = getShort(code, ip+2);
System.out.println("eval sempred "+ ruleIndex+", "+predIndex);
if ( sempred(ruleIndex, predIndex) ) {
addToClosure(closure, ip+4, alt, context);
}
break;
default :
// optimization: only add edges of closure to closure list; reduces what we walk later
// we don't want to have to ignore CALL, RET, etc... later
closure.add(t);
break;
}
}
List<ThreadState> computeStartState(int ip) {
// if we're starting at a SPLIT, add closure of all SPLIT targets
// else just add closure of ip
List<ThreadState> closure = new ArrayList<ThreadState>();
if ( code[ip]!=Bytecode.SPLIT ) {
addToClosure(closure, ip, 1, NFAStack.EMPTY);
return closure;
}
ip++;
int nalts = getShort(code, ip);
ip += 2;
// add split addresses to work queue in reverse order ('cept first one)
for (int i=1; i<=nalts; i++) {
addToClosure(closure, getShort(code, ip), i, NFAStack.EMPTY);
ip += Bytecode.ADDR_SIZE;
}
return closure;
}
// ---------------------------------------------------------------------
// this stuff below can't do SAVE nor CALL/RET but faster. (nor preds)
/*
public int execThompson_no_stack(CharStream input, int ip) {
int c = input.LA(1);
if ( c==Token.EOF ) return Token.EOF;
List<Integer> closure = new ArrayList<Integer>();
List<Integer> reach = new ArrayList<Integer>();
int prevAcceptAddr = Integer.MAX_VALUE;
int prevAcceptLastCharIndex = -1;
int prevAcceptInputMarker = -1;
int firstAcceptInputMarker = -1;
addToClosure_no_stack(closure, ip);
do { // while more work
c = input.LA(1);
int i = 0;
processOneChar:
while ( i<closure.size() ) {
//for (int i=0; i<closure.size(); i++) {
//System.out.println("input["+input.index()+"]=="+(char)c+" closure="+closure+", i="+i+", reach="+ reach);
ip = closure.get(i);
trace(ip);
short opcode = code[ip];
ip++; // move to next instruction or first byte of operand
switch (opcode) {
case Bytecode.MATCH8 :
if ( c == code[ip] ) {
addToClosure_no_stack(reach, ip+1);
}
break;
case Bytecode.MATCH16 :
if ( c == getShort(code, ip) ) {
addToClosure_no_stack(reach, ip+2);
}
break;
case Bytecode.RANGE8 :
if ( c>=code[ip] && c<=code[ip+1] ) {
addToClosure_no_stack(reach, ip+2);
}
break;
case Bytecode.RANGE16 :
if ( c<getShort(code, ip) || c>getShort(code, ip+2) ) {
addToClosure_no_stack(reach, ip+4);
}
break;
case Bytecode.WILDCARD :
if ( c!=Token.EOF ) addToClosure_no_stack(reach, ip);
break;
case Bytecode.ACCEPT :
int tokenLastCharIndex = input.index() - 1;
int ttype = getShort(code, ip);
System.out.println("ACCEPT "+ ttype +" with last char position "+ tokenLastCharIndex);
if ( tokenLastCharIndex > prevAcceptLastCharIndex ) {
prevAcceptLastCharIndex = tokenLastCharIndex;
// choose longest match so far regardless of rule priority
System.out.println("replacing old best match @ "+prevAcceptAddr);
prevAcceptAddr = ip-1;
prevAcceptInputMarker = input.mark();
firstAcceptInputMarker = prevAcceptInputMarker;
}
else if ( tokenLastCharIndex == prevAcceptLastCharIndex ) {
// choose first rule matched if match is of same length
if ( ip-1 < prevAcceptAddr ) { // it will see both accepts for ambig rules
System.out.println("replacing old best match @ "+prevAcceptAddr);
prevAcceptAddr = ip-1;
prevAcceptInputMarker = input.mark();
}
}
// if we reach accept state, toss out any addresses in rest
// of work list associated with accept's rule; that rule is done
int ruleStart = altToAddr[ttype];
int ruleStop = code.length;
if ( ttype+1 < altToAddr.length ) {
ruleStop = altToAddr[ttype+1]-1;
}
System.out.println("kill range "+ruleStart+".."+ruleStop);
int j=i+1;
while ( j<closure.size() ) {
Integer cl = closure.get(j);
System.out.println("remaining "+ cl);
if ( cl>=ruleStart || cl<=ruleStop ) closure.remove(j);
else j++;
}
// then, move to next char, looking for longer match
// (we continue processing if there are states in reach)
break;
//break processOneChar;
case Bytecode.JMP : // ignore
case Bytecode.SPLIT :
break;
default :
throw new RuntimeException("invalid instruction @ "+ip+": "+opcode);
}
i++;
}
if ( reach.size()>0 ) { // if we reached other states, consume and process them
input.consume();
}
// swap to avoid reallocating space
List<Integer> tmp = reach;
reach = closure;
closure = tmp;
reach.clear();
} while ( closure.size()>0 );
if ( prevAcceptAddr >= code.length ) return Token.INVALID_TOKEN_TYPE;
int ttype = getShort(code, prevAcceptAddr+1);
System.out.println("done at index "+input.index());
System.out.println("accept marker="+prevAcceptInputMarker);
input.rewind(prevAcceptInputMarker); // does nothing if we accept'd at input.index() but might need to rewind
input.release(firstAcceptInputMarker); // kill any other markers in stream we made
System.out.println("leaving with index "+input.index());
return ttype;
}
void addToClosure_no_stack(List<Integer> closure, int ip) {
//System.out.println("add to closure "+ip+" "+closure);
if ( closure.contains(ip) ) return; // TODO: VERY INEFFICIENT! use int[num-states] as set test
closure.add(ip);
short opcode = code[ip];
ip++; // move to next instruction or first byte of operand
switch (opcode) {
case Bytecode.JMP :
addToClosure_no_stack(closure, getShort(code, ip));
break;
case Bytecode.SAVE :
int labelIndex = getShort(code, ip);
ip += 2;
addToClosure_no_stack(closure, ip); // do closure pass SAVE
// TODO: impl
break;
case Bytecode.SPLIT :
int nopnds = getShort(code, ip);
ip += 2;
// add split addresses to work queue in reverse order ('cept first one)
for (int i=0; i<nopnds; i++) {
addToClosure_no_stack(closure, getShort(code, ip+i*2));
}
break;
}
}
*/
// subclass needs to override these if there are sempreds or actions in lexer rules
public boolean sempred(int ruleIndex, int actionIndex) {
return true;
}
public void action(int ruleIndex, int actionIndex) {
}
void trace(ThreadState t, List<ThreadState> reach) {
int ip = t.addr;
String instr = Bytecode.disassembleInstruction(code, ip, true);
System.out.println(instr+"\t\t reach="+reach);
}
void traceDFA(int ip) {
String instr = Bytecode.disassembleInstruction(code, ip, false);
System.out.println(instr);
}
public static int getShort(byte[] memory, int index) {
return (memory[index]&0xFF) <<(8*1) | (memory[index+1]&0xFF); // prevent sign extension with mask
}
public static class Context {
public int ip;
public int inputMarker;
public Context(int ip, int inputMarker) {
this.ip = ip;
this.inputMarker = inputMarker;
}
}
public int execNoRecursion(TokenStream input, int ip) {
System.out.println("execNoRecursion @"+ip);
List<Context> work = new ArrayList<Context>();
work.add(new Context(ip, input.mark()));
workLoop:
while ( work.size()>0 ) {
Context ctx = work.remove(work.size()-1); // treat like stack
ip = ctx.ip;
input.rewind(ctx.inputMarker);
while ( ip < code.length ) {
int c = input.LA(1);
traceDFA(ip);
short opcode = code[ip];
ip++; // move to next instruction or first byte of operand
switch (opcode) {
case Bytecode.MATCH8 :
if ( c != code[ip] ) continue workLoop;
ip++;
input.consume();
break;
case Bytecode.MATCH16 :
if ( c != getShort(code, ip) ) continue workLoop;
ip += 2;
input.consume();
break;
case Bytecode.RANGE8 :
if ( c<code[ip] || c>code[ip+1] ) continue workLoop;
ip += 2;
input.consume();
break;
case Bytecode.RANGE16 :
if ( c<getShort(code, ip) || c>getShort(code, ip+2) ) continue workLoop;
ip += 4;
input.consume();
break;
case Bytecode.ACCEPT :
int altIndex = getShort(code, ip);
ip += 2;
System.out.println("accept "+altIndex);
// returning gives first match not longest; i.e., like PEG
return altIndex;
case Bytecode.JMP :
int target = getShort(code, ip);
ip = target;
continue;
case Bytecode.SPLIT :
int nopnds = getShort(code, ip);
ip += 2;
// add split addresses to work queue in reverse order ('cept first one)
for (int i=nopnds-1; i>=1; i--) {
int addr = getShort(code, ip+i*2);
//System.out.println("try alt "+i+" at "+addr);
work.add(new Context(addr, input.mark()));
}
// try first alternative (w/o adding to work list)
int addr = getShort(code, ip);
ip = addr;
//System.out.println("try alt "+nopnds+" at "+addr);
continue;
default :
throw new RuntimeException("invalid instruction @ "+ip+": "+opcode);
}
}
}
return 0;
}
/*
public int exec(CharStream input, String ruleName) {
return exec(input, ruleToAddr.get(ruleName));
}
public int exec(CharStream input) { return exec(input, 0); }
public int exec(CharStream input, int ip) {
while ( ip < code.length ) {
int c = input.LA(1);
trace(ip);
short opcode = code[ip];
ip++; // move to next instruction or first byte of operand
switch (opcode) {
case Bytecode.MATCH8 :
if ( c != code[ip] ) return 0;
ip++;
input.consume();
break;
case Bytecode.MATCH16 :
if ( c != getShort(code, ip) ) return 0;
ip += 2;
input.consume();
break;
case Bytecode.RANGE8 :
if ( c<code[ip] || c>code[ip+1] ) return 0;
ip += 2;
input.consume();
break;
case Bytecode.RANGE16 :
if ( c<getShort(code, ip) || c>getShort(code, ip+2) ) return 0;
ip += 4;
input.consume();
break;
case Bytecode.ACCEPT :
int ruleIndex = getShort(code, ip);
ip += 2;
System.out.println("accept "+ruleIndex);
return ruleIndex;
case Bytecode.JMP :
int target = getShort(code, ip);
ip = target;
continue;
case Bytecode.SPLIT :
int nopnds = getShort(code, ip);
ip += 2;
for (int i=1; i<=nopnds-1; i++) {
int addr = getShort(code, ip);
ip += 2;
//System.out.println("try alt "+i+" at "+addr);
int m = input.mark();
int r = exec(input, addr);
if ( r>0 ) { input.release(m); return r; }
input.rewind(m);
}
// try final alternative (w/o recursion)
int addr = getShort(code, ip);
ip = addr;
//System.out.println("try alt "+nopnds+" at "+addr);
continue;
default :
throw new RuntimeException("invalid instruction @ "+ip+": "+opcode);
}
}
return 0;
}
*/
}

41
runtime/Java/src/org/antlr/v4/runtime/pda/ThreadState.java
View File

@ -1,41 +0,0 @@
package org.antlr.v4.runtime.pda;
/** NFA simulation thread state */
public class ThreadState {
public int addr;
public int alt; // or speculatively matched token type for lexers
public NFAStack context;
public int inputIndex = -1; // char (or token?) index from 0
public int inputMarker = -1; // accept states track input markers in case we need to rewind
public ThreadState(int addr, int alt, NFAStack context) {
this.addr = addr;
this.alt = alt;
this.context = context;
}
public ThreadState(ThreadState t) {
this.addr = t.addr;
this.alt = t.alt;
this.context = t.context;
this.inputIndex = t.inputIndex;
}
public boolean equals(Object o) {
if ( o==null ) return false;
if ( this==o ) return true;
ThreadState other = (ThreadState)o;
return this.addr==other.addr &&
this.alt==other.alt &&
this.context.equals(other.context);
}
public int hashCode() { return addr + context.hashCode(); }
public String toString() {
if ( context.parent==null ) {
return "("+addr+","+alt+")";
}
return "("+addr+","+alt+","+context+")";
}
}

223
runtime/Java/src/org/antlr/v4/runtime/tree/CommonTree.java
View File

@ -1,223 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.tree;
import org.antlr.runtime.BitSet;
import org.antlr.runtime.Token;
import org.antlr.runtime.tree.BaseTree;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.runtime.tree.gui.ASTViewer;
/** A tree node that is wrapper for a Token object. After 3.0 release
* while building tree rewrite stuff, it became clear that computing
* parent and child index is very difficult and cumbersome. Better to
* spend the space in every tree node. If you don't want these extra
* fields, it's easy to cut them out in your own BaseTree subclass.
*/
public class CommonTree extends BaseTree {
/** A single token is the payload */
public Token token;
/** What token indexes bracket all tokens associated with this node
* and below?
*/
protected int startIndex=-1, stopIndex=-1;
/** Who is the parent node of this node; if null, implies node is root */
public CommonTree parent;
/** What index is this node in the child list? Range: 0..n-1 */
public int childIndex = -1;
public CommonTree() { }
public CommonTree(CommonTree node) {
super(node);
this.token = node.token;
this.startIndex = node.startIndex;
this.stopIndex = node.stopIndex;
}
public CommonTree(Token t) {
this.token = t;
}
public Token getToken() {
return token;
}
public Tree dupNode() {
return new CommonTree(this);
}
public boolean isNil() {
return token==null;
}
public int getType() {
if ( token==null ) {
return Token.INVALID_TOKEN_TYPE;
}
return token.getType();
}
public String getText() {
if ( token==null ) {
return null;
}
return token.getText();
}
public int getLine() {
if ( token==null || token.getLine()==0 ) {
if ( getChildCount()>0 ) {
return getChild(0).getLine();
}
return 0;
}
return token.getLine();
}
public int getCharPositionInLine() {
if ( token==null || token.getCharPositionInLine()==-1 ) {
if ( getChildCount()>0 ) {
return getChild(0).getCharPositionInLine();
}
return 0;
}
return token.getCharPositionInLine();
}
public int getTokenStartIndex() {
if ( startIndex==-1 && token!=null ) {
return token.getTokenIndex();
}
return startIndex;
}
public void setTokenStartIndex(int index) {
startIndex = index;
}
public int getTokenStopIndex() {
if ( stopIndex==-1 && token!=null ) {
return token.getTokenIndex();
}
return stopIndex;
}
public void setTokenStopIndex(int index) {
stopIndex = index;
}
/** For every node in this subtree, make sure it's start/stop token's
* are set. Walk depth first, visit bottom up. Only updates nodes
* with at least one token index < 0.
*/
public void setUnknownTokenBoundaries() {
if ( children==null ) {
if ( startIndex<0 || stopIndex<0 ) {
startIndex = stopIndex = token.getTokenIndex();
}
return;
}
for (int i=0; i<children.size(); i++) {
((CommonTree)children.get(i)).setUnknownTokenBoundaries();
}
if ( startIndex>=0 && stopIndex>=0 ) return; // already set
if ( children.size() > 0 ) {
CommonTree firstChild = (CommonTree)children.get(0);
CommonTree lastChild = (CommonTree)children.get(children.size()-1);
startIndex = firstChild.getTokenStartIndex();
stopIndex = lastChild.getTokenStopIndex();
}
}
public int getChildIndex() {
return childIndex;
}
public Tree getParent() {
return parent;
}
public void setParent(Tree t) {
this.parent = (CommonTree)t;
}
public void setChildIndex(int index) {
this.childIndex = index;
}
// TODO: move to basetree when i settle on how runtime works
public void inspect() {
ASTViewer viewer = new ASTViewer(this);
viewer.open();
}
// TODO: move to basetree when i settle on how runtime works
// TODO: don't include this node!!
// TODO: reuse other method
public CommonTree getFirstDescendantWithType(int type) {
if ( getType()==type ) return this;
if ( children==null ) return null;
for (Object c : children) {
CommonTree t = (CommonTree)c;
if ( t.getType()==type ) return t;
CommonTree d = t.getFirstDescendantWithType(type);
if ( d!=null ) return d;
}
return null;
}
// TODO: don't include this node!!
public CommonTree getFirstDescendantWithType(BitSet types) {
if ( types.member(getType()) ) return this;
if ( children==null ) return null;
for (Object c : children) {
CommonTree t = (CommonTree)c;
if ( types.member(t.getType()) ) return t;
CommonTree d = t.getFirstDescendantWithType(types);
if ( d!=null ) return d;
}
return null;
}
public String toString() {
if ( isNil() ) {
return "nil";
}
if ( getType()==Token.INVALID_TOKEN_TYPE ) {
return "<errornode>";
}
if ( token==null ) {
return null;
}
return token.getText();
}
}

136
runtime/Java/src/org/antlr/v4/runtime/tree/TreeFilter.java
View File

@ -1,136 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.tree;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.RecognizerSharedState;
import org.antlr.runtime.TokenStream;
import org.antlr.runtime.tree.*;
/**
Cut-n-paste from material I'm not using in the book anymore (edit later
to make sense):
Now, how are we going to test these tree patterns against every
subtree in our original tree? In what order should we visit nodes?
For this application, it turns out we need a simple ``apply once''
rule application strategy and a ``down then up'' tree traversal
strategy. Let's look at rule application first.
As we visit each node, we need to see if any of our patterns match. If
a pattern matches, we execute the associated tree rewrite and move on
to the next node. In other words, we only look for a single rule
application opportunity (we'll see below that we sometimes need to
repeatedly apply rules). The following method applies a rule in a @cl
TreeParser (derived from a tree grammar) to a tree:
here is where weReferenced code/walking/patterns/TreePatternMatcher.java
It uses reflection to lookup the appropriate rule within the generated
tree parser class (@cl Simplify in this case). Most of the time, the
rule will not match the tree. To avoid issuing syntax errors and
attempting error recovery, it bumps up the backtracking level. Upon
failure, the invoked rule immediately returns. If you don't plan on
using this technique in your own ANTLR-based application, don't sweat
the details. This method boils down to ``call a rule to match a tree,
executing any embedded actions and rewrite rules.''
At this point, we know how to define tree grammar rules and how to
apply them to a particular subtree. The final piece of the tree
pattern matcher is the actual tree traversal. We have to get the
correct node visitation order. In particular, we need to perform the
scalar-vector multiply transformation on the way down (preorder) and
we need to reduce multiply-by-zero subtrees on the way up (postorder).
To implement a top-down visitor, we do a depth first walk of the tree,
executing an action in the preorder position. To get a bottom-up
visitor, we execute an action in the postorder position. ANTLR
provides a standard @cl TreeVisitor class with a depth first search @v
visit method. That method executes either a @m pre or @m post method
or both. In our case, we need to call @m applyOnce in both. On the way
down, we'll look for @r vmult patterns. On the way up,
we'll look for @r mult0 patterns.
*/
public class TreeFilter extends TreeParser {
public interface fptr {
public void rule() throws RecognitionException;
}
protected TokenStream originalTokenStream;
protected TreeAdaptor originalAdaptor;
public TreeFilter(TreeNodeStream input) {
this(input, new RecognizerSharedState());
}
public TreeFilter(TreeNodeStream input, RecognizerSharedState state) {
super(input, state);
originalAdaptor = (TreeAdaptor) input.getTreeAdaptor();
originalTokenStream = input.getTokenStream();
}
public void applyOnce(Object t, fptr whichRule) {
if ( t==null ) return;
try {
// share TreeParser object but not parsing-related state
state = new RecognizerSharedState();
input = new CommonTreeNodeStream(originalAdaptor, t);
((CommonTreeNodeStream)input).setTokenStream(originalTokenStream);
setBacktrackingLevel(1);
whichRule.rule();
setBacktrackingLevel(0);
}
catch (RecognitionException e) { ; }
}
public void downup(Object t) {
TreeVisitor v = new TreeVisitor(new CommonTreeAdaptor());
TreeVisitorAction actions = new TreeVisitorAction() {
public Object pre(Object t) { applyOnce(t, topdown_fptr); return t; }
public Object post(Object t) { applyOnce(t, bottomup_fptr); return t; }
};
v.visit(t, actions);
}
fptr topdown_fptr = new fptr() {
public void rule() throws RecognitionException {
topdown();
}
};
fptr bottomup_fptr = new fptr() {
public void rule() throws RecognitionException {
bottomup();
}
};
// methods the downup strategy uses to do the up and down rules.
// to override, just define tree grammar rule topdown and turn on
// filter=true.
public void topdown() throws RecognitionException {;}
public void bottomup() throws RecognitionException {;}
}

240
runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java
View File

@ -1,240 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.runtime.tree;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.TreeAdaptor;
import org.antlr.runtime.tree.TreeNodeStream;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
/** A parser for a stream of tree nodes. "tree grammars" result in a subclass
* of this. All the error reporting and recovery is shared with Parser via
* the BaseRecognizer superclass.
*/
public class TreeParser extends BaseRecognizer {
public static final int DOWN = Token.DOWN;
public static final int UP = Token.UP;
// precompiled regex used by inContext
static String dotdot = ".*[^.]\\.\\.[^.].*";
static String doubleEtc = ".*\\.\\.\\.\\s+\\.\\.\\..*";
static Pattern dotdotPattern = Pattern.compile(dotdot);
static Pattern doubleEtcPattern = Pattern.compile(doubleEtc);
protected TreeNodeStream input;
public TreeParser(TreeNodeStream input) {
super(); // highlight that we go to super to set state object
setTreeNodeStream(input);
}
public TreeParser(TreeNodeStream input, RecognizerSharedState state) {
super(state); // share the state object with another parser
setTreeNodeStream(input);
}
public void reset() {
super.reset(); // reset all recognizer state variables
if ( input!=null ) {
input.seek(0); // rewind the input
}
}
/** Set the input stream */
public void setTreeNodeStream(TreeNodeStream input) {
this.input = input;
}
public TreeNodeStream getTreeNodeStream() {
return input;
}
public String getSourceName() {
return input.getSourceName();
}
protected Object getCurrentInputSymbol(IntStream input) {
return ((TreeNodeStream)input).LT(1);
}
protected Object getMissingSymbol(IntStream input,
RecognitionException e,
int expectedTokenType,
BitSet follow)
{
String tokenText =
"<missing "+getTokenNames()[expectedTokenType]+">";
TreeAdaptor adaptor = ((TreeNodeStream)e.input).getTreeAdaptor();
return adaptor.create(new CommonToken(expectedTokenType, tokenText));
}
/** Match '.' in tree parser has special meaning. Skip node or
* entire tree if node has children. If children, scan until
* corresponding UP node.
*/
public void matchAny(IntStream ignore) { // ignore stream, copy of input
state.errorRecovery = false;
state.failed = false;
Object look = input.LT(1);
if ( input.getTreeAdaptor().getChildCount(look)==0 ) {
input.consume(); // not subtree, consume 1 node and return
return;
}
// current node is a subtree, skip to corresponding UP.
// must count nesting level to get right UP
int level=0;
int tokenType = input.getTreeAdaptor().getType(look);
while ( tokenType!=Token.EOF && !(tokenType==UP && level==0) ) {
input.consume();
look = input.LT(1);
tokenType = input.getTreeAdaptor().getType(look);
if ( tokenType == DOWN ) {
level++;
}
else if ( tokenType == UP ) {
level--;
}
}
input.consume(); // consume UP
}
/** We have DOWN/UP nodes in the stream that have no line info; override.
* plus we want to alter the exception type. Don't try to recover
* from tree parser errors inline...
*/
protected Object recoverFromMismatchedToken(IntStream input,
int ttype,
BitSet follow)
throws RecognitionException
{
throw new MismatchedTreeNodeException(ttype, (TreeNodeStream)input);
}
/** Prefix error message with the grammar name because message is
* always intended for the programmer because the parser built
* the input tree not the user.
*/
public String getErrorHeader(RecognitionException e) {
return getGrammarFileName()+": node from "+
(e.approximateLineInfo?"after ":"")+"line "+e.line+":"+e.charPositionInLine;
}
/** Tree parsers parse nodes they usually have a token object as
* payload. Set the exception token and do the default behavior.
*/
public String getErrorMessage(RecognitionException e, String[] tokenNames) {
if ( this instanceof TreeParser ) {
TreeAdaptor adaptor = ((TreeNodeStream)e.input).getTreeAdaptor();
e.token = adaptor.getToken(e.node);
if ( e.token==null ) { // could be an UP/DOWN node
e.token = new CommonToken(adaptor.getType(e.node),
adaptor.getText(e.node));
}
}
return super.getErrorMessage(e, tokenNames);
}
/** Check if current node in input has a context. Context means sequence
* of nodes towards root of tree. For example, you might say context
* is "MULT" which means my parent must be MULT. "CLASS VARDEF" says
* current node must be child of a VARDEF and whose parent is a CLASS node.
* You can use "..." to mean zero-or-more nodes. "METHOD ... VARDEF"
* means my parent is VARDEF and somewhere above that is a METHOD node.
* The first node in the context is not necessarily the root. The context
* matcher stops matching and returns true when it runs out of context.
* There is no way to force the first node to be the root.
*/
public boolean inContext(String context) {
return inContext(input.getTreeAdaptor(), getTokenNames(), input.LT(1), context);
}
/** The worker for inContext. It's static and full of parameters for
* testing purposes.
*/
public static boolean inContext(TreeAdaptor adaptor,
String[] tokenNames,
Object t,
String context)
{
Matcher dotdotMatcher = dotdotPattern.matcher(context);
Matcher doubleEtcMatcher = doubleEtcPattern.matcher(context);
if ( dotdotMatcher.find() ) { // don't allow "..", must be "..."
throw new IllegalArgumentException("invalid syntax: ..");
}
if ( doubleEtcMatcher.find() ) { // don't allow double "..."
throw new IllegalArgumentException("invalid syntax: ... ...");
}
context = context.replaceAll("\\.\\.\\.", " ... "); // ensure spaces around ...
context = context.trim();
String[] nodes = context.split("\\s+");
int ni = nodes.length-1;
t = adaptor.getParent(t);
while ( ni>=0 && t!=null ) {
if ( nodes[ni].equals("...") ) {
// walk upwards until we see nodes[ni-1] then continue walking
if ( ni==0 ) return true; // ... at start is no-op
String goal = nodes[ni-1];
Object ancestor = getAncestor(adaptor, tokenNames, t, goal);
if ( ancestor==null ) return false;
t = ancestor;
ni--;
}
String name = tokenNames[adaptor.getType(t)];
if ( !name.equals(nodes[ni]) ) {
//System.err.println("not matched: "+nodes[ni]+" at "+t);
return false;
}
// advance to parent and to previous element in context node list
ni--;
t = adaptor.getParent(t);
}
if ( t==null && ni>=0 ) return false; // at root but more nodes to match
return true;
}
/** Helper for static inContext */
protected static Object getAncestor(TreeAdaptor adaptor, String[] tokenNames, Object t, String goal) {
while ( t!=null ) {
String name = tokenNames[adaptor.getType(t)];
if ( name.equals(goal) ) return t;
t = adaptor.getParent(t);
}
return null;
}
public void traceIn(String ruleName, int ruleIndex) {
super.traceIn(ruleName, ruleIndex, input.LT(1));
}
public void traceOut(String ruleName, int ruleIndex) {
super.traceOut(ruleName, ruleIndex, input.LT(1));
}
}

3
runtime/Java/src/org/antlr/v4/runtime/tree/gui/ANTLRASTViewer.jfdproj
View File

@ -1,3 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project />

43
runtime/Java/src/org/antlr/v4/runtime/tree/gui/ASTViewFrame.java
View File

@ -1,43 +0,0 @@
package org.antlr.v4.runtime.tree.gui;
import javax.swing.*;
import java.awt.*;
/*
* Created by JFormDesigner on Mon Jan 18 14:54:16 PST 2010
*/
/**
* @author Terence Parr
*/
public class ASTViewFrame extends JFrame {
public ASTViewFrame() {
initComponents();
}
private void initComponents() {
// JFormDesigner - Component initialization - DO NOT MODIFY //GEN-BEGIN:initComponents
// Generated using JFormDesigner non-commercial license
scrollPane1 = new JScrollPane();
tree = new JTree();
//======== this ========
setTitle("ANTLR AST Viewer");
Container contentPane = getContentPane();
contentPane.setLayout(new GridLayout(1, 1));
//======== scrollPane1 ========
{
scrollPane1.setViewportView(tree);
}
contentPane.add(scrollPane1);
pack();
setLocationRelativeTo(getOwner());
// JFormDesigner - End of component initialization //GEN-END:initComponents
}
// JFormDesigner - Variables declaration - DO NOT MODIFY //GEN-BEGIN:variables
// Generated using JFormDesigner non-commercial license
private JScrollPane scrollPane1;
public JTree tree;
// JFormDesigner - End of variables declaration //GEN-END:variables
}

73
runtime/Java/src/org/antlr/v4/runtime/tree/gui/ASTViewFrame.jfd
View File

@ -1,73 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<java version="1.6.0_15" class="java.beans.XMLDecoder">
<object class="com.jformdesigner.model.FormModel">
<void property="contentType">
<string>form/swing</string>
</void>
<void property="root">
<object class="com.jformdesigner.model.FormRoot">
<void method="add">
<object class="com.jformdesigner.model.FormWindow">
<string>javax.swing.JFrame</string>
<object class="com.jformdesigner.model.FormLayoutManager">
<class>java.awt.GridLayout</class>
<void method="setProperty">
<string>columns</string>
<int>1</int>
</void>
</object>
<void method="setProperty">
<string>title</string>
<string>ANTLR AST Viewer</string>
</void>
<void method="add">
<object class="com.jformdesigner.model.FormContainer">
<string>javax.swing.JScrollPane</string>
<object class="com.jformdesigner.model.FormLayoutManager">
<class>javax.swing.JScrollPane</class>
</object>
<void property="name">
<string>scrollPane1</string>
</void>
<void method="add">
<object class="com.jformdesigner.model.FormComponent">
<string>javax.swing.JTree</string>
<void property="name">
<string>tree</string>
</void>
<void method="auxiliary">
<void method="setProperty">
<string>JavaCodeGenerator.variableModifiers</string>
<int>1</int>
</void>
</void>
</object>
</void>
</object>
</void>
<void property="name">
<string>this</string>
</void>
</object>
<object class="com.jformdesigner.model.FormLayoutConstraints">
<null/>
<void method="setProperty">
<string>location</string>
<object class="java.awt.Point">
<int>0</int>
<int>0</int>
</object>
</void>
<void method="setProperty">
<string>size</string>
<object class="java.awt.Dimension">
<int>400</int>
<int>300</int>
</object>
</void>
</object>
</void>
</object>
</void>
</object>
</java>

28
runtime/Java/src/org/antlr/v4/runtime/tree/gui/ASTViewer.java
View File

@ -1,28 +0,0 @@
package org.antlr.v4.runtime.tree.gui;
import org.antlr.runtime.tree.CommonTreeAdaptor;
import org.antlr.runtime.tree.TreeAdaptor;
/** */
public class ASTViewer {
TreeAdaptor adaptor;
Object root;
public ASTViewer(TreeAdaptor adaptor, Object root) {
this.adaptor = adaptor;
this.root = root;
}
public ASTViewer(Object root) {
this.adaptor = new CommonTreeAdaptor();
this.root = root;
}
public void open() {
ASTViewFrame m = new ASTViewFrame();
m.tree.setModel(new JTreeASTModel(adaptor, root));
m.pack();
m.setSize(800,600);
m.setVisible(true);
}
}

51
runtime/Java/src/org/antlr/v4/runtime/tree/gui/JTreeASTModel.java
View File

@ -1,51 +0,0 @@
package org.antlr.v4.runtime.tree.gui;
import org.antlr.runtime.tree.CommonTreeAdaptor;
import org.antlr.runtime.tree.TreeAdaptor;
import javax.swing.event.TreeModelListener;
import javax.swing.tree.TreeModel;
import javax.swing.tree.TreePath;
public class JTreeASTModel implements TreeModel {
TreeAdaptor adaptor;
Object root;
public JTreeASTModel(TreeAdaptor adaptor, Object root) {
this.adaptor = adaptor;
this.root = root;
}
public JTreeASTModel(Object root) {
this.adaptor = new CommonTreeAdaptor();
this.root = root;
}
public int getChildCount(Object parent) {
return adaptor.getChildCount(parent);
}
public int getIndexOfChild(Object parent, Object child){
if ( parent==null ) return -1;
return adaptor.getChildIndex(child);
}
public Object getChild(Object parent, int index){
return adaptor.getChild(parent, index);
}
public boolean isLeaf(Object node) {
return getChildCount(node)==0;
}
public Object getRoot() { return root; }
public void valueForPathChanged(TreePath treePath, Object o) {
}
public void addTreeModelListener(TreeModelListener treeModelListener) {
}
public void removeTreeModelListener(TreeModelListener treeModelListener) {
}
}

0
tool/build.properties
View File

10
tool/build.xml
View File

@ -1,10 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project name="tool" default="compile">
<property file="build.properties"/>
<target name="compile">
<javac srcdir="src" destdir="../build/classes"/>
</target>
</project>

456
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -1,456 +0,0 @@
javaTypeInitMap ::= [
"int":"0",
"long":"0",
"float":"0.0f",
"double":"0.0",
"boolean":"false",
"byte":"0",
"short":"0",
"char":"0",
default:"null" // anything other than an atomic type
]
// args must be <object-model-object>, <fields-resulting-in-STs>
ParserFile(file, parser, dfaDecls, bitSetDecls, namedActions) ::= <<
// $ANTLR ANTLRVersion> <file.fileName> generatedTimestamp>
<namedActions.header>
import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.EarlyExitException;
import org.antlr.v4.runtime.ParserSharedState;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.misc.*;
import org.antlr.runtime.*;
<parser>
>>
Parser(parser, scopes, funcs) ::= <<
public class <parser.name> extends Parser {
<!
public enum TokenType {
EOF(-1),
<parser.tokens.keys:{k | <k>(<parser.tokens.(k)>)}; separator=", ">
;
public int type;
TokenType(int type) { this.type = type; }
}
!>
<parser.tokens.keys:{k | public static final int <k>=<parser.tokens.(k)>;}; separator="\n">
<scopes>
<namedActions.members>
<parser:ctor()>
<funcs; separator="\n">
<dfaDecls; separator="\n">
<bitSetDecls; separator="\n">
}
>>
ctor(p) ::= <<
public <p.name>(TokenStream input) {
this(input, new ParserSharedState());
}
public <p.name>(TokenStream input, ParserSharedState state) {
super(input, state);
}
>>
/*
// S.g:5:1: b returns [String q, float x] : A ;
public final S.b_return b() throws RecognitionException {
b_stack.push(new b_scope());
S.b_return retval = new S.b_return();
*/
RuleFunction(f,code,decls,context,scope,namedActions,finallyAction) ::= <<
<context>
<scope>
<if(f.modifiers)><f.modifiers:{f | <f> }><else>public final <endif><f.ctxType> <f.name>(<f.ctxType> _ctx) throws RecognitionException {
state.ctx.push(_ctx);
<if(f.scope)>
<f.scope.name>_stack.push(new <f.scope.name>());
<endif>
<f.globalScopesUsed:{s | <s>_stack.push(new <s>());}; separator="\n">
<namedActions.init>
<decls; separator="\n">
try {
<code>
}
catch (RecognitionException re) {
reportError(re);
recover();
}
finally {
<namedActions.after>
<f.globalScopesUsed:{s | <s>_stack.pop();}; separator="\n">
<if(f.scope)><f.scope.name>_stack.pop();<endif>
<finallyAction>
<if(f.ctxType)>return (<f.ctxType>)state.ctx.pop();<endif>
}
}
>>
/** Convenience method to call from outside */
StartRuleFunction(f) ::= <<
<if(f.modifiers)><f.modifiers:{f | <f> }><else>public final <endif><f.ctxType> <f.name>(<f.args; separator=", ">) throws RecognitionException {
return <f.name>(new <f.ctxType>(<f.args:{a | <a.name>, }>LABitSet.EOF_SET));
}
>>
CodeBlock(c, ops) ::= <<
<ops; separator="\n">
>>
LL1AltBlock(choice, alts, error) ::= <<
switch ( state.input.LA(1) ) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<alt>
break;}; separator="\n">
default :
<error>
}
>>
// follow set included as choice by analysis
LL1OptionalBlock ::= LL1AltBlock
LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= <<
<preamble; separator="\n">
if ( <expr> ) {
<alts; separator="\n">
}
else if ( !(<followExpr>) ) <error>
>>
LL1StarBlock(choice, alts, sync) ::= <<
<choice.loopLabel>:
while (true) {
switch ( state.input.LA(1) ) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<alt>
break;}; separator="\n">
<cases(ttypes=choice.exitLook)>
break <choice.loopLabel>;
}
<sync>
}
>>
LL1StarBlockSingleAlt(choice, expr, alts, preamble, iteration, sync) ::= <<
<preamble; separator="\n">
while ( <expr> ) {
<alts; separator="\n">
<iteration>
<sync>
}
>>
LL1PlusBlock(choice, alts, earlyExitError, sync, iterationSync) ::= <<
<sync>
int <choice.loopCounterVar> = 0;
<choice.loopLabel>:
while (true) {
switch ( state.input.LA(1) ) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<alt>
break;}; separator="\n">
<cases(ttypes=choice.exitLook)>
if ( <choice.loopCounterVar> >= 1 ) break <choice.loopLabel>;
else <earlyExitError>
}
<choice.loopCounterVar>++;
<iterationSync>
}
>>
LL1PlusBlockSingleAlt(choice, expr, alts, preamble, iteration,
earlyExitError, sync, iterationSync) ::=
<<
<sync>
<preamble; separator="\n">
do {
<alts; separator="\n">
<iteration>
<iterationSync>
} while ( <expr> );
>>
Sync(s) ::= "sync(<s.expecting.name>);"
ThrowNoViableAlt(t) ::= "throw new NoViableAltException(this, <t.expecting.name>);"
ThrowEarlyExitException(t) ::= "throw new EarlyExitException(this, <t.expecting.name>);"
TestSet(s) ::= <<
<s.set.name>.member(state.input.LA(1))
>>
TestSetInline(s) ::= <<
<s.ttypes:{ttype | <s.varName>==<ttype>}; separator=" || ">
>>
cases(ttypes) ::= <<
<ttypes:{t | case <t>:}; separator="\n">
>>
InvokeRule(r) ::= <<
<if(r.labels)><r.labels:{l | <l> = }><endif><r.name>(new <r.ctxName>(<r.argExprs:{e|<e>,}><r.follow.name>));
>>
MatchToken(m) ::= <<
<if(m.labels)><m.labels:{l | <l> = }>(Token)<endif>match(<m.name>, <m.follow.name>);
>>
// ACTION STUFF
Action(a, chunks) ::= "<chunks>"
SemPred(p) ::= <<
if (!(<p.ast.text>)) throw new FailedPredicateException(this,"<ruleName>", "<description>");
>>
ActionText(t) ::= "<t.text>"
ArgRef(a) ::= "_ctx.<a.name>"
RetValueRef(a) ::= "_ctx.<a.name>"
QRetValueRef(a) ::= "<a.dict>.<a.name>"
/** How to translate $tokenLabel */
TokenRef(t) ::= "<t.name>"
SetAttr(s,rhsChunks) ::= "_ctx.<s.name> = <rhsChunks>;"
SetQAttr(s,rhsChunks) ::= "<s.dict>.<s.name> = <rhsChunks>;"
TokenPropertyRef_text(t) ::= "(<t.label>!=null?<t.label>.getText():null)"
TokenPropertyRef_type(t) ::= "(<t.label>!=null?<t.label>.getType():0)"
TokenPropertyRef_line(t) ::= "(<t.label>!=null?<t.label>.getLine():0)"
TokenPropertyRef_pos(t) ::= "(<t.label>!=null?<t.label>.getCharPositionInLine():0)"
TokenPropertyRef_channel(t) ::= "(<t.label>!=null?<t.label>.getChannel():0)"
TokenPropertyRef_index(t) ::= "(<t.label>!=null?<t.label>.getTokenIndex():0)"
TokenPropertyRef_tree(t) ::= "<t.label>_tree"
TokenPropertyRef_int(t) ::= "(<t.label>!=null?Integer.valueOf(<t.label>.getText()):0)"
RulePropertyRef_start(r) ::= "(<r.label>!=null?((<file.TokenLabelType>)<r.label>.start):null)"
RulePropertyRef_stop(r) ::= "(<r.label>!=null?((<file.TokenLabelType>)<r.label>.stop):null)"
RulePropertyRef_tree(r) ::= "(<r.label>!=null?((<file.ASTLabelType>)<r.label>.tree):null)"
RulePropertyRef_text(r) ::= "(<r.label>!=null?((TokenStream)state.input).toString(<r.label>.start,<r.label>.stop):null)"
RulePropertyRef_st(r) ::= "(<r.label>!=null?<r.label>.st:null)"
DynScopeRef(s) ::= "<s.scope>_stack"
DynScopeAttrRef(s) ::= "<s.scope>_stack.peek().<s.attr>"
DynScopeAttrRef_negIndex(s, indexChunks) ::=
"<s.scope>_stack.get(<s.scope>_stack.size()-<indexChunks>-1).<s.attr>"
DynScopeAttrRef_index(s, indexChunks) ::=
"<s.scope>_stack.get(<indexChunks>).<s.attr>"
SetDynScopeAttr(s, rhsChunks) ::=
"<s.scope>_stack.peek().<s.attr> =<rhsChunks>;"
SetDynScopeAttr_negIndex(s, indexChunks, rhsChunks) ::=
"<s.scope>_stack.get(<s.scope>_stack.size()-<indexChunks>-1).<s.attr> =<rhsChunks>;"
SetDynScopeAttr_index(s, indexChunks, rhsChunks) ::=
"<s.scope>_stack.get(<indexChunks>).<s.attr> =<rhsChunks>;"
AddToList(a) ::= "<a.listName>.add(<first(a.opWithResultToAdd.labels)>);"
TokenDecl(t) ::= "Token <t.name>;"
TokenTypeDecl(t) ::= "int <t.name>;"
TokenListDecl(t) ::= "List\<Token> <t.name> = new ArrayList\<Token>();"
RuleContextDecl(r) ::= "<r.ctxName> <r.name>;"
CaptureNextToken(d) ::= "<d.varName> = state.input.LT(1);"
CaptureNextTokenType(d) ::= "<d.varName> = state.input.LA(1);"
StructDecl(s,attrs) ::= <<
public static class <s.name> extends ParserRuleContext {
<attrs:{a | <a>;}; separator="\n">
<if(s.ctorAttrs)>
public <s.name>(<s.ctorAttrs:{a | <a>,}> LABitSet follow) {
super(follow);
<s.ctorAttrs:{a | this.<a.name> = <a.name>;}; separator="\n">
}
<endif>
};
>>
DynamicScopeStruct(d,attrs) ::= <<
public static class <d.name> {
<attrs:{a | <a>;}; separator="\n">
};
public QStack\<<d.name>\> <d.name>_stack = new QStack\<<d.name>\>();
>>
AttributeDecl(d) ::= "<d.decl>"
DFADecl(dfa) ::= <<
// define <dfa.name>
>>
BitSetDecl(b) ::= <<
public static final LABitSet <b.name>=new LABitSet(new long[]{<b.hexWords:{it|<it>L};separator=",">}<if(b.fset.EOF)>, true<endif>);
>>
LexerFile(fileName, lexer) ::= <<
// $ANTLR ANTLRVersion> <fileName> generatedTimestamp>
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.LexerSharedState;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.pda.*;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.*;
import org.antlr.runtime.*;
<lexer>
>>
Lexer(lexerName, modes, dfas, pdas, tokens, actions, sempreds, namedActions) ::= <<
public class <lexerName> extends Lexer {
<tokens.keys:{k | public static final int <k>=<tokens.(k)>;}; separator="\n">
<modes:{m| public static final int <m> = <i0>;}; separator="\n">
public <lexerName>(CharStream input) {
this(input, new LexerSharedState());
}
public <lexerName>(CharStream input, LexerSharedState state) {
super(input,state);
// modeToPDA = new PDA[] { <modes:{m | new <m>_PDA()}; separator=", "> };
modeToDFA = new DFA[] { <modes:{m | new <m>_DFA()}; separator=", "> };
}
public String getGrammarFileName() { return "<fileName>"; }
<namedActions.members>
<dfas>
<actions>
<sempreds>
<pdas>
}
>>
DFA(name, model) ::= <<
public static final short[] <name>_accept = {
<model.accept; separator=", ">
};
public static final short[] <name>_eof = {
<model.eof; separator=", ">
};
public static final char[] <name>_max = {
<model.max; separator=", ">
};
public static final short[][] <name>_transition = {
<model.transition:{t | {<t; separator=", ">\}}; separator=",\n", null="null">
};
public static final int[][] <name>_set_edges = {
<model.set_edges:{edges | {<edges; separator=", ">\}}; separator=",\n", null="null">
};
public static final int[][] <name>_pred_edges = {
<model.pred_edges:{edges | {<edges; separator=", ">\}}; separator=",\n", null="null">
};
public static final short[] <name>_action_index = {
<model.action_index; separator=", ">
};
public final class <name>_DFA extends DFA {
<if(model.actions)> <! TODO: FACTOR OUT !>
public void action(int action) {
switch ( action ) {
<model.actions:{a |
case <i0> :
<a>
break;
}>
}
}
<endif>
<if(model.sempreds)>
public boolean sempred(int sempred) {
switch ( sempred ) {
<model.sempreds:{p |
case <i0> :
return <p>;
break;
}>
}
return false;
}
<endif>
public <name>_DFA() {
this.eof = <name>_eof;
this.max = <name>_max;
this.accept = <name>_accept;
this.transition = <name>_transition;
this.set_edges = <name>_set_edges;
this.pred_edges = <name>_pred_edges;
this.action_index = <name>_action_index;
}
}
>>
PDA(name, model, actions, sempreds) ::= <<
public static final byte[] <name>_code = {
<model.code; separator=", ">
};
public static final int[] <name>_tokenTypeToAddr = {
<model.altToAddr; separator=", ">
};
public final class <name>_PDA extends PDA {
<if(actions)>
public void action(int r, int a) {
switch ( r ) {
<actions:{a |
case <a.ruleIndex> : <a.name>_actions(a); break;
}>
}
}
<endif>
<if(sempreds)>
public void sempred(int r, int a) {
switch ( r ) {
<sempreds:{p |
case <p.ruleIndex> : return <p.name>_sempreds(a);
}>
}
}
<endif>
public <name>_PDA() {
super(<name>_code, <name>_tokenTypeToAddr, <model.nLabels>);
}
}<\n>
>>
actionMethod(name, actions) ::= <<
public void <name>_actions(int action) {
System.out.println("exec action "+action);
switch ( action ) {
<actions:{a |
case <i0> :
<a>
break;
}>
}
}
>>
sempredMethod(name, preds) ::= <<
public boolean <name>_sempreds(int pred) {
switch ( pred ) {
<preds:{p |
case <i0> :
return <p>;
}>
default : return false;
}
}
>>
/** Using a type to init value map, try to init a type; if not in table
* must be an object, default value is "null".
*/
initValue(typeName) ::= <<
<javaTypeInitMap.(typeName)>
>>
codeFileExtension() ::= ".java"
true() ::= "true"
false() ::= "false"

12
tool/resources/org/antlr/v4/tool/templates/depend.stg
View File

@ -1,12 +0,0 @@
/** templates used to generate make-compatible dependencies */
group depend;
/** Generate "f : x, y, z" dependencies for input
* dependencies and generated files. in and out
* are File objects. For example, you can say
* <f.canonicalPath>
*/
dependencies(grammarFileName,in,out) ::= <<
<if(in)><grammarFileName>: <in; separator=", "><endif>
<out:{f | <f> : <grammarFileName>}; separator="\n">
>>

3
tool/resources/org/antlr/v4/tool/templates/dot/action-edge.st
View File

@ -1,3 +0,0 @@
action-edge() ::= <<
<src> -> <target> [fontsize=11, fontname="Courier", arrowsize=.7, label = "<label>"<if(arrowhead)>, arrowhead = <arrowhead><endif>];
>>

4
tool/resources/org/antlr/v4/tool/templates/dot/decision-rank.st
View File

@ -1,4 +0,0 @@
decision-rank() ::= <<
{rank=same; rankdir=TB; <states; separator="; ">}
>>

8
tool/resources/org/antlr/v4/tool/templates/dot/dfa.st
View File

@ -1,8 +0,0 @@
dfa(startState, states, edges, rankdir, decisionRanks, useBox) ::= <<
digraph NFA {
<if(rankdir)>rankdir=<rankdir>;<endif>
<decisionRanks; separator="\n">
<states; separator="\n">
<edges; separator="\n">
}
>>

4
tool/resources/org/antlr/v4/tool/templates/dot/edge.st
View File

@ -1,4 +0,0 @@
edge(src,target,label,arrowhead) ::= <<
<src> -> <target> [fontsize=11, fontname="Courier", arrowsize=.7, label = "<label>"<if(arrowhead)>, arrowhead = <arrowhead><endif>];
>>

3
tool/resources/org/antlr/v4/tool/templates/dot/epsilon-edge.st
View File

@ -1,3 +0,0 @@
epsilon-edge(src,label,target,arrowhead) ::= <<
<src> -> <target> [fontname="Times-Italic", label = "e"];
>>

8
tool/resources/org/antlr/v4/tool/templates/dot/nfa.st
View File

@ -1,8 +0,0 @@
nfa(startState, states, edges, rankdir, decisionRanks) ::= <<
digraph NFA {
rankdir=LR;
<decisionRanks; separator="\n">
<states; separator="\n">
<edges; separator="\n">
}
>>

3
tool/resources/org/antlr/v4/tool/templates/dot/state.st
View File

@ -1,3 +0,0 @@
state(state, label, name) ::= <<
node [fontsize=11, label="<label>", <if(useBox)>shape=box, fixedsize=false<else>shape=circle, fixedsize=true, width=.4<endif>, peripheries=1]; <name>
>>

3
tool/resources/org/antlr/v4/tool/templates/dot/stopstate.st
View File

@ -1,3 +0,0 @@
stopstate(name,label) ::= <<
node [fontsize=11, label="<label>", <if(useBox)>shape=polygon,sides=4,peripheries=2,fixedsize=false<else>shape=doublecircle, fixedsize=true, width=.6<endif>]; <name>
>>

40
tool/resources/org/antlr/v4/tool/templates/messages/formats/antlr.stg
View File

@ -1,40 +0,0 @@
/*
[The "BSD licence"]
Copyright (c) 2006 Kay Roepke
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
This file contains the actual layout of the messages emitted by ANTLR.
The text itself is coming out of the languages/*stg files, according to the chosen locale.
This file contains the default format ANTLR uses.
*/
location(file, line, column) ::= "<file>:<line>:<column>:"
message(id, text) ::= "(<id>) <text>"
report(location, message, type) ::= "<type>(<message.id>): <location> <message.text>"
wantsSingleLineMessage() ::= "false"

40
tool/resources/org/antlr/v4/tool/templates/messages/formats/gnu.stg
View File

@ -1,40 +0,0 @@
/*
[The "BSD licence"]
Copyright (c) 2006 Kay Roepke
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
This file contains the actual layout of the messages emitted by ANTLR.
The text itself is coming out of the languages/*stg files, according to the chosen locale.
This file contains the format that mimicks GCC output.
*/
location(file, line, column) ::= "<file>:<line>:"
message(id, text) ::= "<text> (<id>)"
report(location, message, type) ::= "<location> <type>: <message>"
wantsSingleLineMessage() ::= "true"

40
tool/resources/org/antlr/v4/tool/templates/messages/formats/vs2005.stg
View File

@ -1,40 +0,0 @@
/*
[The "BSD licence"]
Copyright (c) 2006 Kay Roepke
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
This file contains the actual layout of the messages emitted by ANTLR.
The text itself is coming out of the languages/*stg files, according to the chosen locale.
This file contains the default format ANTLR uses.
*/
location(file, line, column) ::= "<file>(<line>,<column>)"
message(id, text) ::= "error <id> : <text>"
report(location, message, type) ::= "<location> : <type> <message.id> : <message.text>"
wantsSingleLineMessage() ::= "true"

316
tool/resources/org/antlr/v4/tool/templates/messages/languages/en.stg
View File

@ -1,316 +0,0 @@
/*
[The "BSD licence"]
Copyright (c) 2005-2006 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
This file only contains the messages in English, but no
information about which file, line, or column it occurred in.
The location and message ids are taken out of the formats directory.
Kay Roepke
*/
INVALID() ::= <<this can never be printed>>
// TOOL ERRORS
// file errors
CANNOT_WRITE_FILE(arg,exception,stackTrace) ::= <<
cannot write file <arg>: <exception>
<stackTrace; separator="\n">
>>
CANNOT_CLOSE_FILE(arg,exception,stackTrace) ::= "cannot close file <arg>: <exception>"
CANNOT_FIND_TOKENS_FILE(arg) ::= "cannot find tokens file <arg>"
ERROR_READING_TOKENS_FILE(arg,exception,stackTrace) ::= <<
problem reading token vocabulary file <arg>: <exception>
<stackTrace; separator="\n">
>>
DIR_NOT_FOUND(arg,exception,stackTrace) ::= "directory not found: <arg>"
OUTPUT_DIR_IS_FILE(arg,arg2) ::= "output directory is a file: <arg>"
CANNOT_OPEN_FILE(arg,arg2) ::= "cannot find or open file: <arg><if(arg2)>; reason: <arg2><endif>"
CIRCULAR_DEPENDENCY() ::= "your grammars contain a circular dependency and cannot be sorted into a valid build order."
INTERNAL_ERROR(arg,arg2,exception,stackTrace) ::= <<
internal error: <arg> <arg2><if(exception)>: <exception><endif>
<stackTrace; separator="\n">
>>
INTERNAL_WARNING(arg) ::= "internal warning: <arg>"
ERROR_CREATING_ARTIFICIAL_RULE(arg,exception,stackTrace) ::= <<
problems creating lexer rule listing all tokens: <exception>
<stackTrace; separator="\n">
>>
TOKENS_FILE_SYNTAX_ERROR(arg,arg2) ::=
"problems parsing token vocabulary file <arg> on line <arg2>"
CANNOT_GEN_DOT_FILE(arg,exception,stackTrace) ::=
"cannot write DFA DOT file <arg>: <exception>"
BAD_ACTION_AST_STRUCTURE(exception,stackTrace) ::=
"bad internal tree structure for action '<arg>': <exception>"
BAD_AST_STRUCTURE(arg,msg) ::= <<
bad internal tree structure '<arg>': <msg>
>>
FILE_AND_GRAMMAR_NAME_DIFFER(arg,arg2) ::=
"file <arg2> contains grammar <arg>; names must be identical"
FILENAME_EXTENSION_ERROR(arg) ::=
"file <arg> must end in a file extension, normally .g"
// code gen errors
MISSING_CODE_GEN_TEMPLATES(arg) ::=
"cannot find code generation templates for language <arg>"
MISSING_CYCLIC_DFA_CODE_GEN_TEMPLATES() ::=
"cannot find code generation cyclic DFA templates for language <arg>"
CODE_GEN_TEMPLATES_INCOMPLETE(arg) ::=
"missing code generation template <arg>"
CANNOT_CREATE_TARGET_GENERATOR(arg,exception,stackTrace) ::=
"cannot create target <arg> code generator: <exception>"
CODE_TEMPLATE_ARG_ISSUE(arg,arg2) ::=
"code generation template <arg> has missing, misnamed, or incomplete arg list: <arg2>"
NO_MODEL_TO_TEMPLATE_MAPPING(arg) ::=
"no mapping to template name for output model class <arg>"
CANNOT_COMPUTE_SAMPLE_INPUT_SEQ() ::=
"cannot generate a sample input sequence from lookahead DFA"
// grammar interpretation errors
/*
NO_VIABLE_DFA_ALT(arg,arg2) ::=
"no viable transition from state <arg> on <arg2> while interpreting DFA"
*/
// GRAMMAR ERRORS
SYNTAX_ERROR(arg) ::= "<arg>"
RULE_REDEFINITION(arg) ::= "rule <arg> redefinition"
SCOPE_REDEFINITION(arg) ::= "scope <arg> redefinition"
LEXER_RULES_NOT_ALLOWED(arg) ::= "lexer rule <arg> not allowed in parser"
PARSER_RULES_NOT_ALLOWED(arg) ::= "parser rule <arg> not allowed in lexer"
MODE_NOT_IN_LEXER(arg,arg2) ::= "lexical modes are only allowed in lexer grammars"
TOKEN_NAMES_MUST_START_UPPER(arg) ::=
"token names must start with an uppercase letter: <arg>"
CANNOT_FIND_ATTRIBUTE_NAME_IN_DECL(arg) ::=
"cannot find an attribute name in attribute declaration"
NO_TOKEN_DEFINITION(arg) ::=
"no lexer rule corresponding to token: <arg>"
UNDEFINED_RULE_REF(arg) ::=
"reference to undefined rule: <arg>"
LITERAL_NOT_ASSOCIATED_WITH_LEXER_RULE(arg) ::=
"literal has no associated lexer rule: <arg>"
CANNOT_ALIAS_TOKENS(arg) ::=
"can't assign string value to token name <arg> in non-combined grammar"
ATTRIBUTE_REF_NOT_IN_RULE(arg,arg2) ::=
"reference to attribute outside of a rule: <arg><if(arg2)>.<arg2><endif>"
UNKNOWN_ATTRIBUTE_IN_SCOPE(arg,arg2) ::=
"attribute <arg> isn't a valid property in <arg2>"
UNKNOWN_RULE_ATTRIBUTE(arg,arg2,arg3) ::=
"unknown attribute <arg> for rule <arg2> in <arg3>"
UNKNOWN_SIMPLE_ATTRIBUTE(arg,arg2) ::=
"unknown attribute reference <arg> in <arg2>"
ISOLATED_RULE_REF(arg,arg2) ::=
"missing attribute access on rule reference <arg> in <arg2>"
INVALID_RULE_PARAMETER_REF(arg,arg2) ::=
"cannot access rule <arg>'s parameter: <arg2>"
INVALID_RULE_SCOPE_ATTRIBUTE_REF(arg,arg2) ::=
"cannot access rule <arg>'s dynamically-scoped attribute: <arg2>"
SYMBOL_CONFLICTS_WITH_GLOBAL_SCOPE(arg) ::=
"symbol <arg> conflicts with global dynamic scope with same name"
WRITE_TO_READONLY_ATTR(arg,arg2,arg3) ::=
"cannot write to read only attribute: $<arg><if(arg2)>.<arg2><endif>"
LABEL_CONFLICTS_WITH_RULE(arg) ::=
"label <arg> conflicts with rule with same name"
LABEL_CONFLICTS_WITH_TOKEN(arg) ::=
"label <arg> conflicts with token with same name"
LABEL_CONFLICTS_WITH_RULE_SCOPE_ATTRIBUTE(arg,arg2) ::=
"label <arg> conflicts with rule <arg2>'s dynamically-scoped attribute with same name"
LABEL_CONFLICTS_WITH_RULE_ARG_RETVAL(arg,arg2) ::=
"label <arg> conflicts with rule <arg2>'s return value or parameter with same name"
ATTRIBUTE_CONFLICTS_WITH_RULE(arg,arg2) ::=
"rule <arg2>'s dynamically-scoped attribute <arg> conflicts with the rule name"
ATTRIBUTE_CONFLICTS_WITH_RULE_ARG_RETVAL(arg,arg2) ::=
"rule <arg2>'s dynamically-scoped attribute <arg> conflicts with <arg2>'s return value or parameter with same name"
LABEL_TYPE_CONFLICT(arg,arg2) ::=
"label <arg> type mismatch with previous definition: <arg2>"
ARG_RETVAL_CONFLICT(arg,arg2) ::=
"rule <arg2>'s argument <arg> conflicts a return value with same name"
NONUNIQUE_REF(arg) ::=
"<arg> is a non-unique reference"
FORWARD_ELEMENT_REF(arg) ::=
"illegal forward reference: <arg>"
MISSING_RULE_ARGS(arg) ::=
"missing parameter(s) on rule reference: <arg>"
RULE_HAS_NO_ARGS(arg) ::=
"rule <arg> has no defined parameters"
ARGS_ON_TOKEN_REF(arg) ::=
"token reference <arg> may not have parameters"
/*
NONCHAR_RANGE() ::=
"range operator can only be used in the lexer"
*/
ILLEGAL_OPTION(arg) ::=
"illegal option <arg>"
LIST_LABEL_INVALID_UNLESS_RETVAL_STRUCT(arg) ::=
"rule '+=' list labels are not allowed w/o output option: <arg>"
REWRITE_ELEMENT_NOT_PRESENT_ON_LHS(arg) ::=
"reference to rewrite element <arg> not found to left of ->"
//UNDEFINED_TOKEN_REF_IN_REWRITE(arg) ::=
// "reference to undefined token in rewrite rule: <arg>"
//UNDEFINED_LABEL_REF_IN_REWRITE(arg) ::=
// "reference to undefined label in rewrite rule: $<arg>"
NO_GRAMMAR_START_RULE (arg) ::=
"grammar <arg>: no start rule (no rule can obviously be followed by EOF)"
EMPTY_COMPLEMENT(arg) ::= <<
<if(arg)>
set complement ~<arg> is empty
<else>
set complement is empty
<endif>
>>
REPEATED_PREQUEL(arg) ::=
"repeated grammar prequel spec (option, token, or import); please merge"
UNKNOWN_DYNAMIC_SCOPE(arg, arg2) ::=
"unknown dynamic scope: <arg> in <arg2>"
UNKNOWN_DYNAMIC_SCOPE_ATTRIBUTE(arg,arg2,arg3) ::=
"unknown dynamically-scoped attribute for scope <arg>: <arg2> in <arg3>"
RULE_REF_AMBIG_WITH_RULE_IN_ALT(arg) ::=
"reference $<arg> is ambiguous; rule <arg> is enclosing rule and referenced in the production (assuming enclosing rule)"
ISOLATED_RULE_ATTRIBUTE(arg) ::=
"reference to locally-defined rule scope attribute without rule name: <arg>"
INVALID_ACTION_SCOPE(arg,arg2) ::=
"unknown or invalid action scope for <arg2> grammar: <arg>"
ACTION_REDEFINITION(arg) ::=
"redefinition of <arg> action"
DOUBLE_QUOTES_ILLEGAL(arg) ::=
"string literals must use single quotes (such as \'begin\'): <arg>"
INVALID_TEMPLATE_ACTION(arg) ::=
"invalid StringTemplate % shorthand syntax: '<arg>'"
ARG_INIT_VALUES_ILLEGAL(arg) ::=
"rule parameters may not have init values: <arg>"
REWRITE_OR_OP_WITH_NO_OUTPUT_OPTION(arg) ::=
"<if(arg)>rule <arg> uses <endif>rewrite syntax or operator with no output option"
AST_OP_WITH_NON_AST_OUTPUT_OPTION(arg) ::=
"AST operator with non-AST output option: <arg>"
NO_RULES(arg,arg2) ::= "<if(arg2.implicitLexerOwner)>implicitly generated <endif>grammar <arg> has no rules"
MISSING_AST_TYPE_IN_TREE_GRAMMAR(arg) ::=
"tree grammar <arg> has no ASTLabelType option"
REWRITE_FOR_MULTI_ELEMENT_ALT(arg) ::=
"with rewrite=true, alt <arg> not simple node or obvious tree element; text attribute for rule not guaranteed to be correct"
RULE_INVALID_SET(arg) ::=
"Cannot complement rule <arg>; not a simple set or element"
HETERO_ILLEGAL_IN_REWRITE_ALT(arg) ::=
"alts with rewrites can't use heterogeneous types left of ->"
NO_SUCH_GRAMMAR_SCOPE(arg,arg2) ::=
"reference to undefined grammar in rule reference: <arg>.<arg2>"
NO_SUCH_RULE_IN_SCOPE(arg,arg2) ::=
"rule <arg2> is not defined in grammar <arg>"
TOKEN_ALIAS_CONFLICT(arg,arg2) ::=
"cannot alias <arg>; string already assigned to <arg2>"
TOKEN_ALIAS_REASSIGNMENT(arg,arg2) ::=
"cannot alias <arg>; token name already <if(arg2)>assigned to <arg2><else>defined<endif>"
TOKEN_VOCAB_IN_DELEGATE(arg,arg2) ::=
"tokenVocab option ignored in imported grammar <arg>"
TOKEN_ALIAS_IN_DELEGATE(arg,arg2) ::=
"can't assign string to token name <arg> to string in imported grammar <arg2>"
INVALID_IMPORT(arg,arg2) ::=
"<arg.typeString> grammar <arg.name> cannot import <arg2.typeString> grammar <arg2.name>"
IMPORTED_TOKENS_RULE_EMPTY(arg,arg2) ::=
"no lexer rules contributed to <arg> from imported grammar <arg2>"
IMPORT_NAME_CLASH(arg,arg2) ::=
"<arg.typeString> grammar <arg.name> and imported <arg2.typeString> grammar <arg2.name> both generate <arg2.recognizerName>"
AST_OP_IN_ALT_WITH_REWRITE(arg,arg2) ::=
"rule <arg> alt <arg2> uses rewrite syntax and also an AST operator"
WILDCARD_AS_ROOT(arg) ::= "Wildcard invalid as root; wildcard can itself be a tree"
CONFLICTING_OPTION_IN_TREE_FILTER(arg,arg2) ::=
"option <arg>=<arg2> conflicts with tree grammar filter mode"
// GRAMMAR WARNINGS
AMBIGUITY(arg) ::= <<
Decision can match input such as "<arg.input>" using multiple alternatives:
<arg.conflictingPaths.keys:{alt | alt <alt> via token <arg.conflictingPaths.(alt):{t | <t.text> at line <t.line>:<t.charPositionInLine>}; separator=" then ">}; separator=" and ">
<if(arg.hasPredicateBlockedByAction)><\n>At least one possibly relevant semantic predicate was hidden by action(s).<endif>
>>
/*
GRAMMAR_NONDETERMINISM(input,conflictingAlts,paths,disabled,hasPredicateBlockedByAction) ::=
<<
<if(paths)>
Decision can match input such as "<input>" using multiple alternatives:
<paths:{ alt <it.alt> via NFA path <it.states; separator=","><\n>}>
<else>
Decision can match input such as "<input>" using multiple alternatives: <conflictingAlts; separator=", ">
<endif>
<if(disabled)><\n>As a result, alternative(s) <disabled; separator=","> were disabled for that input<endif><if(hasPredicateBlockedByAction)><\n>Semantic predicates were present but were hidden by actions.<endif>
>>
*/
DANGLING_STATE(danglingAlts,input) ::= <<
the decision cannot distinguish between alternative(s) <danglingAlts; separator=","> for input such as "<input>"
>>
UNREACHABLE_ALTS(arg) ::= <<
The following alternatives can never be matched: <arg.alts; separator=","><\n>
>>
INSUFFICIENT_PREDICATES(arg) ::= <<
Input such as "<arg.input>" is insufficiently covered with predicates at locations: <arg.altToLocations.keys:{alt|alt <alt>: <arg.altToLocations.(alt):{loc| line <loc.line>:<loc.charPositionInLine> at <loc.text>}; separator=", ">}; separator=", ">
<if(arg.hasPredicateBlockedByAction)><\n>At least one possibly relevant semantic predicate was hidden by action(s).<endif>
>>
ANALYSIS_TIMEOUT(arg) ::= <<
ANTLR could not analyze this decision in rule <arg.enclosingRule>; often this is because of recursive rule references visible from the left edge of alternatives. ANTLR will re-analyze the decision with a fixed lookahead of k=1. Consider using "options {k=1;}" for that decision and possibly adding a syntactic predicate
>>
/*
RECURSION_OVERFLOW(arg) ::= <<
Recursion overflow to <arg.targetRule.name> from alternative <arg.alt> of <arg.sourceRule.name> after matching input such as <arg.input>
>>
*/
LEFT_RECURSION_CYCLES(arg) ::= <<
The following sets of rules are mutually left-recursive <arg:{c| [<c:{r|<r.name>}; separator=", ">]}; separator=" and ">
>>
/*
MULTIPLE_RECURSIVE_ALTS(arg) ::= <<
[fatal] rule <arg.ruleName> has non-LL(*) decision due to recursive rule invocations reachable from alts <arg.alts; separator=",">. Resolve by left-factoring or using syntactic predicates or using backtrack=true option.
>>
*/
UNREACHABLE_TOKENS(tokens) ::= <<
The following token definitions can never be matched because prior tokens match the same input: <tokens; separator=",">
>>
DUPLICATE_SET_ENTRY(arg) ::=
"duplicate token type <arg> when collapsing subrule into set"
TOKEN_NONDETERMINISM(input,conflictingTokens,paths,disabled,hasPredicateBlockedByAction) ::=
<<
<if(paths)>
Decision can match input such as "<input>" using multiple alternatives:
<paths:{it | alt <it.alt> via NFA path <it.states; separator=","><\n>}>
<else>
Multiple token rules can match input such as "<input>": <conflictingTokens; separator=", "><\n>
<endif>
<if(disabled)><\n>As a result, token(s) <disabled; separator=","> were disabled for that input<endif><if(hasPredicateBlockedByAction)><\n>Semantic predicates were present but were hidden by actions.<endif>
>>
/* l10n for message levels */
WARNING() ::= "warning"
ERROR() ::= "error"

858
tool/src/org/antlr/v4/Tool.java
View File

@ -1,858 +0,0 @@
package org.antlr.v4;
import org.antlr.runtime.*;
import org.antlr.runtime.tree.TreeWizard;
import org.antlr.v4.analysis.AnalysisPipeline;
import org.antlr.v4.automata.DFA;
import org.antlr.v4.automata.LexerNFAFactory;
import org.antlr.v4.automata.NFAFactory;
import org.antlr.v4.automata.ParserNFAFactory;
import org.antlr.v4.codegen.CodeGenPipeline;
import org.antlr.v4.parse.ANTLRLexer;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.parse.ToolANTLRParser;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.tool.*;
import java.io.*;
import java.util.*;
public class Tool {
public final Properties antlrSettings = new Properties();
public String VERSION = "!Unknown version!";
//public static final String VERSION = "${project.version}";
public static final String UNINITIALIZED_DIR = "<unset-dir>";
public ErrorManager errMgr = new ErrorManager(this);
List<ANTLRToolListener> listeners =
Collections.synchronizedList(new ArrayList<ANTLRToolListener>());
/** Track separately so if someone adds a listener, it's the only one
* instead of it and the default stderr listener.
*/
DefaultToolListener defaultListener = new DefaultToolListener(this);
Map<String, Grammar> grammars = new HashMap<String, Grammar>();
// GRAMMARS
public List<String> grammarFileNames = new ArrayList<String>();
// COMMAND-LINE OPTIONS
public boolean generate_NFA_dot = false;
public boolean generate_DFA_dot = false;
public String outputDirectory = ".";
public boolean haveOutputDir = false;
public String inputDirectory = null;
public String parentGrammarDirectory;
public String grammarOutputDirectory;
public boolean haveInputDir = false;
public String libDirectory = ".";
public boolean debug = false;
public boolean trace = false;
public boolean profile = false;
public boolean report = false;
public boolean printGrammar = false;
public boolean depend = false;
public boolean forceAllFilesToOutputDir = false;
public boolean forceRelativeOutput = false;
public boolean deleteTempLexer = true;
public boolean minimizeDFA = true;
public boolean verbose = false;
/** Don't process grammar file if generated files are newer than grammar */
/**
* Indicate whether the tool should analyze the dependencies of the provided grammar
* file list and ensure that the grammars with dependencies are built
* after any of the other gramamrs in the list that they are dependent on. Setting
* this option also has the side effect that any grammars that are includes for other
* grammars in the list are excluded from individual analysis, which allows the caller
* to invoke the tool via org.antlr.tool -make *.g and not worry about the inclusion
* of grammars that are just includes for other grammars or what order the grammars
* appear on the command line.
*
* This option was coded to make life easier for tool integration (such as Maven) but
* may also be useful at the command line.
*
* @param make
*/
public boolean make = false;
public boolean showBanner = true;
/** Exit after showing version or whatever */
public static boolean exitNow = false;
// The internal options are for my use on the command line during dev
public static boolean internalOption_PrintGrammarTree = false;
public static boolean internalOption_PrintDFA = false;
public static boolean internalOption_ShowNFAConfigsInDFA = false;
public static boolean internalOption_watchNFAConversion = false;
public static boolean internalOption_saveTempLexer = false;
public static void main(String[] args) {
Tool antlr = new Tool(args);
if (!exitNow) {
antlr.processGrammarsOnCommandLine();
if (antlr.errMgr.getNumErrors() > 0) {
antlr.exit(1);
}
antlr.exit(0);
}
}
public Tool() { }
public Tool(String[] args) {
this();
processArgs(args);
}
public void exit(int e) { System.exit(e); }
public void panic() { throw new Error("ANTLR panic"); }
public void processArgs(String[] args) {
if (verbose) {
info("ANTLR Parser Generator Version " + VERSION);
showBanner = false;
}
if (args == null || args.length == 0) {
help();
return;
}
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-o") || args[i].equals("-fo")) {
if (i + 1 >= args.length) {
System.err.println("missing output directory with -fo/-o option; ignoring");
}
else {
if (args[i].equals("-fo")) { // force output into dir
forceAllFilesToOutputDir = true;
}
i++;
outputDirectory = args[i];
if (outputDirectory.endsWith("/") ||
outputDirectory.endsWith("\\")) {
outputDirectory =
outputDirectory.substring(0, outputDirectory.length() - 1);
}
File outDir = new File(outputDirectory);
haveOutputDir = true;
if (outDir.exists() && !outDir.isDirectory()) {
errMgr.toolError(ErrorType.OUTPUT_DIR_IS_FILE, outputDirectory);
libDirectory = ".";
}
}
}
else if (args[i].equals("-lib")) {
if (i + 1 >= args.length) {
System.err.println("missing library directory with -lib option; ignoring");
}
else {
i++;
libDirectory = args[i];
if (libDirectory.endsWith("/") ||
libDirectory.endsWith("\\")) {
libDirectory = libDirectory.substring(0, libDirectory.length() - 1);
}
File outDir = new File(libDirectory);
if (!outDir.exists()) {
errMgr.toolError(ErrorType.DIR_NOT_FOUND, libDirectory);
libDirectory = ".";
}
}
}
else if (args[i].equals("-nfa")) {
generate_NFA_dot = true;
}
else if (args[i].equals("-dfa")) {
generate_DFA_dot = true;
}
else if (args[i].equals("-debug")) {
debug = true;
}
else if (args[i].equals("-trace")) {
trace = true;
}
else if (args[i].equals("-report")) {
report = true;
}
else if (args[i].equals("-profile")) {
profile = true;
}
else if (args[i].equals("-print")) {
printGrammar = true;
}
else if (args[i].equals("-depend")) {
depend = true;
}
else if (args[i].equals("-verbose")) {
verbose = true;
}
else if (args[i].equals("-version")) {
version();
exitNow = true;
}
else if (args[i].equals("-make")) {
make = true;
}
else if (args[i].equals("-message-format")) {
if (i + 1 >= args.length) {
System.err.println("missing output format with -message-format option; using default");
}
else {
i++;
//ErrorManager.setFormat(args[i]);
}
}
else if (args[i].equals("-Xgrtree")) {
internalOption_PrintGrammarTree = true; // print grammar tree
}
else if (args[i].equals("-Xdfa")) {
internalOption_PrintDFA = true;
}
else if (args[i].equals("-Xnominimizedfa")) {
minimizeDFA = false;
}
else if (args[i].equals("-Xnoprune")) {
//DFAOptimizer.PRUNE_EBNF_EXIT_BRANCHES = false;
}
else if (args[i].equals("-Xnocollapse")) {
//DFAOptimizer.COLLAPSE_ALL_PARALLEL_EDGES = false;
}
else if (args[i].equals("-Xdbgconversion")) {
//NFAToDFAConverter.debug = true;
}
else if (args[i].equals("-Xmultithreaded")) {
//NFAToDFAConverter.SINGLE_THREADED_NFA_CONVERSION = false;
}
else if (args[i].equals("-Xnomergestopstates")) {
//DFAOptimizer.MERGE_STOP_STATES = false;
}
else if (args[i].equals("-Xdfaverbose")) {
internalOption_ShowNFAConfigsInDFA = true;
}
else if (args[i].equals("-Xsavelexer")) {
internalOption_saveTempLexer = true;
}
else if (args[i].equals("-Xwatchconversion")) {
internalOption_watchNFAConversion = true;
}
else if (args[i].equals("-XdbgST")) {
//CodeGenerator.EMIT_TEMPLATE_DELIMITERS = true;
}
else if (args[i].equals("-Xmaxinlinedfastates")) {
if (i + 1 >= args.length) {
System.err.println("missing max inline dfa states -Xmaxinlinedfastates option; ignoring");
}
else {
i++;
// CodeGenerator.MAX_ACYCLIC_DFA_STATES_INLINE = Integer.parseInt(args[i]);
}
}
else if (args[i].equals("-Xmaxswitchcaselabels")) {
if (i + 1 >= args.length) {
System.err.println("missing max switch case labels -Xmaxswitchcaselabels option; ignoring");
}
else {
i++;
// CodeGenerator.MAX_SWITCH_CASE_LABELS = Integer.parseInt(args[i]);
}
}
else if (args[i].equals("-Xminswitchalts")) {
if (i + 1 >= args.length) {
System.err.println("missing min switch alternatives -Xminswitchalts option; ignoring");
}
else {
i++;
// CodeGenerator.MIN_SWITCH_ALTS = Integer.parseInt(args[i]);
}
}
else if (args[i].equals("-Xm")) {
if (i + 1 >= args.length) {
System.err.println("missing max recursion with -Xm option; ignoring");
}
else {
i++;
//NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK = Integer.parseInt(args[i]);
}
}
else if (args[i].equals("-Xmaxdfaedges")) {
if (i + 1 >= args.length) {
System.err.println("missing max number of edges with -Xmaxdfaedges option; ignoring");
}
else {
i++;
// DFA.MAX_STATE_TRANSITIONS_FOR_TABLE = Integer.parseInt(args[i]);
}
}
else if (args[i].equals("-Xconversiontimeout")) {
if (i + 1 >= args.length) {
System.err.println("missing max time in ms -Xconversiontimeout option; ignoring");
}
else {
i++;
//DFA.MAX_TIME_PER_DFA_CREATION = Integer.parseInt(args[i]);
}
}
else if (args[i].equals("-Xnfastates")) {
//DecisionProbe.verbose = true;
}
else if (args[i].equals("-X")) {
Xhelp();
}
else {
if (args[i].charAt(0) != '-') {
// Must be the grammar file
addGrammarFile(args[i]);
}
}
}
}
public GrammarAST load(String fileName) {
ANTLRFileStream in = null;
try {
in = new ANTLRFileStream(fileName);
}
catch (IOException ioe) {
errMgr.toolError(ErrorType.CANNOT_OPEN_FILE, fileName, ioe);
}
return load(in);
}
public GrammarAST loadFromString(String grammar) {
return load(new ANTLRStringStream(grammar));
}
public GrammarAST load(CharStream in) {
try {
ANTLRLexer lexer = new ANTLRLexer(in);
CommonTokenStream tokens = new CommonTokenStream(lexer);
ToolANTLRParser p = new ToolANTLRParser(tokens, this);
p.setTreeAdaptor(new GrammarASTAdaptor(in));
ParserRuleReturnScope r = p.grammarSpec();
GrammarAST root = (GrammarAST) r.getTree();
if ( root instanceof GrammarRootAST ) {
((GrammarRootAST)root).hasErrors = p.getNumberOfSyntaxErrors()>0;
}
return root;
}
catch (RecognitionException re) {
// TODO: do we gen errors now?
errMgr.internalError("can't generate this message at moment; antlr recovers");
}
return null;
}
public void processGrammarsOnCommandLine() {
// TODO: process all files
GrammarAST t = load(grammarFileNames.get(0));
if ( t instanceof GrammarASTErrorNode ) return; // came back as error node
if ( ((GrammarRootAST)t).hasErrors ) return;
GrammarRootAST ast = (GrammarRootAST)t;
Grammar g = createGrammar(ast);
g.fileName = grammarFileNames.get(0);
process(g);
if ( ast!=null && ast.grammarType==ANTLRParser.COMBINED && !ast.hasErrors ) {
GrammarRootAST lexerAST = extractImplicitLexer(g); // alters ast
if ( lexerAST!=null ) {
LexerGrammar lexerg = new LexerGrammar(this, lexerAST);
lexerg.fileName = grammarFileNames.get(0);
g.implicitLexer = lexerg;
lexerg.implicitLexerOwner = g;
process(lexerg);
}
}
}
public Grammar createGrammar(GrammarRootAST ast) {
if ( ast.grammarType==ANTLRParser.LEXER ) return new LexerGrammar(this, ast);
else return new Grammar(this, ast);
}
public void process(Grammar g) {
grammars.put(g.name, g);
g.loadImportedGrammars();
if ( g.ast!=null && internalOption_PrintGrammarTree ) System.out.println(g.ast.toStringTree());
//g.ast.inspect();
// MAKE SURE GRAMMAR IS SEMANTICALLY CORRECT (FILL IN GRAMMAR OBJECT)
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if ( errMgr.getNumErrors()>0 ) return;
if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
for (Grammar imp : g.getImportedGrammars()) {
process(imp);
}
}
// BUILD NFA FROM AST
NFAFactory factory = new ParserNFAFactory(g);
if ( g.isLexer() ) factory = new LexerNFAFactory((LexerGrammar)g);
g.nfa = factory.createNFA();
if ( generate_NFA_dot ) generateNFAs(g);
// PERFORM GRAMMAR ANALYSIS ON NFA: BUILD DECISION DFAs
AnalysisPipeline anal = new AnalysisPipeline(g);
anal.process();
if ( generate_DFA_dot ) generateDFAs(g);
if ( g.tool.getNumErrors()>0 ) return;
// GENERATE CODE
CodeGenPipeline gen = new CodeGenPipeline(g);
gen.process();
}
// TODO: Move to ast manipulation class?
/** Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (scope Blort { int x; })
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Careful: nodes are shared between
* trees after this call.
*/
public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
GrammarRootAST combinedAST = combinedGrammar.ast;
//System.out.println("before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
List<GrammarAST> elements = combinedAST.getChildren();
// MAKE A GRAMMAR ROOT and ID
String lexerName = combinedAST.getChild(0).getText()+"Lexer";
GrammarRootAST lexerAST =
new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR,"LEXER_GRAMMAR"));
lexerAST.token.setInputStream(combinedAST.token.getInputStream());
lexerAST.addChild((GrammarAST)adaptor.create(ANTLRParser.ID, lexerName));
// MOVE OPTIONS
GrammarAST optionsRoot =
(GrammarAST)combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
if ( optionsRoot!=null ) {
GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.dupNode(optionsRoot);
lexerAST.addChild(lexerOptionsRoot);
List<GrammarAST> options = optionsRoot.getChildren();
for (GrammarAST o : options) {
String optionName = o.getChild(0).getText();
if ( !Grammar.doNotCopyOptionsToLexer.contains(optionName) ) {
lexerOptionsRoot.addChild(o);
}
}
}
// MOVE lexer:: actions
List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
for (GrammarAST e : elements) {
if ( e.getType()==ANTLRParser.AT ) {
if ( e.getChild(0).getText().equals("lexer") ) {
lexerAST.addChild(e);
actionsWeMoved.add(e);
}
}
}
elements.removeAll(actionsWeMoved);
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.getFirstChildWithType(ANTLRParser.RULES);
if ( combinedRulesRoot==null ) return lexerAST;
TreeWizard wiz = new TreeWizard(adaptor,ANTLRParser.tokenNames);
// MOVE lexer rules
GrammarAST lexerRulesRoot =
(GrammarAST)adaptor.create(ANTLRParser.RULES, "RULES");
lexerAST.addChild(lexerRulesRoot);
List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
List<GrammarASTWithOptions> rules = combinedRulesRoot.getChildren();
for (GrammarASTWithOptions r : rules) {
String ruleName = r.getChild(0).getText();
if ( Character.isUpperCase(ruleName.charAt(0)) ) {
lexerRulesRoot.addChild(r);
rulesWeMoved.add(r);
}
}
int nLexicalRules = rulesWeMoved.size();
rules.removeAll(rulesWeMoved);
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
Map<String,String> litAliases =
Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
if ( nLexicalRules==0 && (litAliases==null||litAliases.size()==0) &&
combinedGrammar.stringLiteralToTypeMap.size()==0 )
{
// no rules, tokens{}, or 'literals' in grammar
return null;
}
// add strings from combined grammar (and imported grammars) into to lexer
for (String lit : combinedGrammar.stringLiteralToTypeMap.keySet()) {
if ( litAliases!=null && litAliases.containsKey(lit) ) continue; // already has rule
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
//TreeWizard wiz = new TreeWizard(adaptor,ANTLRParser.tokenNames);
String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
GrammarAST litRule = (GrammarAST)
wiz.create("(RULE ID["+rname+"] (BLOCK (ALT STRING_LITERAL["+lit+"])))");
lexerRulesRoot.addChild(litRule);
}
//System.out.println("after ="+combinedAST.toStringTree());
System.out.println("lexer ="+lexerAST.toStringTree());
return lexerAST;
}
public void generateNFAs(Grammar g) {
DOTGenerator dotGenerator = new DOTGenerator(g);
List<Grammar> grammars = new ArrayList<Grammar>();
grammars.add(g);
List<Grammar> imported = g.getAllImportedGrammars();
if ( imported!=null ) grammars.addAll(imported);
for (Grammar ig : grammars) {
for (Rule r : ig.rules.values()) {
try {
String dot = dotGenerator.getDOT(g.nfa.ruleToStartState.get(r));
if (dot != null) {
writeDOTFile(g, r, dot);
}
} catch (IOException ioe) {
errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
}
}
}
}
public void generateDFAs(Grammar g) {
if ( g.isLexer() ) {
LexerGrammar lg = (LexerGrammar)g;
for (String modeName : lg.modes.keySet()) {
generateDFA(g, lg.modeToDFA.get(modeName));
}
return;
}
for (DFA dfa : g.decisionDFAs.values()) {
generateDFA(g, dfa);
}
}
public void generateDFA(Grammar g, DFA dfa) {
DOTGenerator dotGenerator = new DOTGenerator(g);
String dot = dotGenerator.getDOT(dfa.startState);
String dec = "dec-";
//if ( dfa.minimized ) dec += "min-";
String dotFileName = g.name + "." + dec + dfa.decision;
if (g.implicitLexer!=null) {
dotFileName = g.name +
Grammar.getGrammarTypeToFileNameSuffix(g.getType()) +
"." + dec + dfa.decision;
}
try {
writeDOTFile(g, dotFileName, dot);
}
catch (IOException ioe) {
errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, dotFileName, ioe);
}
}
protected void writeDOTFile(Grammar g, Rule r, String dot) throws IOException {
writeDOTFile(g, r.g.name + "." + r.name, dot);
}
protected void writeDOTFile(Grammar g, String name, String dot) throws IOException {
Writer fw = getOutputFile(g, name + ".dot");
fw.write(dot);
fw.close();
}
/** This method is used by all code generators to create new output
* files. If the outputDir set by -o is not present it will be created.
* The final filename is sensitive to the output directory and
* the directory where the grammar file was found. If -o is /tmp
* and the original grammar file was foo/t.g then output files
* go in /tmp/foo.
*
* The output dir -o spec takes precedence if it's absolute.
* E.g., if the grammar file dir is absolute the output dir is given
* precendence. "-o /tmp /usr/lib/t.g" results in "/tmp/T.java" as
* output (assuming t.g holds T.java).
*
* If no -o is specified, then just write to the directory where the
* grammar file was found.
*
* If outputDirectory==null then write a String.
*/
public Writer getOutputFile(Grammar g, String fileName) throws IOException {
if (outputDirectory == null) {
return new StringWriter();
}
// output directory is a function of where the grammar file lives
// for subdir/T.g, you get subdir here. Well, depends on -o etc...
// But, if this is a .tokens file, then we force the output to
// be the base output directory (or current directory if there is not a -o)
//
File outputDir;
if ( fileName.endsWith(".tokens") ) {// CodeGenerator.VOCAB_FILE_EXTENSION)) {
if (haveOutputDir) {
outputDir = new File(outputDirectory);
}
else {
outputDir = new File(".");
}
}
else {
outputDir = getOutputDirectory(g.fileName);
}
File outputFile = new File(outputDir, fileName);
if (!outputDir.exists()) {
outputDir.mkdirs();
}
FileWriter fw = new FileWriter(outputFile);
return new BufferedWriter(fw);
}
/**
* Return the Path to the directory in which ANTLR will search for ancillary
* files such as .tokens vocab files and imported grammar files.
*
* @return the lib Directory
*/
public String getLibraryDirectory() {
return libDirectory;
}
/**
* Return the Path to the base output directory, where ANTLR
* will generate all the output files for the current language target as
* well as any ancillary files such as .tokens vocab files.
*
* @return the output Directory
*/
public String getOutputDirectory() {
return outputDirectory;
}
/**
* Return the location where ANTLR will generate output files for a given file. This is a
* base directory and output files will be relative to here in some cases
* such as when -o option is used and input files are given relative
* to the input directory.
*
* @param fileNameWithPath path to input source
* @return
*/
public File getOutputDirectory(String fileNameWithPath) {
File outputDir = new File(outputDirectory);
String fileDirectory;
// Some files are given to us without a PATH but should should
// still be written to the output directory in the relative path of
// the output directory. The file directory is either the set of sub directories
// or just or the relative path recorded for the parent grammar. This means
// that when we write the tokens files, or the .java files for imported grammars
// taht we will write them in the correct place.
//
if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) {
// No path is included in the file name, so make the file
// directory the same as the parent grammar (which might sitll be just ""
// but when it is not, we will write the file in the correct place.
//
fileDirectory = grammarOutputDirectory;
}
else {
fileDirectory = fileNameWithPath.substring(0, fileNameWithPath.lastIndexOf(File.separatorChar));
}
if ( fileDirectory == null ) {
fileDirectory = ".";
}
if (haveOutputDir) {
// -o /tmp /var/lib/t.g => /tmp/T.java
// -o subdir/output /usr/lib/t.g => subdir/output/T.java
// -o . /usr/lib/t.g => ./T.java
if ((fileDirectory != null && !forceRelativeOutput) &&
(new File(fileDirectory).isAbsolute() ||
fileDirectory.startsWith("~")) || // isAbsolute doesn't count this :(
forceAllFilesToOutputDir) {
// somebody set the dir, it takes precendence; write new file there
outputDir = new File(outputDirectory);
}
else {
// -o /tmp subdir/t.g => /tmp/subdir/t.g
if (fileDirectory != null) {
outputDir = new File(outputDirectory, fileDirectory);
}
else {
outputDir = new File(outputDirectory);
}
}
}
else {
// they didn't specify a -o dir so just write to location
// where grammar is, absolute or relative, this will only happen
// with command line invocation as build tools will always
// supply an output directory.
//
outputDir = new File(fileDirectory);
}
return outputDir;
}
public void addListener(ANTLRToolListener tl) {
if ( tl!=null ) listeners.add(tl);
}
public void removeListener(ANTLRToolListener tl) { listeners.remove(tl); }
public void removeListeners() { listeners.clear(); }
public List<ANTLRToolListener> getListeners() { return listeners; }
public void info(String msg) {
if ( listeners.size()==0 ) {
defaultListener.info(msg);
return;
}
for (ANTLRToolListener l : listeners) l.info(msg);
}
public void error(Message msg) {
if ( listeners.size()==0 ) {
defaultListener.error(msg);
return;
}
for (ANTLRToolListener l : listeners) l.error(msg);
}
public void warning(Message msg) {
if ( listeners.size()==0 ) {
defaultListener.warning(msg);
return;
}
for (ANTLRToolListener l : listeners) l.warning(msg);
}
public void version() {
info("ANTLR Parser Generator Version " + new Tool().VERSION);
}
public void help() {
info("ANTLR Parser Generator Version " + new Tool().VERSION);
System.err.println("usage: java org.antlr.Tool [args] file.g [file2.g file3.g ...]");
System.err.println(" -o outputDir specify output directory where all output is generated");
System.err.println(" -fo outputDir same as -o but force even files with relative paths to dir");
System.err.println(" -lib dir specify location of token files");
System.err.println(" -depend generate file dependencies");
System.err.println(" -report print out a report about the grammar(s) processed");
System.err.println(" -print print out the grammar without actions");
System.err.println(" -debug generate a parser that emits debugging events");
System.err.println(" -profile generate a parser that computes profiling information");
System.err.println(" -nfa generate an NFA for each rule");
System.err.println(" -dfa generate a DFA for each decision point");
System.err.println(" -message-format name specify output style for messages");
System.err.println(" -verbose generate ANTLR version and other information");
System.err.println(" -make only build if generated files older than grammar");
System.err.println(" -version print the version of ANTLR and exit.");
System.err.println(" -X display extended argument list");
}
public void Xhelp() {
info("ANTLR Parser Generator Version " + new Tool().VERSION);
System.err.println(" -Xgrtree print the grammar AST");
System.err.println(" -Xdfa print DFA as text");
System.err.println(" -Xnominimizedfa don't minimize decision DFA");
System.err.println(" -Xnoprune test lookahead against EBNF block exit branches");
System.err.println(" -Xnocollapse collapse incident edges into DFA states");
System.err.println(" -Xdbgconversion dump lots of info during NFA conversion");
System.err.println(" -Xmultithreaded run the analysis in 2 threads");
System.err.println(" -Xnomergestopstates do not merge stop states");
System.err.println(" -Xdfaverbose generate DFA states in DOT with NFA configs");
System.err.println(" -Xwatchconversion print a message for each NFA before converting");
System.err.println(" -XdbgST put tags at start/stop of all templates in output");
System.err.println(" -Xnfastates for nondeterminisms, list NFA states for each path");
System.err.println(" -Xsavelexer save temp lexer file created for combined grammars");
/*
System.err.println(" -Xm m max number of rule invocations during conversion [" + NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK + "]");
System.err.println(" -Xmaxdfaedges m max \"comfortable\" number of edges for single DFA state [" + DFA.MAX_STATE_TRANSITIONS_FOR_TABLE + "]");
System.err.println(" -Xconversiontimeout t set NFA conversion timeout (ms) for each decision [" + DFA.MAX_TIME_PER_DFA_CREATION + "]");
System.err.println(" -Xmaxinlinedfastates m max DFA states before table used rather than inlining [" + CodeGenerator.MADSI_DEFAULT +"]");
System.err.println(" -Xmaxswitchcaselabels m don't generate switch() statements for dfas bigger than m [" + CodeGenerator.MSCL_DEFAULT +"]");
System.err.println(" -Xminswitchalts m don't generate switch() statements for dfas smaller than m [" + CodeGenerator.MSA_DEFAULT + "]");
*/
}
public void addGrammarFile(String grammarFileName) {
if (!grammarFileNames.contains(grammarFileName)) {
grammarFileNames.add(grammarFileName);
}
}
/**
* Provide the current setting of the conversion timeout on DFA creation.
*
* @return DFA creation timeout value in milliseconds
*/
public int getConversionTimeout() {
//return DFA.MAX_TIME_PER_DFA_CREATION;
return 0;
}
/**
* Returns the current setting of the message format descriptor
* @return Current message format
*/
public String getMessageFormat() {
//return ErrorManager.getMessageFormat().toString();
return null;
}
public int getNumErrors() { return errMgr.getNumErrors(); }
/**Set the message format to one of ANTLR, gnu, vs2005 */
public void setMessageFormat(String format) {
errMgr.setFormat(format);
}
/** Set the location (base directory) where output files should be produced
* by the ANTLR tool.
*/
public void setOutputDirectory(String outputDirectory) {
haveOutputDir = true;
this.outputDirectory = outputDirectory;
}
/**
* Set the base location of input files. Normally (when the tool is
* invoked from the command line), the inputDirectory is not set, but
* for build tools such as Maven, we need to be able to locate the input
* files relative to the base, as the working directory could be anywhere and
* changing workig directories is not a valid concept for JVMs because of threading and
* so on. Setting the directory just means that the getFileDirectory() method will
* try to open files relative to this input directory.
*
* @param inputDirectory Input source base directory
*/
public void setInputDirectory(String inputDirectory) {
inputDirectory = inputDirectory;
haveInputDir = true;
}
}

148
tool/src/org/antlr/v4/analysis/AnalysisPipeline.java
View File

@ -1,148 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.DFA;
import org.antlr.v4.automata.DecisionState;
import org.antlr.v4.automata.NFA;
import org.antlr.v4.misc.BitSet;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.semantics.UseDefAnalyzer;
import org.antlr.v4.tool.*;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class AnalysisPipeline {
public Grammar g;
public AnalysisPipeline(Grammar g) {
this.g = g;
}
public void process() {
// LEFT-RECURSION CHECK
LeftRecursionDetector lr = new LeftRecursionDetector(g.nfa);
lr.check();
if ( lr.listOfRecursiveCycles.size()>0 ) return; // bail out
// BUILD DFA FOR EACH DECISION
if ( g.isLexer() ) processLexer();
else processParserOrTreeParser();
}
void processLexer() {
LexerGrammar lg = (LexerGrammar)g;
for (String modeName : lg.modes.keySet()) {
Set<Rule> rulesInNFA = getRulesInNFA(lg, modeName);
LexerNFAToDFAConverter conv = new LexerNFAToDFAConverter(lg);
DFA dfa = conv.createDFA(modeName);
lg.modeToDFA.put(modeName, dfa);
//TokensStartState startState = g.nfa.modeToStartState.get(modeName);
//g.setLookaheadDFA(startState.decision, dfa);
if ( g.tool.minimizeDFA ) {
int before = dfa.stateSet.size();
DFAMinimizer dmin = new DFAMinimizer(dfa);
dfa.minimized = dmin.minimize();
int after = dfa.stateSet.size();
if ( after < before ) {
System.out.println("DFA minimized from "+before+" to "+after+" states");
}
}
}
}
void processParserOrTreeParser() {
for (DecisionState s : g.nfa.decisionToNFAState) {
System.out.println("\nDECISION "+s.decision);
// TRY LINEAR APPROX FIXED LOOKAHEAD FIRST
LinearApproximator lin = new LinearApproximator(g, s.decision);
DFA dfa = lin.createDFA(s);
// IF NOT LINEAR APPROX, TRY NFA TO DFA CONVERSION
if ( dfa==null ) {
dfa = createDFA(s);
}
g.setLookaheadDFA(s.decision, dfa);
}
}
public DFA createDFA(DecisionState s) {
PredictionDFAFactory conv = new PredictionDFAFactory(g, s);
DFA dfa = conv.createDFA();
System.out.print("DFA="+dfa);
if ( !dfa.valid() ) {
System.out.println("invalid DFA");
}
conv.issueAmbiguityWarnings();
// MINIMIZE DFA
if ( g.tool.minimizeDFA ) {
System.out.println("MINIMIZE");
int before = dfa.stateSet.size();
DFAMinimizer dmin = new DFAMinimizer(dfa);
dfa.minimized = dmin.minimize();
int after = dfa.stateSet.size();
if ( after < before ) {
System.out.println("DFA minimized from "+before+" to "+after+" states");
}
}
return dfa;
}
public Set<Rule> getRulesInNFA(LexerGrammar lg, String modeName) {
Set<Rule> rulesInNFA = getRulesTooComplexForDFA(lg, modeName);
System.out.println("rules in NFA: "+rulesInNFA);
IntervalSet charsPredictingNFARules = new IntervalSet();
for (Rule r : rulesInNFA) {
if ( !r.isFragment() ) {
LinearApproximator approx = new LinearApproximator(lg, NFA.INVALID_DECISION_NUMBER);
IntervalSet fset = approx.FIRST(lg.nfa.ruleToStartState.get(r));
System.out.println("first of "+r.name+"="+fset);
charsPredictingNFARules.addAll(fset);
}
}
System.out.println("charsPredictingNFARules="+charsPredictingNFARules);
// now find any other rules that start with that set
for (Rule r : lg.modes.get(modeName)) {
if ( !r.isFragment() && !rulesInNFA.contains(r) ) {
LinearApproximator approx = new LinearApproximator(lg, NFA.INVALID_DECISION_NUMBER);
IntervalSet fset = approx.FIRST(lg.nfa.ruleToStartState.get(r));
if ( !fset.and(charsPredictingNFARules).isNil() ) {
System.out.println("rule "+r.name+" collides");
rulesInNFA.add(r);
}
}
}
return rulesInNFA;
}
// TODO: oops. find all nongreedy loops too!
public Set<Rule> getRulesTooComplexForDFA(LexerGrammar lg, String modeName) {
Set<Rule> complexRules = new HashSet<Rule>();
Map<Rule, Set<Rule>> dep = UseDefAnalyzer.getRuleDependencies(lg, modeName);
System.out.println("dep="+dep);
for (Rule r : lg.modes.get(modeName)) {
if ( dep.containsKey(r) ) { complexRules.add(r); continue; }
BitSet labelTypes = BitSet.of(ANTLRParser.ASSIGN);
labelTypes.add(ANTLRParser.PLUS_ASSIGN);
List<GrammarAST> labels = r.ast.getNodesWithType(labelTypes);
if ( labels.size()>0 ) { complexRules.add(r); continue; }
List<GrammarAST> actions = r.ast.getNodesWithType(ANTLRParser.ACTION);
ActionAST actionOnFarRight = r.ast.getLexerAction();
for (GrammarAST action : actions) {
if ( action != actionOnFarRight ) complexRules.add(r);
}
}
return complexRules;
}
}

30
tool/src/org/antlr/v4/analysis/CycleDetector.java
View File

@ -1,30 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.DFA;
import org.antlr.v4.automata.DFAState;
import org.antlr.v4.automata.Edge;
// TODO: don't need at moment but leave; will need for code gen
public class CycleDetector {
DFA dfa;
boolean[] busy;
public CycleDetector(DFA dfa) {
this.dfa = dfa;
busy = new boolean[dfa.stateSet.size()+1];
}
public boolean isCyclic() {
return foundCycle(dfa.startState);
}
public boolean foundCycle(DFAState d) {
if ( busy[d.stateNumber] ) return true;
busy[d.stateNumber] = true;
for (Edge e : d.edges) {
if ( foundCycle(e.target) ) return true;
}
busy[d.stateNumber] = false;
return false;
}
}

272
tool/src/org/antlr/v4/analysis/DFAMinimizer.java
View File

@ -1,272 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.DFA;
import org.antlr.v4.automata.DFAState;
import org.antlr.v4.automata.Edge;
import org.antlr.v4.automata.PredicateEdge;
import org.antlr.v4.misc.IntSet;
import org.antlr.v4.misc.Interval;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.misc.OrderedHashSet;
import java.util.*;
/** First consolidate accept states, which leads to smaller DFA. Also,
* consolidate all edges from p to q into a single edge with set.
*/
public class DFAMinimizer {
DFA dfa;
public DFAMinimizer(DFA dfa) {
this.dfa = dfa;
}
public boolean minimize() {
int n = dfa.states.size();
boolean[][] distinct = new boolean[n][n];
Set<IntSet> labels = new HashSet<IntSet>();
for (DFAState d : dfa.states) {
for (Edge e : d.edges) {
// todo: slow? might want to flatten to list of int token types
if ( !(e instanceof PredicateEdge) ) {
labels.add(e.label);
}
}
}
System.out.println("labels="+labels);
// create initial partition distinguishing between states and accept states
// we need to distinguish between accepts for different alts.
// we may have also have multiple accepts per alt--put all of them in same partition
for (int alt=1; alt<=dfa.nAlts; alt++) {
List<DFAState> acceptsForAlt = dfa.altToAcceptStates[alt];
if ( acceptsForAlt==null ) continue; // hmm...must be unreachable
// distinguish all these accepts from every other state
for (DFAState p : acceptsForAlt) {
for (int i=0; i<n; i++) {
DFAState q = dfa.states.get(i);
// if q not accept state or p and q predict diff alts, distinguish them
if ( !q.isAcceptState || q.predictsAlt!=alt ) {
distinct[p.stateNumber][i] = true;
distinct[i][p.stateNumber] = true;
}
}
}
}
// Nobody can merge with a state resolved with predicates to be safe
if ( dfa.converter!=null ) {
for (DFAState d : dfa.converter.resolver.resolvedWithSemanticPredicates) {
for (int i=1; i<n; i++) {
distinct[d.stateNumber][i] = true;
distinct[i][d.stateNumber] = true;
}
}
}
for (int i=1; i<n; i++) {
for (int j=0; j<i; j++) {
DFAState p = dfa.states.get(i);
DFAState q = dfa.states.get(j);
if ( (p.isAcceptState && !q.isAcceptState) ||
(!p.isAcceptState && q.isAcceptState) )
{
// make symmetric even though algorithsm on web don't
// seems that DISTINCT(?(p, a),?(q, a)) might go out of
// range in my examples. Maybe they assume symmetric
// w/o saying it. Didn't see any code.
distinct[i][j] = true;
distinct[j][i] = true;
}
}
}
print(distinct);
boolean changed = true;
while ( changed ) {
changed = false;
for (int i=1; i<n; i++) {
for (int j=0; j<i; j++) {
if ( distinct[i][j] ) continue;
DFAState p = dfa.states.get(i);
DFAState q = dfa.states.get(j);
for (IntSet label : labels) {
// leave all states with gated pred transitions on this label as distinct
SemanticContext p_preds = p.getGatedPredicatesInNFAConfigurations();
SemanticContext q_preds = q.getGatedPredicatesInNFAConfigurations();
boolean preds_present = p_preds!=null || q_preds!=null;
DFAState pt = p.target(label);
DFAState qt = q.target(label);
// System.out.println(p.stateNumber+"-"+label.toString(dfa.g)+"->"+pt);
// System.out.println(q.stateNumber+"-"+label.toString(dfa.g)+"->"+qt);
// if DISTINCT(p,q) is empty and
// DISTINCT(?(p, a),?(q, a)) is not empty
// then DISTINCT(p,q) = a.
// No one seems to show example of case where
// ?(p,a)==nil. I assume that if one of states
// can't transition on label, assume p,q are distinct.
// If both p,q can't transition on label, we don't
// know anything about their distinctness.
// AH! jflex code says alg assumes DFA is "total" so
// it adds error state. If both are errors, same state
// so leave as equiv (nondistinct). If one goes to
// error (pt or qt is null) and other doesn't, must
// be in distinct sets so p,q are distinct.
boolean bothTargetsAreErrors = pt == null && qt == null;
if ( bothTargetsAreErrors && !preds_present ) continue;
if ( pt==null || qt==null ||
preds_present ||
distinct[pt.stateNumber][qt.stateNumber] )
{
distinct[i][j] = true;
distinct[j][i] = true;
changed = true;
break; // we've marked; move to next state
}
}
}
}
}
print(distinct);
// Make equiv sets using transitive property
IntervalSet[] stateToSet = new IntervalSet[n];
for (int i=0; i<n; i++) stateToSet[i] = new IntervalSet();
//System.out.println("equiv pairs:");
for (int i=1; i<n; i++) {
for (int j=0; j<i; j++) {
if ( !distinct[i][j] ) {
// System.out.println(i+","+j);
stateToSet[i].add(i);
stateToSet[i].add(j);
stateToSet[j].add(i);
stateToSet[j].add(j);
}
}
}
//System.out.println("equiv sets:");
OrderedHashSet<IntervalSet> uniq = new OrderedHashSet<IntervalSet>();
for (int i=0; i<stateToSet.length; i++) {
IntervalSet s = stateToSet[i];
if ( s.isNil() ) s.add(i); // i must be it's own set if not equiv
//if ( s.isNil() ) continue;
System.out.println(s);
uniq.add(s);
}
//System.out.println("uniq sets = "+uniq);
if ( uniq.size()==dfa.states.size() ) {
System.out.println("was already minimal");
return false;
}
// minimize the DFA (combine equiv sets)
// merge all edges from a set to first state in set
// newstates[oldstate] = new state number for oldstate
DFAState[] oldToNewStateMap = new DFAState[n];
OrderedHashSet<DFAState> uniqNewStates = new OrderedHashSet<DFAState>();
// first map all states in set to same DFA state (old min)
for (IntervalSet s : uniq) {
int newStateNum = s.getMinElement();
uniqNewStates.add(dfa.states.get(newStateNum));
oldToNewStateMap[newStateNum] = dfa.states.get(newStateNum);
List<Interval> intervals = s.getIntervals();
for (Interval I : intervals) {
for (int i=I.a; i<=I.b; i++) {
oldToNewStateMap[i] = oldToNewStateMap[newStateNum];
}
}
}
// for (DFAState s : oldToNewStateMap) System.out.println(s);
// now do edges
// for (IntervalSet equivStates : uniq) {
// List<Interval> intervals_in_state_set = equivStates.getIntervals();
// System.out.println("do set "+equivStates);
// // for each state in equiv state set, make all edges point at new state
// for (Interval I : intervals_in_state_set) {
// for (int i=I.a; i<=I.b; i++) {
// DFAState p = dfa.states.get(i);
// for (Edge e : p.edges) {
// System.out.println(p.stateNumber+" upon "+e.toString(dfa.g)+
// " used to point at "+e.target.stateNumber+
// " now points at "+ newstates[e.target.stateNumber].stateNumber);
// e.target = newstates[e.target.stateNumber];
// }
// }
// }
// }
// simpler version of above
for (DFAState d : uniqNewStates) {
for (Edge e : d.edges) {
// System.out.println(d.stateNumber+" upon "+e.toString(dfa.g)+
// " used to point at "+e.target.stateNumber+
// " now points at "+ oldToNewStateMap[e.target.stateNumber].stateNumber);
e.target = oldToNewStateMap[e.target.stateNumber];
}
}
// merge all edges from p to q
for (DFAState d : uniqNewStates) {
Map<DFAState, IntervalSet> targetToEdges = new HashMap<DFAState, IntervalSet>();
for (Edge e : d.edges) {
IntervalSet s = targetToEdges.get(e.target);
if ( s==null ) { s = new IntervalSet(e.label); targetToEdges.put(e.target, s); }
else s.addAll(e.label);
}
// System.out.println("state "+d.stateNumber+" has "+d.edges.size()+" edges but "+targetToEdges.size()+" targets");
d.edges.clear();
for (DFAState target : targetToEdges.keySet()) {
d.addEdge(new Edge(target, targetToEdges.get(target)));
}
}
// now kill unused states
for (IntervalSet equivStates : uniq) {
List<Interval> intervals_in_state_set = equivStates.getIntervals();
for (Interval I : intervals_in_state_set) {
for (int i=I.a; i<=I.b; i++) {
if ( oldToNewStateMap[i].stateNumber != i ) { // if not one of our merged states
// System.out.println("kill "+i);
DFAState d = dfa.states.get(i);
dfa.stateSet.remove(d);
if ( d.isAcceptState ) {
dfa.altToAcceptStates[d.predictsAlt].remove(d);
}
dfa.states.set(i, null);
}
}
}
}
// RENUMBER STATES
int i = 0;
for (DFAState d : dfa.states) {
if ( d!=null ) d.stateNumber = i++;
}
return true;
}
void print(boolean[][] distinct) {
int n = distinct.length;
for (int i=0; i<n; i++) {
System.out.print(dfa.states.get(i).stateNumber+":");
for (int j=0; j<n; j++) {
System.out.print(" "+(distinct[i][j]?"T":"F"));
}
System.out.println();
}
System.out.print(" ");
for (int j=0; j<n; j++) System.out.print(" "+j);
System.out.println();
System.out.println();
}
}

115
tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java
View File

@ -1,115 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.*;
import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.tool.Rule;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class LeftRecursionDetector {
public NFA nfa;
/** Holds a list of cycles (sets of rule names). */
public List<Set<Rule>> listOfRecursiveCycles = new ArrayList<Set<Rule>>();
/** Which rule start states have we visited while looking for a single
* left-recursion check?
*/
Set<RuleStartState> rulesVisitedPerRuleCheck = new HashSet<RuleStartState>();
public LeftRecursionDetector(NFA nfa) { this.nfa = nfa; }
public void check() {
for (RuleStartState start : nfa.ruleToStartState.values()) {
//System.out.print("check "+start.rule.name);
rulesVisitedPerRuleCheck.clear();
rulesVisitedPerRuleCheck.add(start);
//FASerializer ser = new FASerializer(nfa.g, start);
//System.out.print(":\n"+ser+"\n");
check(start.rule, start, new HashSet<NFAState>());
}
//System.out.println("cycles="+listOfRecursiveCycles);
if ( listOfRecursiveCycles.size()>0 ) {
nfa.g.tool.errMgr.leftRecursionCycles(nfa.g.fileName, listOfRecursiveCycles);
}
}
/** From state s, look for any transition to a rule that is currently
* being traced. When tracing r, visitedPerRuleCheck has r
* initially. If you reach a rule stop state, return but notify the
* invoking rule that the called rule is nullable. This implies that
* invoking rule must look at follow transition for that invoking state.
*
* The visitedStates tracks visited states within a single rule so
* we can avoid epsilon-loop-induced infinite recursion here. Keep
* filling the cycles in listOfRecursiveCycles and also, as a
* side-effect, set leftRecursiveRules.
*/
public boolean check(Rule enclosingRule, NFAState s, Set<NFAState> visitedStates) {
if ( s instanceof RuleStopState ) return true;
if ( visitedStates.contains(s) ) return false;
visitedStates.add(s);
//System.out.println("visit "+s);
int n = s.getNumberOfTransitions();
boolean stateReachesStopState = false;
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( t instanceof RuleTransition ) {
RuleTransition rt = (RuleTransition) t;
Rule r = rt.rule;
if ( rulesVisitedPerRuleCheck.contains((RuleStartState)t.target) ) {
addRulesToCycle(enclosingRule, r);
}
else {
// must visit if not already visited; mark target, pop when done
rulesVisitedPerRuleCheck.add((RuleStartState)t.target);
// send new visitedStates set per rule invocation
boolean nullable = check(r, t.target, new HashSet<NFAState>());
// we're back from visiting that rule
rulesVisitedPerRuleCheck.remove((RuleStartState)t.target);
if ( nullable ) {
stateReachesStopState |= check(enclosingRule, rt.followState, visitedStates);
}
}
}
else if ( t.isEpsilon() ) {
stateReachesStopState |= check(enclosingRule, t.target, visitedStates);
}
// else ignore non-epsilon transitions
}
return stateReachesStopState;
}
/** enclosingRule calls targetRule. Find the cycle containing
* the target and add the caller. Find the cycle containing the caller
* and add the target. If no cycles contain either, then create a new
* cycle.
*/
protected void addRulesToCycle(Rule enclosingRule, Rule targetRule) {
//System.err.println("left-recursion to "+targetRule.name+" from "+enclosingRule.name);
boolean foundCycle = false;
for (int i = 0; i < listOfRecursiveCycles.size(); i++) {
Set<Rule> rulesInCycle = listOfRecursiveCycles.get(i);
// ensure both rules are in same cycle
if ( rulesInCycle.contains(targetRule) ) {
rulesInCycle.add(enclosingRule);
foundCycle = true;
}
if ( rulesInCycle.contains(enclosingRule) ) {
rulesInCycle.add(targetRule);
foundCycle = true;
}
}
if ( !foundCycle ) {
Set<Rule> cycle = new OrderedHashSet<Rule>();
cycle.add(targetRule);
cycle.add(enclosingRule);
listOfRecursiveCycles.add(cycle);
}
}
}

305
tool/src/org/antlr/v4/analysis/LexerNFAToDFAConverter.java
View File

@ -1,305 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.*;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.RuleAST;
import java.util.*;
// TODO: might not need anymore if NFA simulator is fast enough
public class LexerNFAToDFAConverter {
Grammar g;
/** DFA we are creating */
DFA dfa;
/** A list of DFA states we still need to process during NFA conversion */
List<LexerState> work = new LinkedList<LexerState>();
/** The set of rule stop NFA states we encountered during conversion.
* Walk this list to find ambig stop states (split if we have preds).
*/
Set<LexerState> accepts = new HashSet<LexerState>();
//int[] altToRuleIndex;
/** Used to prevent the closure operation from looping to itself and
* hence looping forever. Sensitive to the NFA state, the alt, and
* the stack context.
*/
Set<NFAConfig> closureBusy;
public static boolean debug = false;
public LexerNFAToDFAConverter(LexerGrammar g) {
this.g = g;
//altToRuleIndex = new int[g.getNumRules()+1]; // alts <= num rules
}
public DFA createDFA() { return createDFA(LexerGrammar.DEFAULT_MODE_NAME); }
public DFA createDFA(String modeName) {
TokensStartState startState = g.nfa.modeToStartState.get(modeName);
dfa = new DFA(g, startState);
closureBusy = new HashSet<NFAConfig>();
LexerState start = computeStartState();
dfa.startState = start;
dfa.addState(start); // make sure dfa knows about this state
work.add((LexerState)dfa.startState);
// while more DFA states to check, process them
while ( work.size()>0 ) {
LexerState d = work.get(0);
reach(d);
work.remove(0); // we're done with this DFA state
}
defineLexerAcceptStates();
closureBusy = null; // wack all that memory used during closure
return dfa;
}
// walk accept states, informing DFA.
// get list of NFA states per each DFA accept so we can get list of
// rules matched (sorted by NFA state num, which gives priority to
// rules appearing first in grammar).
// Also, track any extreme right edge actions in
// DFA accept state (pick action of first of any ambig rules).
void defineLexerAcceptStates() {
int aaa = 0;
System.out.println("accepts ="+accepts);
for (LexerState d : accepts) {
if ( d.edges.size()==0 ) aaa++;
// First get NFA accept states and associated DFA alts for this DFA state
SortedSet<Integer> nfaAcceptStates = new TreeSet<Integer>();
SortedSet<Integer> sortedAlts = new TreeSet<Integer>();
OrderedHashSet<Rule> predictedRules = new OrderedHashSet<Rule>();
for (NFAConfig c : d.nfaConfigs) {
NFAState s = c.state;
if ( s instanceof RuleStopState && !s.rule.isFragment() ) {
nfaAcceptStates.add(Utils.integer(s.stateNumber));
sortedAlts.add(c.alt);
predictedRules.add(s.rule);
}
}
// Look for and count preds
Map<Integer, SemanticContext> predsPerAlt = d.getPredicatesForAlts();
int npreds = 0;
for (SemanticContext ctx : predsPerAlt.values()) if ( ctx!=null ) npreds++;
// If unambiguous, make it a DFA accept state, else resolve with preds if possible
if ( predictedRules.size()==1 || npreds==0 ) { // unambig or no preds
d.predictsRule = predictedRules.get(0);
d.action = ((RuleAST)d.predictsRule.ast).getLexerAction();
Integer minAlt = sortedAlts.first();
dfa.defineAcceptState(minAlt, d);
}
if ( predictedRules.size()>1 && npreds>0 ) {
System.out.println(d.stateNumber+" ambig upon "+ predictedRules+" but we have preds");
// has preds; add new accept states
d.isAcceptState = false; // this state isn't a stop state anymore
d.resolvedWithPredicates = true;
for (Rule r : predictedRules) {
SemanticContext preds = predsPerAlt.get(r.index);
LexerState predDFATarget = dfa.newLexerState();
predDFATarget.predictsRule = r;
for (NFAConfig c : d.getNFAConfigsForAlt(r.index)) {
predDFATarget.addNFAConfig(c);
}
// new DFA state is a target of the predicate from d
//predDFATarget.addNFAConfig(c);
dfa.addAcceptState(r.index, predDFATarget);
// add a transition to pred target from d
if ( preds!=null ) {
d.addEdge(new PredicateEdge(preds, predDFATarget));
}
else {
d.addEdge(new PredicateEdge(new SemanticContext.TruePredicate(), predDFATarget));
}
}
}
}
System.out.println("#accepts ="+accepts.size()+" and "+aaa+" with no edges");
}
/** */
public LexerState computeStartState() {
LexerState d = dfa.newLexerState();
// add config for each alt start, then add closure for those states
for (int alt=1; alt<=dfa.nAlts; alt++) {
Transition t = dfa.decisionNFAStartState.transition(alt-1);
NFAState altStart = t.target;
//altToRuleIndex[alt] = altStart.rule.index;
d.addNFAConfig(
new NFAConfig(altStart, alt,
NFAContext.EMPTY(),
SemanticContext.EMPTY_SEMANTIC_CONTEXT));
}
closure(d, true);
return d;
}
/** From this node, add a d--a-->t transition for all
* labels 'a' where t is a DFA node created
* from the set of NFA states reachable from any NFA
* configuration in DFA state d.
*/
void reach(LexerState d) {
OrderedHashSet<IntervalSet> labels = DFA.getReachableLabels(d);
for (IntervalSet label : labels) {
LexerState t = reach(d, label);
if ( debug ) {
System.out.println("DFA state after reach -" +
label.toString(g)+"->"+t);
}
closure(t, true); // add any NFA states reachable via epsilon
addTransition(d, label, t); // make d-label->t transition
}
}
/** Add t if not in DFA yet and then make d-label->t */
void addTransition(LexerState d, IntervalSet label, LexerState t) {
LexerState existing = (LexerState)dfa.stateSet.get(t);
if ( existing != null ) { // seen before; point at old one
d.addEdge(new Edge(existing, label));
return;
}
//System.out.println("ADD "+t);
work.add(t); // add to work list to continue NFA conversion
dfa.addState(t); // add state we've never seen before
if ( t.isAcceptState ) accepts.add(t);
d.addEdge(new Edge(t, label));
}
/** Given the set of NFA states in DFA state d, find all NFA states
* reachable traversing label arcs. By definition, there can be
* only one DFA state reachable by a single label from DFA state d so we must
* find and merge all NFA states reachable via label. Return a new
* LexerState that has all of those NFA states.
*/
public LexerState reach(LexerState d, IntervalSet label) {
//System.out.println("reach "+label.toString(g)+" from "+d.stateNumber);
LexerState labelTarget = dfa.newLexerState();
for (NFAConfig c : d.nfaConfigs) {
NFAState s = c.state;
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) { // for each transition
Transition t = s.transition(i);
// found a transition with label; does it collide with label?
if ( !t.isEpsilon() && !t.label().and(label).isNil() ) {
//System.out.println("found edge with "+label.toString(g)+" from NFA state "+s);
// add NFA target to (potentially) new DFA state
labelTarget.addNFAConfig(
new NFAConfig(c, t.target, c.semanticContext));
}
}
}
return labelTarget;
}
/** For all NFA states in d, compute the epsilon closure; that is, find
* all NFA states reachable from the NFA states in d purely via epsilon
* transitions.
*/
public void closure(LexerState d, boolean collectPredicates) {
if ( debug ) {
System.out.println("closure("+d+")");
}
List<NFAConfig> configs = new ArrayList<NFAConfig>();
configs.addAll(d.nfaConfigs.elements()); // dup initial list; avoid walk/update issue
for (NFAConfig c : configs) {
closure(d, c.state, c.alt, c.context, c.semanticContext, collectPredicates); // update d.nfaStates
}
closureBusy.clear();
if ( debug ) {
System.out.println("after closure("+d+")");
}
//System.out.println("after closure d="+d);
}
// TODO: make pass NFAConfig like other DFA
public void closure(LexerState d, NFAState s, int ruleIndex, NFAContext context,
SemanticContext semanticContext, boolean collectPredicates) {
NFAConfig proposedNFAConfig =
new NFAConfig(s, ruleIndex, context, semanticContext);
if ( closureBusy.contains(proposedNFAConfig) ) return;
closureBusy.add(proposedNFAConfig);
// s itself is always in closure
d.nfaConfigs.add(proposedNFAConfig);
if ( s instanceof RuleStopState ) {
// TODO: chase FOLLOW links if recursive
if ( !context.isEmpty() ) {
closure(d, context.returnState, ruleIndex, context.parent, semanticContext, collectPredicates);
// do nothing if context not empty and already added to nfaStates
}
else {
d.isAcceptState = true;
}
}
else {
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( t instanceof RuleTransition ) {
// simulate an r=0 recursion limited conversion by avoiding
// any recursive call. It approximates recursive lexer
// rules with loops. Later we can try rule for real.
if ( !context.contains(((RuleTransition)t).followState) ) {
NFAContext newContext =
new NFAContext(context, ((RuleTransition)t).followState);
closure(d, t.target, ruleIndex, newContext, semanticContext, collectPredicates);
}
}
else if ( t instanceof ActionTransition ) {
collectPredicates = false; // can't see past actions
closure(d, t.target, ruleIndex, context, semanticContext, collectPredicates);
}
else if ( t instanceof PredicateTransition ) {
SemanticContext labelContext = ((PredicateTransition)t).semanticContext;
SemanticContext newSemanticContext = semanticContext;
if ( collectPredicates ) {
// AND the previous semantic context with new pred
//System.out.println("&"+labelContext+" enclosingRule="+c.state.rule);
newSemanticContext =
SemanticContext.and(semanticContext, labelContext);
}
closure(d, t.target, ruleIndex, context, newSemanticContext, collectPredicates);
}
else if ( t.isEpsilon() ) {
closure(d, t.target, ruleIndex, context, semanticContext, collectPredicates);
}
}
}
}
// void ruleStopStateClosure(LexerState d, NFAState s) {
// //System.out.println("FOLLOW of "+s+" context="+context);
// // follow all static FOLLOW links
// int n = s.getNumberOfTransitions();
// for (int i=0; i<n; i++) {
// Transition t = s.transition(i);
// if ( !(t instanceof EpsilonTransition) ) continue; // ignore EOF transitions
// if ( !d.nfaStates.contains(t.target) ) closure(d, t.target);
// }
// return;
// }
}

365
tool/src/org/antlr/v4/analysis/LinearApproximator.java
View File

@ -1,365 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.runtime.Token;
import org.antlr.v4.automata.*;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.tool.Grammar;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/** From NFA, return a linear approximate DFA if deterministic at
* a particular depth less than a max. Compute all depths at once rather
* than, like v2, trying more and more lookahead.
*
* No side effects outside class.
*/
public class LinearApproximator {
public int MAX_LINEAR_APPROXIMATE_DEPTH = 2;
Grammar g;
int decision;
int max_k = MAX_LINEAR_APPROXIMATE_DEPTH;
/** Used during LOOK to detect computation cycles. E.g., ()* causes
* infinite loop without it. If we get to same state with same k
* and same context, must be infinite loop. Analogous to
* closureBusy in NFA to DFA conversion.
*/
Set<LookaheadNFAConfig> lookBusy = new HashSet<LookaheadNFAConfig>();
/** The lookahead associated with an alternative, 1..k. A WORK ARRAY. */
IntervalSet[] look;
/** Our goal is to produce a DFA that looks like we created the
* usual way through subset construction. To look the same, we
* have to store a set of NFA configurations within each DFA state.
*
* A WORK ARRAY. Stores the NFA configurations for each lookahead
* depth, 1..k.
*/
OrderedHashSet<NFAConfig>[] configs;
/** Records state of a LOOK operation; used just for lookahead busy checks */
static class LookaheadNFAConfig {
public NFAState s;
public int k;
public NFAContext context;
public LookaheadNFAConfig(NFAState s, int k, NFAContext context) {
this.s = s;
this.k = k;
this.context = context;
}
public int hashCode() { return s.stateNumber+k; }
public boolean equals(Object obj) {
LookaheadNFAConfig ac = (LookaheadNFAConfig)obj;
return this.s == ac.s &&
this.k == ac.k &&
this.context.equals(ac.context);
}
}
public LinearApproximator(Grammar g, int decision) {
this.g = g;
this.decision = decision;
}
public LinearApproximator(Grammar g, int decision, int k) {
this(g, decision);
max_k = k;
}
public DFA createDFA(DecisionState s) {
List<IntervalSet[]> altLook = new ArrayList<IntervalSet[]>();
List<OrderedHashSet[]> altConfigs = new ArrayList<OrderedHashSet[]>();
altLook.add(null); // alt 0 invalid
altConfigs.add(null);
look = new IntervalSet[max_k+1];
configs = (OrderedHashSet<NFAConfig>[])Array.newInstance(OrderedHashSet.class, max_k+1);
// COLLECT LOOKAHEAD 1..k
for (int i=0; i<s.getNumberOfTransitions(); i++) {
Transition t = s.transition(i);
LOOK(t.target, MAX_LINEAR_APPROXIMATE_DEPTH);
altLook.add(look.clone());
altConfigs.add(configs.clone());
// for (int k=1; k<=MAX_LINEAR_APPROXIMATE_DEPTH; k++) {
// System.out.println(s.rule.name+"["+(i+1)+"]["+k+"]="+look[k].toString(g));
// System.out.println("configs["+(i+1)+"]["+k+"]="+ configs[k].toString());
// }
}
// FIND MIN DISJOINT k
int k = disjoint(altLook);
if ( k==0 ) return null;
System.out.println("disjoint at k="+k);
// BUILD DFA
return createApproximateDFA(altLook, altConfigs, k);
}
/** Return lookahead depth at which lookahead sets are disjoint or return 0 */
int disjoint(List<IntervalSet[]> altLook) {
int k = 1;
while ( k<=MAX_LINEAR_APPROXIMATE_DEPTH ) {
boolean collision = false;
IntervalSet combined = new IntervalSet();
for (int a=1; a<altLook.size(); a++) {
IntervalSet look = altLook.get(a)[k];
if ( !look.and(combined).isNil() ) {
System.out.println("alt "+a+" not disjoint with "+combined+"; look = "+look);
collision = true;
break;
}
combined.addAll(look);
}
if ( !collision ) return k;
k++;
}
return 0;
}
DFA createApproximateDFA(List<IntervalSet[]> altLook,
List<OrderedHashSet[]> altConfigs,
int depth)
{
int nAlts = altLook.size() - 1;
DFA dfa = new DFA(g, nAlts);
DFAState start = new DFAState(dfa);
dfa.startState = start;
dfa.decision = decision;
dfa.addState(start);
for (int a=1; a<=nAlts; a++) {
DFAState d = start;
IntervalSet[] look = altLook.get(a);
for (int k=1; k<=depth; k++) {
DFAState t = new DFAState(dfa);
t.nfaConfigs = altConfigs.get(a)[k];
dfa.addState(t);
if ( k==depth ) dfa.addAcceptState(a, t);
Edge e = new Edge(t, look[k]);
d.addEdge(e);
d = t;
}
}
return dfa;
}
/** From linear approximate LL(1) DFA, get lookahead per alt; 1..n */
public static IntervalSet[] getLL1LookaheadSets(DFA dfa) {
IntervalSet[] look = new IntervalSet[dfa.nAlts+1];
DFAState s0 = dfa.startState;
for (int a=1; a<=dfa.nAlts; a++) {
look[a] = s0.edges.get(a-1).label;
}
return look;
}
/** From an NFA state, s, find the set of all labels reachable from s at
* depth k.
*/
public IntervalSet[] LOOK(NFAState s, int k) {
System.out.println("LOOK("+s.stateNumber+", "+k+")");
lookBusy.clear();
for (int i=1; i<=max_k; i++) { // init / reset work arrays
look[i] = new IntervalSet();
configs[i] = new OrderedHashSet<NFAConfig>();
}
_LOOK(s, k, NFAContext.EMPTY());
return look;
}
void _LOOK(NFAState s, int k, NFAContext context) {
//System.out.println("_LOOK("+s.stateNumber+", "+k+", ctx="+context);
LookaheadNFAConfig ac = new LookaheadNFAConfig(s,k,context);
if ( lookBusy.contains(ac) ) return;
lookBusy.add(ac);
if ( s instanceof RuleStopState && !context.isEmpty() ) {
_LOOK(context.returnState, k, context.parent);
return;
}
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( t instanceof RuleTransition ) {
NFAContext newContext =
new NFAContext(context, ((RuleTransition)t).followState);
_LOOK(t.target, k, newContext);
}
else if ( t.isEpsilon() ) {
_LOOK(t.target, k, context);
}
else {
System.out.println("adding "+ t.label().toString(g) +" @ i="+(max_k-k+1));
look[max_k-k+1].addAll( t.label() );
NFAConfig c = new NFAConfig(t.target, 0, context,
SemanticContext.EMPTY_SEMANTIC_CONTEXT);
configs[max_k-k+1].add(c);
if ( k>1 ) _LOOK(t.target, k-1, context);
}
}
}
/* A bit set used for prediction contains all possible tokens
that can predict a particular alternative or set of alternatives.
Bit sets used for error recovery and expecting, however, are incomplete.
They only contain tokens extracted from the current rule. They don't include
any tokens from rules that invoke it (when the lookahead computation
reaches the end of the rule). Instead, the dynamic follow is used
because it contains the exact set of tokens that can follow an
invocation instead of all possible. It's the true expected set
of tokens at runtime. To indicate that a bit set is incomplete,
we include EOR (end of rule) token type. If we reach end of
a start rule, include EOF.
See BaseRecognizer.computeErrorRecoverySet() and friends for more
information on combining run-time bit sets.
*/
/** Compute set of tokens that we can reach from s, but don't leave rule
* to compute global, context-free FOLLOW. Used for error handling
* after rule invocation and match tokens. Also used in exceptions
* to show what we were expecting.
*/
public IntervalSet FIRST(NFAState s) {
//System.out.println("FIRST("+s.stateNumber+")");
lookBusy.clear();
IntervalSet fset = new IntervalSet();
_FIRST(s, NFAContext.EMPTY(), fset);
return fset;
}
void _FIRST(NFAState s, NFAContext context, IntervalSet fset) {
//System.out.println("_FIRST("+s.stateNumber+", "+k+", ctx="+context);
LookaheadNFAConfig ac = new LookaheadNFAConfig(s,1,context);
if ( lookBusy.contains(ac) ) return;
lookBusy.add(ac);
if ( s instanceof RuleStopState ) {
if ( !context.isEmpty() ) _FIRST(context.returnState, context.parent, fset);
else fset.add(Token.EOR_TOKEN_TYPE); // hit end of rule
return;
}
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = s.transition(i);
if ( t instanceof RuleTransition ) {
NFAContext newContext =
new NFAContext(context, ((RuleTransition)t).followState);
_FIRST(t.target, newContext, fset);
}
else if ( t.isEpsilon() ) {
_FIRST(t.target, context, fset);
}
else {
fset.addAll( t.label() );
}
}
}
// LookaheadSet ____LOOK(NFAState s, int k, NFAState context) {
// //System.out.println("_LOOK("+s.stateNumber+", "+k+", "+context+")");
//
//// if ( lookBusy.contains(s) ) {
//// // return a copy of an empty set; we may modify set inline
//// return new LookaheadSet();
//// }
//// lookBusy.add(s);
//
// if ( s instanceof RuleStopState && context!=null ) {
// return LookaheadSet.missingDepth(k);
// }
//
// LookaheadSet tset = new LookaheadSet();
// int n = s.getNumberOfTransitions();
// for (int i=0; i<n; i++) {
// Transition t = s.transition(i);
// LookaheadSet look = null;
// if ( t instanceof RuleTransition ) {
// look = _LOOK(t.target, k, ((RuleTransition)t).followState);
// if ( look.eorDepths!=null ) {
// for (Integer _k : look.eorDepths.toList() ) {
// look.combine( _LOOK(((RuleTransition)t).followState, _k, context) );
// }
// look.eorDepths = null;
// }
// }
// else if ( t.isEpsilon() ) look = _LOOK(t.target, k, context);
// else if ( k==1 ) look = new LookaheadSet( t.label() );
// else if ( k>1 ) look = _LOOK(t.target, k-1, context);
// tset.combine( look );
// }
//
// //lookBusy.remove(s);
//
// return tset;
// }
// public LookaheadSet FOLLOW(Rule r) {
// LookaheadSet f = FOLLOWCache.get(r);
// if ( f!=null ) return f;
// f = _FIRST(r.stopState, true);
// FOLLOWCache.put(r, f);
// return f;
// }
// public LinearApproximator(DFA dfa) {
// this.dfa = dfa;
// // make room for max lookahead of num states [1..nAlts][1..numstates]
// look = new LookaheadSet[dfa.nAlts+1][dfa.stateSet.size()+1];
// max = new int[dfa.nAlts+1];
// fillLookaheadSets(dfa.startState, 1);
// for (IntSet[] l : look) System.out.println(Arrays.toString(l));
// System.out.println("max k="+maxDepth);
// }
//
// public LookaheadSet[] getLookaheadSets() {
// LookaheadSet[] altLook = new LookaheadSet[dfa.nAlts+1];
// for (int a=1; a<=dfa.nAlts; a++) altLook[a] = look[a][max[a]];
// return altLook;
// }
// public int getLookaheadDepth(int alt) { return max[alt]; }
//
// public boolean isDeterministic() {
// // lookahead at smallest lookahead depth for alts i and j must be disjoint
// for (int i=1; i<=dfa.nAlts; i++) {
// for (int j=i+1; j<=dfa.nAlts; j++) {
// int k = Math.min(max[i], max[j]);
// // two alts aren't disjoint at depth k. nondeterministic; bolt.
// if ( !look[i][k].and(look[j][k]).isNil() ) return false;
// }
// }
// return true;
// }
//
//
// void fillLookaheadSets(DFAState d, int k) {
// for (Edge e : d.edges) {
// // if ( e instanceof PredicateEdge ) continue; NO PREDS IF NOTAMBIG
// Set<Integer> alts = e.target.getAltSet();
// for (int a : alts) {
// if ( look[a][k]==null ) look[a][k] = new LookaheadSet();
// max[a] = Math.max(max[a], k);
// maxDepth = Math.max(maxDepth, k);
// look[a][k].addAll(e.label);
// }
// fillLookaheadSets(e.target, k+1);
// }
// }
}

145
tool/src/org/antlr/v4/analysis/MachineProbe.java
View File

@ -1,145 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.runtime.Token;
import org.antlr.v4.automata.*;
import org.antlr.v4.misc.IntSet;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.tool.Grammar;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class MachineProbe {
DFA dfa;
public MachineProbe(DFA dfa) { this.dfa = dfa; }
List<DFAState> getAnyDFAPathToTarget(DFAState targetState) {
Set<DFAState> visited = new HashSet<DFAState>();
return getAnyDFAPathToTarget(dfa.startState, targetState, visited);
}
public List<DFAState> getAnyDFAPathToTarget(DFAState startState,
DFAState targetState,
Set<DFAState> visited)
{
List<DFAState> dfaStates = new ArrayList<DFAState>();
visited.add(startState);
if ( startState.equals(targetState) ) {
dfaStates.add(targetState);
return dfaStates;
}
for (Edge e : startState.edges) { // walk edges looking for valid path
if ( !visited.contains(e.target) ) {
List<DFAState> path =
getAnyDFAPathToTarget(e.target, targetState, visited);
if ( path!=null ) { // found path, we're done
dfaStates.add(startState);
dfaStates.addAll(path);
return dfaStates;
}
}
}
return null;
}
/** Return a list of edge labels from start state to targetState. */
public List<IntSet> getEdgeLabels(DFAState targetState) {
List<DFAState> dfaStates = getAnyDFAPathToTarget(targetState);
List<IntSet> labels = new ArrayList<IntSet>();
if ( dfaStates==null ) return labels;
for (int i=0; i<dfaStates.size()-1; i++) {
DFAState d = dfaStates.get(i);
DFAState nextState = dfaStates.get(i + 1);
// walk looking for edge whose target is next dfa state
for (Edge e : d.edges) {
if ( e.target.stateNumber == nextState.stateNumber ) {
labels.add(e.label);
}
}
}
return labels;
}
/** Given List<IntSet>, return a String with a useful representation
* of the associated input string. One could show something different
* for lexers and parsers, for example.
*/
public String getInputSequenceDisplay(Grammar g, List<IntSet> labels) {
List<String> tokens = new ArrayList<String>();
for (IntSet label : labels) tokens.add(label.toString(g));
return Utils.join(tokens.iterator(), " ");
}
/** Given an alternative associated with a DFA state, return the list
* of tokens (from grammar) associated with path through NFA following
* the labels sequence. The nfaStates gives the set of NFA states
* associated with alt that take us from start to stop. One of the
* NFA states in nfaStates[i] will have an edge intersecting with
* labels[i].
*/
public List<Token> getGrammarLocationsForInputSequence(List<Set<NFAState>> nfaStates,
List<IntSet> labels)
{
List<Token> tokens = new ArrayList<Token>();
for (int i=0; i<nfaStates.size()-1; i++) {
Set<NFAState> cur = nfaStates.get(i);
Set<NFAState> next = nfaStates.get(i + 1);
IntSet label = labels.get(i);
// find NFA state with edge whose label matches labels[i]
nfaConfigLoop:
for (NFAState p : cur) {
// walk p's transitions, looking for label
for (int j=0; j<p.getNumberOfTransitions(); j++) {
Transition t = p.transition(j);
if ( !t.isEpsilon() &&
!t.label().and(label).isNil() &&
next.contains(t.target) )
{
tokens.add(p.ast.token);
break nfaConfigLoop; // found path, move to next NFAState set
}
}
}
}
return tokens;
}
// /** Used to find paths through syntactically ambiguous DFA. If we've
// * seen statement number before, what did we learn?
// */
// protected Map<Integer, Integer> stateReachable;
//
// public Map<DFAState, Set<DFAState>> getReachSets(Collection<DFAState> targets) {
// Map<DFAState, Set<DFAState>> reaches = new HashMap<DFAState, Set<DFAState>>();
// // targets can reach themselves
// for (final DFAState d : targets) {
// reaches.put(d,new HashSet<DFAState>() {{add(d);}});
// }
//
// boolean changed = true;
// while ( changed ) {
// changed = false;
// for (DFAState d : dfa.states.values()) {
// if ( d.getNumberOfEdges()==0 ) continue;
// Set<DFAState> r = reaches.get(d);
// if ( r==null ) {
// r = new HashSet<DFAState>();
// reaches.put(d, r);
// }
// int before = r.size();
// // add all reaches from all edge targets
// for (Edge e : d.edges) {
// //if ( targets.contains(e.target) ) r.add(e.target);
// r.addAll( reaches.get(e.target) );
// }
// int after = r.size();
// if ( after>before) changed = true;
// }
// }
// return reaches;
// }
}

10
tool/src/org/antlr/v4/analysis/MultipleRecursiveAltsSignal.java
View File

@ -1,10 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.misc.IntSet;
public class MultipleRecursiveAltsSignal extends RuntimeException {
public IntSet recursiveAltSet;
public MultipleRecursiveAltsSignal(IntSet recursiveAltSet) {
this.recursiveAltSet = recursiveAltSet;
}
}

137
tool/src/org/antlr/v4/analysis/NFAConfig.java
View File

@ -1,137 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.NFAState;
/** An NFA state, predicted alt, and syntactic/semantic context.
* The syntactic context is a pointer into the rule invocation
* chain used to arrive at the state. The semantic context is
* the unordered set semantic predicates encountered before reaching
* an NFA state.
*/
public class NFAConfig {
/** The NFA state associated with this configuration */
public NFAState state;
/** What alt is predicted by this configuration */
public int alt;
/** Record the NFA state that invoked another rule's start state */
public NFAContext context;
/** The set of semantic predicates associated with this NFA
* configuration. The predicates were found on the way to
* the associated NFA state in this syntactic context.
*/
public SemanticContext semanticContext = SemanticContext.EMPTY_SEMANTIC_CONTEXT;
/** Indicate that this configuration has been resolved and no further
* DFA processing should occur with it. Essentially, this is used
* as an "ignore" bit so that upon a set of nondeterministic configurations
* such as (s|2) and (s|3), I can set (s|3) to resolved=true (and any
* other configuration associated with alt 3).
*/
public boolean resolved;
/** This bit is used to indicate a semantic predicate will be
* used to resolve the conflict. Method
* DFA.findNewDFAStatesAndAddDFATransitions will add edges for
* the predicates after it performs the reach operation. The
* nondeterminism resolver sets this when it finds a set of
* nondeterministic configurations (as it does for "resolved" field)
* that have enough predicates to resolve the conflit.
*/
boolean resolvedWithPredicate;
public NFAConfig(NFAState state,
int alt,
NFAContext context,
SemanticContext semanticContext)
{
this.state = state;
this.alt = alt;
this.context = context;
this.semanticContext = semanticContext;
}
public NFAConfig(NFAConfig c) {
this.state = c.state;
this.alt = c.alt;
this.context = c.context;
this.semanticContext = c.semanticContext;
}
public NFAConfig(NFAConfig c, NFAState state) {
this(c);
this.state = state;
}
public NFAConfig(NFAConfig c, NFAState state, NFAContext context) {
this(c);
this.state = state;
this.context = context;
}
public NFAConfig(NFAConfig c, NFAContext context) {
this(c);
this.context = context;
}
public NFAConfig(NFAConfig c, NFAState state, SemanticContext semanticContext) {
this(c);
this.state = state;
this.semanticContext = semanticContext;
}
/** An NFA configuration is equal to another if both have
* the same state, they predict the same alternative, and
* syntactic/semantic contexts are the same.
*/
public boolean equals(Object o) {
if ( o==null ) return false;
if ( this==o ) return true;
NFAConfig other = (NFAConfig)o;
return this.state==other.state &&
this.alt==other.alt &&
this.context.equals(other.context) &&
this.semanticContext.equals(other.semanticContext);
}
public int hashCode() {
if ( state==null || context==null ) {
System.out.println("eh?");
}
int h = state.stateNumber + alt + context.hashCode();
return h;
}
public String toString() {
return toString(true);
}
public String toString(boolean showAlt) {
StringBuffer buf = new StringBuffer();
buf.append(state);
if ( showAlt ) {
buf.append("|");
buf.append(alt);
}
if ( context!=null && !context.isEmpty() ) {
buf.append("|");
buf.append(context);
}
if ( semanticContext!=null && semanticContext!=SemanticContext.EMPTY_SEMANTIC_CONTEXT ) {
buf.append("|");
buf.append(semanticContext);
}
if ( resolved ) {
buf.append("|resolved");
}
if (resolvedWithPredicate) {
buf.append("|resolveWithPredicate");
}
if ( context.approximated ) {
buf.append("|approx");
}
return buf.toString();
}
}

252
tool/src/org/antlr/v4/analysis/NFAContext.java
View File

@ -1,252 +0,0 @@
/*
[The "BSD license"]
Copyright (c) 2005-2009 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.analysis;
import org.antlr.v4.automata.NFAState;
/** A tree node for tracking the call chains for NFAs that invoke
* other NFAs. These trees only have to point upwards to their parents
* so we can walk back up the tree (i.e., pop stuff off the stack). We
* never walk from stack down down through the children.
*
* Each alt predicted in a decision has its own context tree,
* representing all possible return nodes. The initial stack has
* EOF ("$") in it. So, for m alternative productions, the lookahead
* DFA will have m NFAContext trees.
*
* To "push" a new context, just do "new NFAContext(context-parent, state)"
* which will add itself to the parent. The root is NFAContext(null, null).
*
* The complete context for an NFA configuration is the set of invoking states
* on the path from this node thru the parent pointers to the root.
*/
public class NFAContext {
/** This is similar to Bermudez's m constant in his LAR(m) where
* he bounds the stack so your state sets don't explode. The main difference
* is that I bound only recursion on the stack, not the simple stack size.
* This looser constraint will let the conversion roam further to find
* lookahead to resolve a decision.
*
* We restrict the size of an NFA configuration to be finite because a
* stack component may mention the same NFA invocation state at
* most m times. Hence, the number of DFA states will not grow forever.
*
* m=0 implies you can make as many calls as you want--you just
* can't ever visit a state that is on your rule invocation stack.
* I.e., you cannot ever recurse.
* m=1 implies you are able to recurse once (i.e., call a rule twice
* from the same place).
* ...
*
* This tracks recursion to a rule specific to an invocation site!
* It does not detect multiple calls to a rule from different rule
* invocation states. We are guaranteed to terminate because the
* stack can only grow as big as the number of NFA states * m.
*
* I noticed that the Java grammar didn't work with m=1 in ANTLR v3,
* but it did work with m=4. Let's set to 4. Recursion is sometimes
* needed to resolve some fixed lookahead decisions.
*/
public static int MAX_RECURSION_DEPTH_PER_NFA_CONFIG_STACK = 1;
public NFAContext parent;
/** The NFA state following state that invoked another rule's start state
* is recorded on the rule invocation context stack.
*/
public NFAState returnState;
/** Indicates this config led to recursive closure request. Everything
* derived from here is approximation.
*/
public boolean approximated;
/** Computing the hashCode is very expensive and closureBusy()
* uses it to track when it's seen a state|ctx before to avoid
* infinite loops. As we add new contexts, record the hash code
* as this.invokingState + parent.cachedHashCode. Avoids walking
* up the tree for every hashCode(). Note that this caching works
* because a context is a monotonically growing tree of context nodes
* and nothing on the stack is ever modified...ctx just grows
* or shrinks.
*/
protected int cachedHashCode;
public NFAContext(NFAContext parent, NFAState returnState) {
this.parent = parent;
this.returnState = returnState;
if ( returnState !=null ) {
this.cachedHashCode = returnState.stateNumber;
}
if ( parent!=null ) {
this.cachedHashCode += parent.cachedHashCode;
}
}
/** Dup context so we can turn on approximated or whatever */
public NFAContext(NFAContext proto) {
this.parent = proto.parent;
this.returnState = proto.returnState;
this.cachedHashCode = proto.cachedHashCode;
this.approximated = proto.approximated;
}
public static NFAContext EMPTY() { return new NFAContext(null, null); }
/** Is s anywhere in the context? */
public boolean contains(NFAState s) {
NFAContext sp = this;
while ( sp!=null ) {
if ( sp.returnState == s ) return true;
sp = sp.parent;
}
return false;
}
/** Two contexts are equals() if both have
* same call stack; walk upwards to the root.
* Recall that the root sentinel node has no parent.
* Note that you may be comparing contextsv in different alt trees.
*/
public boolean equals(Object o) {
NFAContext other = ((NFAContext)o);
if ( this.cachedHashCode != other.cachedHashCode ) {
return false; // can't be same if hash is different
}
if ( this==other ) return true;
// System.out.println("comparing "+this+" with "+other);
NFAContext sp = this;
while ( sp.parent!=null && other.parent!=null ) {
if ( sp.returnState != other.returnState) return false;
sp = sp.parent;
other = other.parent;
}
if ( !(sp.parent==null && other.parent==null) ) {
return false; // both pointers must be at their roots after walk
}
return true;
}
/** [$] suffix any context
* [21 $] suffix [21 12 $]
* [21 12 $] suffix [21 $]
* [21 18 $] suffix [21 18 12 9 $]
* [21 18 12 9 $] suffix [21 18 $]
* [21 12 $] not suffix [21 9 $]
*
* Example "[21 $] suffix [21 12 $]" means: rule r invoked current rule
* from state 21. Rule s invoked rule r from state 12 which then invoked
* current rule also via state 21. While the context prior to state 21
* is different, the fact that both contexts emanate from state 21 implies
* that they are now going to track perfectly together. Once they
* converged on state 21, there is no way they can separate. In other
* words, the prior stack state is not consulted when computing where to
* go in the closure operation. beta $ and beta alpha $ are considered the same stack.
* If beta is popped off then $ and alpha$ remain; there is now an empty and
* nonempty context comparison. So, if one stack is a suffix of
* another, then it will still degenerate to the simple empty / nonempty stack
* comparison case.
*/
protected boolean suffix(NFAContext other) {
NFAContext sp = this;
// if one of the contexts is empty, it never enters loop and returns true
while ( sp.parent!=null && other.parent!=null ) {
if ( sp.returnState != other.returnState) {
return false;
}
sp = sp.parent;
other = other.parent;
}
//System.out.println("suffix");
return true;
}
/** Given an NFA state number, how many times does it appear on stack?
* The NFA-to-DFA conversion pushes "return" states as it does
* rule invocations. The NFA state number must be a rule return state
* (following state from invocation state).
*/
public int occurrences(int state) {
NFAContext sp = this;
int n = 0; // track recursive invocations of target from this state
//System.out.println("this.context is "+sp);
while ( sp.parent!=null ) {
if ( sp.returnState.stateNumber == state ) {
n++;
}
sp = sp.parent;
}
return n;
}
public int hashCode() {
return cachedHashCode; // works with tests; don't recompute.
// int h = 0;
// NFAContext sp = this;
// while ( sp.parent!=null ) {
// h += sp.returnState.stateNumber;
// sp = sp.parent;
// }
// return h;
}
/** How many rule invocations in this context? I.e., how many
* elements in stack (path to root, not including root placeholder)?
*/
public int depth() {
int n = 0;
NFAContext sp = this;
while ( !sp.isEmpty() ) {
n++;
sp = sp.parent;
}
return n;
}
/** A context is empty if there is no parent; meaning nobody pushed
* anything on the call stack.
*/
public boolean isEmpty() {
return parent==null;
}
public String toString() {
StringBuffer buf = new StringBuffer();
NFAContext sp = this;
buf.append("[");
while ( sp.parent!=null ) {
buf.append(sp.returnState.stateNumber);
buf.append(" ");
sp = sp.parent;
}
buf.append("$]");
return buf.toString();
}
}

296
tool/src/org/antlr/v4/analysis/PredicateResolver.java
View File

@ -1,296 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.runtime.Token;
import org.antlr.v4.automata.BasicState;
import org.antlr.v4.automata.DFAState;
import org.antlr.v4.automata.Label;
import org.antlr.v4.automata.NFAState;
import org.antlr.v4.misc.BitSet;
import org.antlr.v4.tool.ErrorManager;
import java.util.*;
/** */
public class PredicateResolver {
public Map<DFAState, List<Integer>> statesWithIncompletelyCoveredAlts = new HashMap<DFAState, List<Integer>>();
/** See if a set of nondeterministic alternatives can be disambiguated
* with the semantic predicate contexts of the alternatives.
*
* Without semantic predicates, syntactic conflicts are resolved
* by simply choosing the first viable alternative. In the
* presence of semantic predicates, you can resolve the issue by
* evaluating boolean expressions at run time. During analysis,
* this amounts to suppressing grammar error messages to the
* developer. NFA configurations are always marked as "to be
* resolved with predicates" so that DFA.reach() will know to ignore
* these configurations and add predicate transitions to the DFA
* after adding edge labels.
*
* During analysis, we can simply make sure that for n
* ambiguously predicted alternatives there are at least n-1
* unique predicate sets. The nth alternative can be predicted
* with "not" the "or" of all other predicates. NFA configurations without
* predicates are assumed to have the default predicate of
* "true" from a user point of view. When true is combined via || with
* another predicate, the predicate is a tautology and must be removed
* from consideration for disambiguation:
*
* a : b | B ; // hoisting p1||true out of rule b, yields no predicate
* b : {p1}? B | B ;
*
* This is done down in getPredicatesPerNonDeterministicAlt().
*/
public boolean tryToResolveWithSemanticPredicates(DFAState d,
Set<Integer> ambiguousAlts)
{
Map<Integer, SemanticContext> altToPredMap =
getPredicatesPerAmbiguousAlt(d, ambiguousAlts);
if ( altToPredMap.size()==0 ) return false;
//System.out.println("nondeterministic alts with predicates: "+altToPredMap);
if ( ambiguousAlts.size()-altToPredMap.size()>1 ) {
// too few predicates to resolve; just return.
// We caught/tracked incompletely covered preds in getPredicatesPerNonDeterministicAlt
return false;
}
// Handle case where 1 predicate is missing
// Case 1. Semantic predicates
// If the missing pred is on nth alt, !(union of other preds)==true
// so we can avoid that computation. If naked alt is ith, then must
// test it with !(union) since semantic predicated alts are order
// independent
// Case 2: Syntactic predicates
// The naked alt is always assumed to be true as the order of
// alts is the order of precedence. The naked alt will be a tautology
// anyway as it's !(union of other preds). This implies
// that there is no such thing as noviable alt for synpred edges
// emanating from a DFA state.
if ( altToPredMap.size()==ambiguousAlts.size()-1 ) {
// if there are n-1 predicates for n nondeterministic alts, can fix
BitSet ndSet = BitSet.of(ambiguousAlts);
BitSet predSet = BitSet.of(altToPredMap);
int nakedAlt = ndSet.subtract(predSet).getSingleElement();
SemanticContext nakedAltPred = null;
if ( nakedAlt == Collections.max(ambiguousAlts) ) {
// the naked alt is the last nondet alt and will be the default clause
nakedAltPred = new SemanticContext.TruePredicate();
}
else {
// pretend naked alternative is covered with !(union other preds)
// unless it's a synpred since those have precedence same
// as alt order
SemanticContext unionOfPredicatesFromAllAlts =
getUnionOfPredicates(altToPredMap);
//System.out.println("all predicates "+unionOfPredicatesFromAllAlts);
if ( unionOfPredicatesFromAllAlts.isSyntacticPredicate() ) {
nakedAltPred = new SemanticContext.TruePredicate();
}
else {
nakedAltPred =
SemanticContext.not(unionOfPredicatesFromAllAlts);
}
}
//System.out.println("covering naked alt="+nakedAlt+" with "+nakedAltPred);
altToPredMap.put(nakedAlt, nakedAltPred);
// set all config with alt=nakedAlt to have the computed predicate
int numConfigs = d.nfaConfigs.size();
for (int i = 0; i < numConfigs; i++) {
NFAConfig configuration = (NFAConfig)d.nfaConfigs.get(i);
if ( configuration.alt == nakedAlt ) {
configuration.semanticContext = nakedAltPred;
}
}
}
if ( altToPredMap.size()==ambiguousAlts.size() ) {
// RESOLVE CONFLICT by picking one NFA configuration for each alt
// and setting its resolvedWithPredicate flag
// First, prevent a recursion warning on this state due to
// pred resolution
// if ( d.abortedDueToRecursionOverflow ) {
// d.dfa.probe.removeRecursiveOverflowState(d);
// }
for (NFAConfig c : d.nfaConfigs) {
SemanticContext semCtx = altToPredMap.get(c.alt);
if ( semCtx!=null ) {
// resolve (first found) with pred
// and remove alt from problem list
c.resolvedWithPredicate = true;
c.semanticContext = semCtx; // reset to combined
altToPredMap.remove(c.alt);
// notify grammar that we've used the preds contained in semCtx
// if ( semCtx.isSyntacticPredicate() ) {
// dfa.nfa.grammar.synPredUsedInDFA(dfa, semCtx);
// }
}
else if ( ambiguousAlts.contains(c.alt) ) {
// resolve all other configurations for nondeterministic alts
// for which there is no predicate context by turning it off
c.resolved = true;
}
}
return true;
}
return false; // couldn't fix the problem with predicates
}
/** Return a mapping from nondeterministc alt to combined list of predicates.
* If both (s|i|semCtx1) and (t|i|semCtx2) exist, then the proper predicate
* for alt i is semCtx1||semCtx2 because you have arrived at this single
* DFA state via two NFA paths, both of which have semantic predicates.
* We ignore deterministic alts because syntax alone is sufficient
* to predict those. Do not include their predicates.
*
* Alts with no predicate are assumed to have {true}? pred.
*
* When combining via || with "true", all predicates are removed from
* consideration since the expression will always be true and hence
* not tell us how to resolve anything. So, if any NFA configuration
* in this DFA state does not have a semantic context, the alt cannot
* be resolved with a predicate.
*
* If nonnull, incidentEdgeLabel tells us what NFA transition label
* we did a reach on to compute state d. d may have insufficient
* preds, so we really want this for the error message.
*/
public Map<Integer, SemanticContext> getPredicatesPerAmbiguousAlt(
DFAState d,
Set<Integer> ambiguousAlts)
{
// map alt to combined SemanticContext
Map<Integer, SemanticContext> altToPredicateContextMap =
new HashMap<Integer, SemanticContext>();
Map<Integer, Set<SemanticContext>> altToSetOfContextsMap =
new HashMap<Integer, Set<SemanticContext>>();
for (int alt : ambiguousAlts) {
altToSetOfContextsMap.put(alt, new HashSet<SemanticContext>());
}
// Create a unique set of predicates from configs
// Also, track the alts with at least one uncovered configuration
// (one w/o a predicate); tracks tautologies like p1||true
Set<Integer> ambigAltsWithUncoveredConfiguration = new HashSet<Integer>();
for (NFAConfig c : d.nfaConfigs) {
if ( ambiguousAlts.contains(c.alt) ) {
if ( c.semanticContext != SemanticContext.EMPTY_SEMANTIC_CONTEXT ) {
Set<SemanticContext> predSet = altToSetOfContextsMap.get(c.alt);
predSet.add(c.semanticContext);
}
else {
// if no predicate, but it's part of nondeterministic alt
// then at least one path exists not covered by a predicate.
// must remove predicate for this alt; track incomplete alts
ambigAltsWithUncoveredConfiguration.add(c.alt);
}
}
}
// Walk semantic contexts for nondet alts, ORing them together
// Also, track the list of incompletely covered alts: those alts
// with at least 1 predicate and at least one configuration w/o a
// predicate. We want this in order to report to the decision probe.
List<Integer> incompletelyCoveredAlts = new ArrayList<Integer>();
for (int alt : ambiguousAlts) {
Set<SemanticContext> contextsForThisAlt = altToSetOfContextsMap.get(alt);
if ( ambigAltsWithUncoveredConfiguration.contains(alt) ) { // >= 1 config has no ctx
if ( contextsForThisAlt.size()>0 ) { // && at least one pred
incompletelyCoveredAlts.add(alt); // this alt incompleted covered
}
continue; // don't include; at least 1 config has no ctx
}
SemanticContext combinedContext = null;
for (Iterator itrSet = contextsForThisAlt.iterator(); itrSet.hasNext();) {
SemanticContext ctx = (SemanticContext) itrSet.next();
combinedContext =
SemanticContext.or(combinedContext,ctx);
}
altToPredicateContextMap.put(alt, combinedContext);
}
if ( incompletelyCoveredAlts.size()>0 ) {
// track these troublesome states later for reporting.
statesWithIncompletelyCoveredAlts.put(d, incompletelyCoveredAlts);
}
return altToPredicateContextMap;
}
public static Map<Integer, Set<Token>> getInsufficientlyPredicatedLocations(DFAState d,
List<Integer> incompletelyCoveredAlts)
{
Map<Integer, Set<Token>> altToLocationsReachableWithoutPredicate = new HashMap<Integer, Set<Token>>();
for (NFAConfig c : d.nfaConfigs) {
if ( incompletelyCoveredAlts.contains(c.alt) &&
c.semanticContext == SemanticContext.EMPTY_SEMANTIC_CONTEXT )
{
NFAState s = c.state;
/*
System.out.print("nondet config w/o context "+configuration+
" incident "+(s.incidentEdgeLabel!=null?s.incidentEdgeLabel.toString(dfa.nfa.grammar):null));
if ( s.associatedASTNode!=null ) {
System.out.print(" token="+s.associatedASTNode.token);
}
else System.out.println();
*/
// We want to report getting to an NFA state with an
// incoming label, unless it's EOF, w/o a predicate.
if ( s instanceof BasicState &&
((BasicState)s).incidentTransition!=null &&
!((BasicState)s).incidentTransition.label().member(Label.EOF) )
{
if ( s.ast==null || s.ast.token==null ) {
ErrorManager.internalError("no AST/token for nonepsilon target w/o predicate");
}
else {
Set<Token> locations = altToLocationsReachableWithoutPredicate.get(c.alt);
if ( locations==null ) {
locations = new HashSet<Token>();
altToLocationsReachableWithoutPredicate.put(c.alt, locations);
}
locations.add(s.ast.token);
}
}
}
}
// get new map sorted by alt nums
Map<Integer, Set<Token>> sortedMap = new LinkedHashMap<Integer, Set<Token>>();
List<Integer> alts = new ArrayList<Integer>();
alts.addAll(altToLocationsReachableWithoutPredicate.keySet());
Collections.sort(alts);
for (int alt : alts) {
sortedMap.put(alt, altToLocationsReachableWithoutPredicate.get(alt));
}
return sortedMap;
}
/** OR together all predicates from the alts. Note that the predicate
* for an alt could itself be a combination of predicates.
*/
public static SemanticContext getUnionOfPredicates(Map altToPredMap) {
Iterator iter;
SemanticContext unionOfPredicatesFromAllAlts = null;
iter = altToPredMap.values().iterator();
while ( iter.hasNext() ) {
SemanticContext semCtx = (SemanticContext)iter.next();
if ( unionOfPredicatesFromAllAlts==null ) {
unionOfPredicatesFromAllAlts = semCtx;
}
else {
unionOfPredicatesFromAllAlts =
SemanticContext.or(unionOfPredicatesFromAllAlts,semCtx);
}
}
return unionOfPredicatesFromAllAlts;
}
}

592
tool/src/org/antlr/v4/analysis/PredictionDFAFactory.java
View File

@ -1,592 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.runtime.Token;
import org.antlr.v4.automata.*;
import org.antlr.v4.misc.IntSet;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.util.*;
/** Code that embodies the NFA conversion to DFA. A new object is needed
* per DFA (also required for thread safety if multiple conversions
* launched).
*/
public class PredictionDFAFactory {
Grammar g;
DecisionState nfaStartState;
/** DFA we are creating */
DFA dfa;
/** A list of DFA states we still need to process during NFA conversion */
List<DFAState> work = new LinkedList<DFAState>();
/** Each alt in an NFA derived from a grammar must have a DFA state that
* predicts it lest the parser not know what to do. Nondeterminisms can
* lead to this situation (assuming no semantic predicates can resolve
* the problem) and when for some reason, I cannot compute the lookahead
* (which might arise from an error in the algorithm or from
* left-recursion etc...).
*/
public Set<Integer> unreachableAlts;
/** Tracks alts insufficiently covered.
* For example, p1||true gets reduced to true and so leaves
* whole alt uncovered. This maps alt num to the set of (Token)
* locations in grammar of uncovered elements.
*/
public Map<DFAState, List<Integer>> statesWithIncompletelyCoveredAlts = new HashMap<DFAState, List<Integer>>();
public boolean hasPredicateBlockedByAction = false;
/** Recursion is limited to a particular depth. Which state tripped it? */
public DFAState recursionOverflowState;
/** Which state found multiple recursive alts? */
public DFAState abortedDueToMultipleRecursiveAltsAt;
/** Are there any loops in this DFA? */
// public boolean cyclic = false;
/** Used to prevent the closure operation from looping to itself and
* hence looping forever. Sensitive to the NFA state, the alt, and
* the stack context.
*/
OrderedHashSet<NFAConfig> closureBusy;
public Resolver resolver;
public static boolean debug = false;
public PredictionDFAFactory(Grammar g, DecisionState nfaStartState) {
this.g = g;
this.nfaStartState = nfaStartState;
dfa = new DFA(g, nfaStartState);
dfa.converter = this;
resolver = new Resolver();
}
public DFA createDFA() {
closureBusy = new OrderedHashSet<NFAConfig>();
computeStartState();
dfa.addState(dfa.startState); // make sure dfa knows about this state
work.add(dfa.startState);
// while more DFA states to check, process them
while ( work.size()>0 ) {
DFAState d = work.get(0);
reach(d);
resolver.resolveDeadState(d);
work.remove(0); // we're done with this DFA state
}
unreachableAlts = getUnreachableAlts();
closureBusy = null; // wack all that memory used during closure
return dfa;
}
/** From this node, add a d--a-->t transition for all
* labels 'a' where t is a DFA node created
* from the set of NFA states reachable from any NFA
* configuration in DFA state d.
*/
void reach(DFAState d) {
OrderedHashSet<IntervalSet> labels = DFA.getReachableLabels(d);
for (IntervalSet label : labels) {
DFAState t = reach(d, label);
if ( debug ) {
System.out.println("DFA state after reach -" +
label.toString(g)+"->"+t);
}
// nothing was reached by label; we must have resolved
// all NFA configs in d, when added to work, that point at label
if ( t==null ) continue;
// if ( t.getUniqueAlt()==NFA.INVALID_ALT_NUMBER ) {
// // Only compute closure if a unique alt number is not known.
// // If a unique alternative is mentioned among all NFA
// // configurations then there is no possibility of needing to look
// // beyond this state; also no possibility of a nondeterminism.
// // This optimization May 22, 2006 just dropped -Xint time
// // for analysis of Java grammar from 11.5s to 2s! Wow.
// closure(t); // add any NFA states reachable via epsilon
// }
//try {
closure(t); // add any NFA states reachable via epsilon
//}
// catch (RecursionOverflowSignal ros) {
// recursionOverflowState = d;
// ErrorManager.recursionOverflow(g.fileName, d, ros.state, ros.altNum, ros.depth);
// }
// catch (MultipleRecursiveAltsSignal mras) {
// abortedDueToMultipleRecursiveAltsAt = d;
// ErrorManager.multipleRecursiveAlts(g.fileName, d, mras.recursiveAltSet);
// }
addTransition(d, label, t); // make d-label->t transition
}
// Add semantic predicate transitions if we resolved when added to work list
if ( d.resolvedWithPredicates ) addPredicateTransitions(d);
}
/** Add t if not in DFA yet, resolving nondet's and then make d-label->t */
void addTransition(DFAState d, IntervalSet label, DFAState t) {
DFAState existing = dfa.stateSet.get(t);
if ( existing != null ) { // seen before; point at old one
d.addEdge(new Edge(existing, label));
return;
}
// resolve any syntactic conflicts by choosing a single alt or
// by using semantic predicates if present.
resolver.resolveAmbiguities(t);
// If deterministic, don't add this state to work list; it's an accept state
// Just return as a valid DFA state
int alt = t.getUniquelyPredictedAlt();
if ( alt > 0 ) { // uniquely predicts an alt?
//System.out.println(t+" predicts "+alt);
// Define new stop state
dfa.addAcceptState(alt, t);
}
else {
// System.out.println("ADD "+t);
work.add(t); // unresolved, add to work list to continue NFA conversion
dfa.addState(t); // add state we've never seen before
}
d.addEdge(new Edge(t, label));
}
/** Given the set of NFA states in DFA state d, find all NFA states
* reachable traversing label arcs. By definition, there can be
* only one DFA state reachable by a single label from DFA state d so we must
* find and merge all NFA states reachable via label. Return a new
* DFAState that has all of those NFA states with their context.
*
* Because we cannot jump to another rule nor fall off the end of a rule
* via a non-epsilon transition, NFA states reachable from d have the
* same configuration as the NFA state in d. So if NFA state 7 in d's
* configurations can reach NFA state 13 then 13 will be added to the
* new DFAState (labelDFATarget) with the same configuration as state
* 7 had.
*/
public DFAState reach(DFAState d, IntervalSet label) {
System.out.println("reach "+label.toString(g)+" from "+d.stateNumber);
DFAState labelTarget = null;
for (NFAConfig c : d.nfaConfigs) {
int n = c.state.getNumberOfTransitions();
// int nEp = 0;
for (int i=0; i<n; i++) { // for each transition
Transition t = c.state.transition(i);
// if ( t.isEpsilon() ) nEp++;
// when we added this state as target of some other state,
// we tried to resolve any conflicts. Ignore anything we
// were able to fix previously
if ( c.resolved || c.resolvedWithPredicate ) continue;
// found a transition with label; does it collide with label?
// [Note: we still must test for isEpsilon here since
// computeStartState has to add these. Non-start-state
// closure ops will not add NFA states with only epsilon
// transitions, however.]
if ( !t.isEpsilon() && !t.label().and(label).isNil() ) {
// add NFA target to (potentially) new DFA state
if ( labelTarget==null ) labelTarget = dfa.newState();
labelTarget.addNFAConfig(new NFAConfig(c, t.target));
}
}
// System.out.println("config "+c+" has "+nEp+'/'+n+" eps edges");
// if ( nEp>0 && nEp!=n ) {
// System.out.println("MISMATCH");
// }
}
// [if we couldn't find any non-resolved edges to add, return nothing]
return labelTarget;
}
/** From this first NFA state of a decision, create a DFA.
* Walk each alt in decision and compute closure from the start of that
* rule, making sure that the closure does not include other alts within
* that same decision. The idea is to associate a specific alt number
* with the starting closure so we can trace the alt number for all states
* derived from this. At a stop state in the DFA, we can return this alt
* number, indicating which alt is predicted.
*/
public void computeStartState() {
DFAState d = dfa.newState();
dfa.startState = d;
// add config for each alt start, then add closure for those states
for (int altNum=1; altNum<=dfa.nAlts; altNum++) {
Transition t = nfaStartState.transition(altNum-1);
NFAState altStart = t.target;
d.addNFAConfig(
new NFAConfig(altStart, altNum,
NFAContext.EMPTY(),
SemanticContext.EMPTY_SEMANTIC_CONTEXT));
}
closure(d);
}
/** For all NFA states (configurations) merged in d,
* compute the epsilon closure; that is, find all NFA states reachable
* from the NFA states in d via purely epsilon transitions.
*/
public void closure(DFAState d) {
if ( debug ) {
System.out.println("closure("+d+")");
}
// Only the start state initiates pred collection; gets turned
// off maybe by actions later hence we need a parameter to carry
// it forward
boolean collectPredicates = (d == dfa.startState);
// TODO: can we avoid this separate list by directly filling d.nfaConfigs?
// OH: concurrent modification. dup initialconfigs? works for lexers, try here to save configs param
List<NFAConfig> configs = new ArrayList<NFAConfig>();
configs.addAll(d.nfaConfigs);
for (NFAConfig c : configs) {
closure(d, c, collectPredicates);
}
closureBusy.clear();
if ( debug ) {
System.out.println("after closure("+d+")");
}
//System.out.println("after closure d="+d);
}
/** Where can we get from NFA state s traversing only epsilon transitions?
*
* A closure operation should abort if that computation has already
* been done or a computation with a conflicting context has already
* been done. If proposed NFA config's state and alt are the same
* there is potentially a problem. If the stack context is identical
* then clearly the exact same computation is proposed. If a context
* is a suffix of the other, then again the computation is in an
* identical context. beta $ and beta alpha $ are considered the same stack.
* We could walk configurations linearly doing suuch a comparison instead
* of a set lookup for exact matches but it's much slower because you can't
* do a Set lookup. I use exact match as ANTLR always detect the conflict
* later when checking for ambiguous configs (it tests context suffixes).
*
* TODO: change comment once I figure out if we can ignore suffixes in favor of empty/non test only
* 4/11/2010 I removed suffix check from getAmbigAlts and it broke; seems I need it.
*
* Side-effect warnings:
*
* Rather than pass in a list of configs to update or return and
* collect lots of little config lists, it's more efficient to
* modify d's config list directly.
*
* Rather than pass closureBusy everywhere, I use a field of this object.
*/
public void closure(DFAState d, NFAConfig c, boolean collectPredicates) {
System.out.println("closure of "+c+" in "+d);
if ( closureBusy.contains(c) ) return; // don't re-attempt same closure(c)
closureBusy.add(c);
// Theory says p is always in closure; in practice, though, we
// we want to reduce the number of NFA configurations in the closure.
// The purpose of the closure operation is to find all NFA states
// reachable from a particular state traversing only epsilon
// transitions. Later, during the reach operation, we're going to
// find all NFA states reachable from those states given a particular
// label (token). The fewer the NFA states we have to walk during
// reach the better. Since reach only cares about states with non-epsilon
// transitions, let's only add those states to the closure. Saves memory
// and time. When I run TestDFAConstruction, printing out the
// NFA configs as I test them in reach(), it reduces output from
// 1436 lines to 74. seriously. like wow.
//
// 5/5/2010: This optimization only occurred to me after I implemented
// the NFA bytecode VM. It had to ignore all SPLIT, JMP states
// during reach. I realized that we could simply avoid adding these
// closures instead of ignoring them later. I retrofitted to parser
// DFA construction.
//
if ( !c.state.onlyHasEpsilonTransitions() ) {
d.nfaConfigs.add(c);
}
if ( c.state instanceof RuleStopState ) {
ruleStopStateClosure(d, c, collectPredicates);
}
else {
commonClosure(d, c, collectPredicates);
}
}
// if we have context info and we're at rule stop state, do
// local follow for invokingRule and global follow for other links
void ruleStopStateClosure(DFAState d, NFAConfig c, boolean collectPredicates) {
if ( !c.context.approximated ) {
//System.out.println("dynamic FOLLOW of "+c.state+" context="+c.context);
if ( c.context.isEmpty() ) {
commonClosure(d, c, collectPredicates); // do global FOLLOW
}
else {
NFAContext newContext = c.context.parent; // "pop" invoking state
closure(d, new NFAConfig(c, c.context.returnState, newContext),
collectPredicates);
}
return;
}
Rule invokingRule = null;
if ( !c.context.isEmpty() ) {
// if stack not empty, get invoking rule from top of stack
invokingRule = c.context.returnState.rule;
}
System.out.println("FOLLOW of "+c+" invoking rule="+invokingRule);
// follow all static FOLLOW links
int n = c.state.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = c.state.transition(i);
if ( !(t instanceof EpsilonTransition) ) continue; // ignore EOF transitions
// Chase global FOLLOW links if they don't point at invoking rule
// else follow link to context state only
if ( t.target.rule != invokingRule ) {
System.out.println("OFF TO "+t.target);
closure(d, new NFAConfig(c, t.target), collectPredicates);
}
else { // t.target is in invoking rule; only follow context's link
if ( t.target == c.context.returnState ) {
System.out.println("OFF TO CALL SITE "+t.target);
// go only to specific call site; pop context
NFAContext newContext = c.context.parent; // "pop" invoking state
closure(d, new NFAConfig(c, t.target, newContext),
collectPredicates);
}
}
}
return;
}
void commonClosure(DFAState d, NFAConfig c, boolean collectPredicates) {
//System.out.println("closure of "+c);
int n = c.state.getNumberOfTransitions();
for (int i=0; i<n; i++) {
Transition t = c.state.transition(i);
if ( t instanceof RuleTransition) {
RuleTransition rt = (RuleTransition) t;
// have we called rt.target before? If so, rt.followState will be in c.context
int depth = c.context.occurrences(rt.followState.stateNumber);
//System.out.println("ctx "+c.context+" c.state "+c.state+" ret state is "+rt.followState.stateNumber);
// Even if not on context stack already, must consider self-recursion.
// If a state in rule r's NFA invokes r's start state (only state
// rule trans can invoke) then it's yet more recursion.
// So we count previous invocations of r first and then
// increment if we're jumping to start state from within r.
if ( c.state.rule == t.target.rule ) depth++;
// System.out.println("recur depth "+depth);
// Detect an attempt to recurse too high
// if this context has hit the max recursions,
// don't allow it to enter rule again
NFAContext newContext = null;
if ( c.context.approximated || depth > NFAContext.MAX_RECURSION_DEPTH_PER_NFA_CONFIG_STACK ) {
// System.out.println("# recursive invoke of "+t.target+" ret to "+rt.followState+" ctx="+c.context);
// don't record recursion, but record we approximated so we know
// what to do at end of rule and for error msgs.
newContext = new NFAContext(c.context);
newContext.approximated = true;
}
else if ( !c.context.approximated ) {// if not already approximating
// otherwise, it's cool to (re)enter target of this rule ref
// first create a new context and push onto call tree,
// recording the fact that we are invoking a rule and
// from which state.
//System.out.println("nonrecursive invoke of "+t.target+" ret to "+retState+" ctx="+c.context);
NFAState retState = ((RuleTransition)t).followState;
newContext = new NFAContext(c.context, retState);
}
// traverse epsilon edge to new rule
closure(d, new NFAConfig(c, t.target, newContext),
collectPredicates);
// NFAState retState = ((RuleTransition)t).followState;
// NFAContext newContext = c.context;
// if ( c.state.rule != t.target.rule &&
// !c.context.contains(((RuleTransition)t).followState) ) { // !recursive?
// // first create a new context and push onto call tree,
// // recording the fact that we are invoking a rule and
// // from which state.
// //System.out.println("nonrecursive invoke of "+t.target+" ret to "+retState+" ctx="+c.context);
// newContext = new NFAContext(c.context, retState);
// }
// else {
// //System.out.println("# recursive invoke of "+t.target+" ret to "+retState+" ctx="+c.context);
// // don't record recursion, but record we did so we know
// // what to do at end of rule.
// c.context.recursed = true;
// }
// // traverse epsilon edge to new rule
// closure(d, new NFAConfig(c, t.target, newContext),
// collectPredicates);
}
else if ( t instanceof ActionTransition ) {
collectPredicates = false; // can't see past actions
closure(d, new NFAConfig(c, t.target), collectPredicates);
}
else if ( t instanceof PredicateTransition ) {
SemanticContext labelContext = ((PredicateTransition)t).semanticContext;
SemanticContext newSemanticContext = c.semanticContext;
if ( collectPredicates ) {
// AND the previous semantic context with new pred
// int walkAlt =
// dfa.decisionNFAStartState.translateDisplayAltToWalkAlt(alt);
NFAState altLeftEdge = dfa.decisionNFAStartState.transition(c.alt-1).target;
/*
System.out.println("state "+p.stateNumber+" alt "+alt+" walkAlt "+walkAlt+" trans to "+transition0.target);
System.out.println("DFA start state "+dfa.decisionNFAStartState.stateNumber);
System.out.println("alt left edge "+altLeftEdge.stateNumber+
", epsilon target "+
altLeftEdge.transition(0).target.stateNumber);
*/
// do not hoist syn preds from other rules; only get if in
// starting state's rule (i.e., context is empty)
if ( !labelContext.isSyntacticPredicate() || c.state==altLeftEdge ) {
//System.out.println("&"+labelContext+" enclosingRule="+c.state.rule);
newSemanticContext =
SemanticContext.and(c.semanticContext, labelContext);
}
}
else {
// if we're not collecting, means we saw an action previously. that blocks this pred
hasPredicateBlockedByAction = true;
}
closure(d, new NFAConfig(c, t.target, newSemanticContext),
collectPredicates);
}
else if ( t.isEpsilon() ) {
closure(d, new NFAConfig(c, t.target), collectPredicates);
}
}
}
/** for each NFA config in d, look for "predicate required" sign we set
* during nondeterminism resolution.
*
* Add the predicate edges sorted by the alternative number; I'm fairly
* sure that I could walk the configs backwards so they are added to
* the predDFATarget in the right order, but it's best to make sure.
* Predicates succeed in the order they are specifed. Alt i wins
* over alt i+1 if both predicates are true.
*/
protected void addPredicateTransitions(DFAState d) {
List<NFAConfig> configsWithPreds = new ArrayList<NFAConfig>();
// get a list of all configs with predicates
for (NFAConfig c : d.nfaConfigs) {
if ( c.resolvedWithPredicate) {
configsWithPreds.add(c);
}
}
// Sort ascending according to alt; alt i has higher precedence than i+1
Collections.sort(configsWithPreds,
new Comparator<NFAConfig>() {
public int compare(NFAConfig a, NFAConfig b) {
if ( a.alt < b.alt ) return -1;
else if ( a.alt > b.alt ) return 1;
return 0;
}
});
List<NFAConfig> predConfigsSortedByAlt = configsWithPreds;
// Now, we can add edges emanating from d for these preds in right order
for (NFAConfig c : predConfigsSortedByAlt) {
DFAState predDFATarget = dfa.newState();
// new DFA state is a target of the predicate from d
predDFATarget.addNFAConfig(c);
dfa.addAcceptState(c.alt, predDFATarget);
// add a transition to pred target from d
d.addEdge(new PredicateEdge(c.semanticContext, predDFATarget));
}
}
public Set<Integer> getUnreachableAlts() {
Set<Integer> unreachable = new HashSet<Integer>();
for (int alt=1; alt<=dfa.nAlts; alt++) {
if ( dfa.altToAcceptStates[alt]==null ) unreachable.add(alt);
}
return unreachable;
}
public void issueAmbiguityWarnings() {
MachineProbe probe = new MachineProbe(dfa);
for (DFAState d : resolver.ambiguousStates) {
Set<Integer> ambigAlts = Resolver.getAmbiguousAlts(d);
List<Integer> sorted = new ArrayList<Integer>(ambigAlts);
Collections.sort(sorted);
//System.err.println("ambig alts="+sorted);
List<DFAState> dfaStates = probe.getAnyDFAPathToTarget(d);
//System.out.print("path =");
for (DFAState d2 : dfaStates) {
// System.out.print(" "+d2.stateNumber);
}
//System.out.println("");
List<IntSet> labels = probe.getEdgeLabels(d);
String input = probe.getInputSequenceDisplay(g, labels);
//System.out.println("input="+ input);
LinkedHashMap<Integer,List<Token>> altPaths = new LinkedHashMap<Integer,List<Token>>();
for (int alt : sorted) {
List<Set<NFAState>> nfaStates = new ArrayList<Set<NFAState>>();
for (DFAState d2 : dfaStates) {
nfaStates.add( d2.getUniqueNFAStates(alt) );
}
//System.out.println("NFAConfigs per state: "+nfaStates);
List<Token> path =
probe.getGrammarLocationsForInputSequence(nfaStates, labels);
altPaths.put(alt, path);
//System.out.println("path = "+path);
}
List<Integer> incompletelyCoveredAlts = statesWithIncompletelyCoveredAlts.get(d);
if ( incompletelyCoveredAlts!=null && incompletelyCoveredAlts.size()>0 ) {
Map<Integer, Set<Token>> insufficientAltToLocations =
PredicateResolver.getInsufficientlyPredicatedLocations(d, incompletelyCoveredAlts);
g.tool.errMgr.insufficientPredicates(g.fileName, d, input,
insufficientAltToLocations,
hasPredicateBlockedByAction);
}
if ( !d.resolvedWithPredicates &&
(incompletelyCoveredAlts==null || incompletelyCoveredAlts.size()==0) )
{
Set<Integer> approxContextAlts = Resolver.getAltsWithApproximateContext(d);
Set<Integer> certainAmbiguousAlt = ambigAlts;
if ( approxContextAlts!=null ) certainAmbiguousAlt.removeAll(approxContextAlts);
//if ( ambigAlts.containsAll()
g.tool.errMgr.ambiguity(g.fileName, d, sorted, input, altPaths,
hasPredicateBlockedByAction);
}
}
if ( unreachableAlts!=null && unreachableAlts.size()>0 ) {
g.tool.errMgr.unreachableAlts(g.fileName, dfa,
unreachableAlts);
}
}
}

14
tool/src/org/antlr/v4/analysis/RecursionOverflowSignal.java
View File

@ -1,14 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.NFAState;
public class RecursionOverflowSignal extends RuntimeException {
int altNum;
int depth;
NFAState state;
public RecursionOverflowSignal(int altNum, int depth, NFAState state) {
this.altNum = altNum;
this.depth = depth;
this.state = state;
}
}

357
tool/src/org/antlr/v4/analysis/Resolver.java
View File

@ -1,357 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.automata.DFAState;
import org.antlr.v4.automata.NFA;
import org.antlr.v4.misc.Utils;
import org.stringtemplate.v4.misc.MultiMap;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/** Code "module" that knows how to resolve LL(*) nondeterminisms. */
public class Resolver {
PredicateResolver semResolver;
/** Track all DFA states with ambiguous configurations.
* By reaching the same DFA state, a path through the NFA for some input
* is able to reach the same NFA state by starting at more than one
* alternative's left edge. If the context is the same or conflicts,
* then we have ambiguity. If the context is different, it's simply
* nondeterministic and we should keep looking for edges that will
* render it deterministic. If we run out of things to add to the DFA,
* we'll get a dangling state; it's non-LL(*). Later we may find that predicates
* resolve the issue, but track ambiguous states anyway.
*/
public Set<DFAState> ambiguousStates = new HashSet<DFAState>();
/** The set of states w/o emanating edges (and w/o resolving sem preds). */
public Set<DFAState> danglingStates = new HashSet<DFAState>();
/** Was a syntactic ambiguity resolved with predicates? Any DFA
* state that predicts more than one alternative, must be resolved
* with predicates or it should be reported to the user.
*/
public Set<DFAState> resolvedWithSemanticPredicates = new HashSet<DFAState>();
public Resolver() {
//this.converter = converter;
semResolver = new PredicateResolver();
}
/** Walk each NFA configuration in this DFA state looking for a conflict
* where (s|i|ctx1) and (s|j|ctx2) exist such that ctx1 and ctx2 conflict.
* That indicates that state s predicts alts i and j. Return an Integer set
* of the alternative numbers that conflict. Two contexts conflict if
* they are equal or one is a stack suffix of the other or one is
* the empty context. The conflict is a true ambiguity. No amount
* of further looking in grammar will resolve issue (only preds help).
*
* Use a hash table to record the lists of configs for each state
* as they are encountered. We need only consider states for which
* there is more than one configuration. The configurations' predicted
* alt must be different or must have different contexts to avoid a
* conflict.
*
* We check exact config match only in closure-busy test so we'll see
* empty and nonempty contexts here for same state and alt; e.g.,
* (s|i|$) and (s|i|[21 3 $]).
*
* TODO: suffix degenerates to one empty one nonempty; avoid some tests?
* TODO: or perhaps check if i, j are already in and don't do compare?
*/
public static Set<Integer> getAmbiguousAlts(DFAState d) {
//System.out.println("getNondetAlts for DFA state "+stateNumber);
Set<Integer> ambiguousAlts = new HashSet<Integer>();
// If only 1 NFA conf then no way it can be nondeterministic;
// save the overhead. There are many o-a->o NFA transitions
// and so we save a hash map and iterator creation for each
// state.
int numConfigs = d.nfaConfigs.size();
if ( numConfigs<=1 ) return null;
// First get a list of configurations for each state.
// Most of the time, each state will have one associated configuration.
MultiMap<Integer, NFAConfig> stateToConfigListMap =
new MultiMap<Integer, NFAConfig>();
for (NFAConfig c : d.nfaConfigs) {
stateToConfigListMap.map(Utils.integer(c.state.stateNumber), c);
}
// potential conflicts are states with > 1 configuration and diff alts
boolean thisStateHasPotentialProblem = false;
for (List<NFAConfig> configsForState : stateToConfigListMap.values()) {
if ( configsForState.size()>1 ) {
int predictedAlt = Resolver.getUniqueAlt(configsForState);
if ( predictedAlt > 0 ) {
// remove NFA state's configurations from
// further checking; no issues with it
// (can't remove as it's concurrent modification; set to null)
stateToConfigListMap.put(configsForState.get(0).state.stateNumber, null);
}
else {
thisStateHasPotentialProblem = true;
}
}
}
// a fast check for potential issues; most states have none
if ( !thisStateHasPotentialProblem ) return null;
// we have a potential problem, so now go through config lists again
// looking for different alts (only states with potential issues
// are left in the states set). Now we will check context.
// For example, the list of configs for NFA state 3 in some DFA
// state might be:
// [3|2|[28 18 $], 3|1|[28 $], 3|1, 3|2]
// I want to create a map from context to alts looking for overlap:
// [28 18 $] -> 2
// [28 $] -> 1
// [$] -> 1,2
// Indeed a conflict exists as same state 3, same context [$], predicts
// alts 1 and 2.
// walk each state with potential conflicting configurations
for (List<NFAConfig> configsForState : stateToConfigListMap.values()) {
// compare each configuration pair s, t to ensure:
// s.ctx different than t.ctx if s.alt != t.alt
int numConfigsForState = 0;
if ( configsForState!=null ) numConfigsForState = configsForState.size();
for (int i = 0; i < numConfigsForState; i++) {
NFAConfig s = (NFAConfig) configsForState.get(i);
for (int j = i+1; j < numConfigsForState; j++) {
NFAConfig t = (NFAConfig)configsForState.get(j);
// conflicts means s.ctx==t.ctx or s.ctx is a stack
// suffix of t.ctx or vice versa (if alts differ).
// Also a conflict if s.ctx or t.ctx is empty
// TODO: might be faster to avoid suffix test if i or j already in ambiguousAlts
// that set is usually 2 alts, maybe three. Use a List.
boolean altConflict = s.alt != t.alt;
boolean ctxConflict = (s.context==null && t.context!=null) ||
(s.context!=null && t.context==null) ||
(s.context.suffix(t.context) || s.context.equals(t.context));
if ( altConflict && ctxConflict ) {
//System.out.println("ctx conflict between "+s+" and "+t);
ambiguousAlts.add(s.alt);
ambiguousAlts.add(t.alt);
}
}
}
}
if ( ambiguousAlts.size()==0 ) return null;
return ambiguousAlts;
}
public static Set<Integer> getAltsWithApproximateContext(DFAState d) {
Set<Integer> approxContextAlts = new HashSet<Integer>();
for (NFAConfig c : d.nfaConfigs) {
if ( c.context.approximated ) {
approxContextAlts.add(Utils.integer(c.alt));
}
}
if ( approxContextAlts.size()==0 ) return null;
return approxContextAlts;
}
public void resolveAmbiguities(DFAState d) {
if ( PredictionDFAFactory.debug ) {
System.out.println("resolveNonDeterminisms "+d.toString());
}
Set<Integer> ambiguousAlts = getAmbiguousAlts(d);
if ( PredictionDFAFactory.debug && ambiguousAlts!=null ) {
System.out.println("ambig alts="+ambiguousAlts);
}
// if no problems return
if ( ambiguousAlts==null ) return;
ambiguousStates.add(d);
// ATTEMPT TO RESOLVE WITH SEMANTIC PREDICATES
boolean resolved =
semResolver.tryToResolveWithSemanticPredicates(d, ambiguousAlts);
if ( resolved ) {
if ( PredictionDFAFactory.debug ) {
System.out.println("resolved DFA state "+d.stateNumber+" with pred");
}
d.resolvedWithPredicates = true;
resolvedWithSemanticPredicates.add(d);
return;
}
// RESOLVE SYNTACTIC CONFLICT BY REMOVING ALL BUT ONE ALT
resolveByPickingMinAlt(d, ambiguousAlts);
}
public void resolveDeadState(DFAState d) {
if ( d.resolvedWithPredicates || d.getNumberOfEdges()>0 ) return;
System.err.println("dangling DFA state "+d+" after reach / closures");
danglingStates.add(d);
// turn off all configurations except for those associated with
// min alt number; somebody has to win else some input will not
// predict any alt.
int minAlt = resolveByPickingMinAlt(d, null);
// force it to be an accept state
d.isAcceptState = true;
d.predictsAlt = minAlt;
// might be adding new accept state for alt, but that's ok
d.dfa.addAcceptState(minAlt, d);
}
/** Turn off all configurations associated with the
* set of incoming alts except the min alt number.
* There may be many alts among the configurations but only turn off
* the ones with problems (other than the min alt of course).
*
* If alts is null then turn off all configs 'cept those
* associated with the minimum alt.
*
* Return the min alt found.
*/
static int resolveByPickingMinAlt(DFAState d, Set<Integer> alts) {
int min = 0;
if ( alts !=null ) {
min = getMinAlt(alts);
}
else {
min = d.getMinAlt();
}
turnOffOtherAlts(d, min, alts);
return min;
}
/** turn off all states associated with alts other than the good one
* (as long as they are one of the ones in alts)
*/
static void turnOffOtherAlts(DFAState d, int min, Set<Integer> alts) {
int numConfigs = d.nfaConfigs.size();
for (int i = 0; i < numConfigs; i++) {
NFAConfig configuration = d.nfaConfigs.get(i);
if ( configuration.alt!=min ) {
if ( alts==null ||
alts.contains(configuration.alt) )
{
configuration.resolved = true;
}
}
}
}
public static int getMinAlt(Set<Integer> alts) {
int min = Integer.MAX_VALUE;
for (Integer altI : alts) {
int alt = altI.intValue();
if ( alt < min ) min = alt;
}
return min;
}
public static int getUniqueAlt(Collection<NFAConfig> nfaConfigs) {
return getUniqueAlt(nfaConfigs, true);
}
public static int getUniqueAlt(Collection<NFAConfig> nfaConfigs,
boolean ignoreResolvedBit)
{
int alt = NFA.INVALID_ALT_NUMBER;
for (NFAConfig c : nfaConfigs) {
if ( !ignoreResolvedBit && c.resolved ) continue;
if ( alt==NFA.INVALID_ALT_NUMBER ) {
alt = c.alt; // found first alt
}
else if ( c.alt!=alt ) {
return NFA.INVALID_ALT_NUMBER;
}
}
return alt;
}
/*
void issueRecursionWarnings() {
// RECURSION OVERFLOW
Set dfaStatesWithRecursionProblems =
converter.stateToRecursionOverflowConfigurationsMap.keySet();
// now walk truly unique (unaliased) list of dfa states with inf recur
// Goal: create a map from alt to map<target,List<callsites>>
// Map<Map<String target, List<NFAState call sites>>
Map<Integer, Map<>> altToT argetToCallSitesMap = new HashMap();
// track a single problem DFA state for each alt
Map<Integer, DFAState> altToDFAState = new HashMap<Integer, DFAState>();
computeAltToProblemMaps(dfaStatesWithRecursionProblems,
converter.stateToRecursionOverflowConfigurationsMap,
altToTargetToCallSitesMap, // output param
altToDFAState); // output param
// walk each alt with recursion overflow problems and generate error
Set<Integer> alts = altToTargetToCallSitesMap.keySet();
List<Integer> sortedAlts = new ArrayList<Integer>(alts);
Collections.sort(sortedAlts);
for (Iterator altsIt = sortedAlts.iterator(); altsIt.hasNext();) {
Integer altI = (Integer) altsIt.next();
Map<Integer, > targetToCallSiteMap =
altToTargetToCallSitesMap.get(altI);
Set targetRules = targetToCallSiteMap.keySet();
Collection callSiteStates = targetToCallSiteMap.values();
DFAState sampleBadState = altToDFAState.get(altI);
ErrorManager.recursionOverflow(this,
sampleBadState,
altI.intValue(),
targetRules,
callSiteStates);
}
}
void computeAltToProblemMaps(Set<DFAState> dfaStatesUnaliased,
Map configurationsMap,
Map<Integer, NFAState> altToTargetToCallSitesMap,
Map altToDFAState)
{
for (DFAState d : dfaStatesUnaliased) {
for (NFAConfig c : d.nfaConfigs) {
NFAState ruleInvocationState = c.state;
RuleTransition rt = (RuleTransition)ruleInvocationState.transition(0);
String targetRule = rt.rule.name;
}
}
for (Iterator it = dfaStatesUnaliased.iterator(); it.hasNext();) {
Integer stateI = (Integer) it.next();
// walk this DFA's config list
List configs = (List)configurationsMap.get(stateI);
for (int i = 0; i < configs.size(); i++) {
NFAConfig c = (NFAConfig) configs.get(i);
NFAState ruleInvocationState = c.state;
Transition transition0 = ruleInvocationState.transition(0);
RuleTransition ref = (RuleTransition)transition0;
String targetRule = ((NFAState) ref.target).rule.name;
Integer altI = org.antlr.misc.Utils.integer(c.alt);
Map<Integer, NFAState> targetToCallSiteMap =
altToTargetToCallSitesMap.get(altI);
if ( targetToCallSiteMap==null ) {
targetToCallSiteMap = new HashMap();
altToTargetToCallSitesMap.put(altI, targetToCallSiteMap);
}
Set<NFAState> callSites = targetToCallSiteMap.get(targetRule);
if ( callSites==null ) {
callSites = new HashSet();
targetToCallSiteMap.put(targetRule, callSites);
}
callSites.add(ruleInvocationState);
// track one problem DFA state per alt
if ( altToDFAState.get(altI)==null ) {
DFAState sampleBadState = converter.dfa.states.get(stateI.intValue());
altToDFAState.put(altI, sampleBadState);
}
}
}
}
*/
}

298
tool/src/org/antlr/v4/analysis/SemanticContext.java
View File

@ -1,298 +0,0 @@
package org.antlr.v4.analysis;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.tool.GrammarAST;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
/** A tree structure used to record the semantic context in which
* an NFA configuration is valid. It's either a single predicate or
* a tree representing an operation tree such as: p1&&p2 or p1||p2.
*
* For NFA o-p1->o-p2->o, create tree AND(p1,p2).
* For NFA (1)-p1->(2)
* | ^
* | |
* (3)-p2----
* we will have to combine p1 and p2 into DFA state as we will be
* adding NFA configurations for state 2 with two predicates p1,p2.
* So, set context for combined NFA config for state 2: OR(p1,p2).
*/
public abstract class SemanticContext {
/** Create a default value for the semantic context shared among all
* NFAConfigurations that do not have an actual semantic context.
* This prevents lots of if!=null type checks all over; it represents
* just an empty set of predicates.
*/
public static final SemanticContext EMPTY_SEMANTIC_CONTEXT = new Predicate();
/** Given a semantic context expression tree, return a tree with all
* nongated predicates set to true and then reduced. So p&&(q||r) would
* return p&&r if q is nongated but p and r are gated.
*/
public abstract SemanticContext getGatedPredicateContext();
public abstract boolean isSyntacticPredicate();
public static class Predicate extends SemanticContext {
/** The AST node in tree created from the grammar holding the predicate */
public GrammarAST predicateAST;
/** Is this a {...}?=> gating predicate or a normal disambiguating {..}?
* If any predicate in expression is gated, then expression is considered
* gated.
*
* The simple Predicate object's predicate AST's type is used to set
* gated to true if type==GATED_SEMPRED.
*/
protected boolean gated = false;
/** syntactic predicates are converted to semantic predicates
* but synpreds are generated slightly differently.
*/
protected boolean synpred = false;
public static final int INVALID_PRED_VALUE = -1;
public static final int TRUE_PRED = 1;
/** sometimes predicates are known to be true or false; we need
* a way to represent this without resorting to a target language
* value like true or TRUE.
*/
protected int constantValue = INVALID_PRED_VALUE;
public Predicate() {
this.gated=false;
}
public Predicate(GrammarAST predicate) {
this.predicateAST = predicate;
this.gated =
predicate.getType()== ANTLRParser.GATED_SEMPRED ||
predicate.getType()==ANTLRParser.SYN_SEMPRED ;
this.synpred =
predicate.getType()==ANTLRParser.SYN_SEMPRED ||
predicate.getType()== ANTLRParser.BACKTRACK_SEMPRED;
}
public Predicate(Predicate p) {
this.predicateAST = p.predicateAST;
this.gated = p.gated;
this.synpred = p.synpred;
this.constantValue = p.constantValue;
}
/** Two predicates are the same if they are literally the same
* text rather than same node in the grammar's AST.
*/
public boolean equals(Object o) {
if ( !(o instanceof Predicate) ) return false;
Predicate p = (Predicate) o;
if ( predicateAST!=null && p.predicateAST!=null )
return predicateAST.getText().equals(p.predicateAST.getText());
return predicateAST==null && p.predicateAST==null;
}
public int hashCode() {
if ( predicateAST ==null ) {
return 0;
}
return predicateAST.getText().hashCode();
}
public SemanticContext getGatedPredicateContext() {
if ( gated ) {
return this;
}
return null;
}
public boolean isSyntacticPredicate() {
return predicateAST !=null &&
( predicateAST.getType()==ANTLRParser.SYN_SEMPRED ||
predicateAST.getType()==ANTLRParser.BACKTRACK_SEMPRED );
}
public String toString() {
if ( predicateAST ==null ) {
return "<nopred>";
}
return predicateAST.getText();
}
}
public static class TruePredicate extends Predicate {
public TruePredicate() {
super();
this.constantValue = TRUE_PRED;
}
public String toString() {
return "true"; // not used for code gen, just DOT and print outs
}
}
public static class AND extends SemanticContext {
protected SemanticContext left,right;
public AND(SemanticContext a, SemanticContext b) {
this.left = a;
this.right = b;
}
public SemanticContext getGatedPredicateContext() {
SemanticContext gatedLeft = left.getGatedPredicateContext();
SemanticContext gatedRight = right.getGatedPredicateContext();
if ( gatedLeft==null ) {
return gatedRight;
}
if ( gatedRight==null ) {
return gatedLeft;
}
return new AND(gatedLeft, gatedRight);
}
public boolean isSyntacticPredicate() {
return left.isSyntacticPredicate()||right.isSyntacticPredicate();
}
public String toString() {
return "("+left+"&&"+right+")";
}
}
public static class OR extends SemanticContext {
protected Set<SemanticContext> operands;
public OR(SemanticContext a, SemanticContext b) {
operands = new HashSet<SemanticContext>();
if ( a instanceof OR ) {
operands.addAll(((OR)a).operands);
}
else if ( a!=null ) {
operands.add(a);
}
if ( b instanceof OR ) {
operands.addAll(((OR)b).operands);
}
else if ( b!=null ) {
operands.add(b);
}
}
public SemanticContext getGatedPredicateContext() {
SemanticContext result = null;
for (Iterator it = operands.iterator(); it.hasNext();) {
SemanticContext semctx = (SemanticContext) it.next();
SemanticContext gatedPred = semctx.getGatedPredicateContext();
if ( gatedPred!=null ) {
result = or(result, gatedPred);
// result = new OR(result, gatedPred);
}
}
return result;
}
public boolean isSyntacticPredicate() {
for (Iterator it = operands.iterator(); it.hasNext();) {
SemanticContext semctx = (SemanticContext) it.next();
if ( semctx.isSyntacticPredicate() ) {
return true;
}
}
return false;
}
public String toString() {
StringBuffer buf = new StringBuffer();
buf.append("(");
int i = 0;
for (Iterator it = operands.iterator(); it.hasNext();) {
SemanticContext semctx = (SemanticContext) it.next();
if ( i>0 ) {
buf.append("||");
}
buf.append(semctx.toString());
i++;
}
buf.append(")");
return buf.toString();
}
}
public static class NOT extends SemanticContext {
protected SemanticContext ctx;
public NOT(SemanticContext ctx) {
this.ctx = ctx;
}
public SemanticContext getGatedPredicateContext() {
SemanticContext p = ctx.getGatedPredicateContext();
if ( p==null ) {
return null;
}
return new NOT(p);
}
public boolean isSyntacticPredicate() {
return ctx.isSyntacticPredicate();
}
public boolean equals(Object object) {
if ( !(object instanceof NOT) ) {
return false;
}
return this.ctx.equals(((NOT)object).ctx);
}
public String toString() {
return "!("+ctx+")";
}
}
public static SemanticContext and(SemanticContext a, SemanticContext b) {
//System.out.println("AND: "+a+"&&"+b);
if ( a==EMPTY_SEMANTIC_CONTEXT || a==null ) {
return b;
}
if ( b==EMPTY_SEMANTIC_CONTEXT || b==null ) {
return a;
}
if ( a.equals(b) ) {
return a; // if same, just return left one
}
//System.out.println("## have to AND");
return new AND(a,b);
}
public static SemanticContext or(SemanticContext a, SemanticContext b) {
//System.out.println("OR: "+a+"||"+b);
if ( a==EMPTY_SEMANTIC_CONTEXT || a==null ) {
return b;
}
if ( b==EMPTY_SEMANTIC_CONTEXT || b==null ) {
return a;
}
if ( a instanceof TruePredicate ) {
return a;
}
if ( b instanceof TruePredicate ) {
return b;
}
if ( a instanceof NOT && b instanceof Predicate ) {
NOT n = (NOT)a;
// check for !p||p
if ( n.ctx.equals(b) ) {
return new TruePredicate();
}
}
else if ( b instanceof NOT && a instanceof Predicate ) {
NOT n = (NOT)b;
// check for p||!p
if ( n.ctx.equals(a) ) {
return new TruePredicate();
}
}
else if ( a.equals(b) ) {
return a;
}
//System.out.println("## have to OR");
return new OR(a,b);
}
public static SemanticContext not(SemanticContext a) {
return new NOT(a);
}
}

29
tool/src/org/antlr/v4/automata/ActionTransition.java
View File

@ -1,29 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarAST;
public class ActionTransition extends Transition {
public GrammarAST actionAST;
public ActionTransition(GrammarAST actionAST, NFAState target) {
super(target);
this.actionAST = actionAST;
}
public boolean isEpsilon() {
return true; // we are to be ignored by analysis 'cept for predicates
}
public int compareTo(Object o) {
return 0;
}
public String toString() {
return "{"+actionAST+"}";
}
public String toString(Grammar g) {
return toString();
}
}

47
tool/src/org/antlr/v4/automata/AtomTransition.java
View File

@ -1,47 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.tool.Grammar;
/** TODO: make all transitions sets? */
public class AtomTransition extends Transition {
/** The token type or character value; or, signifies special label. */
public int label;
public AtomTransition(int label, NFAState target) {
this.label = label;
this.target = target;
}
public IntervalSet label() { return IntervalSet.of(label); }
public int hashCode() { return label; }
public boolean equals(Object o) {
if ( o==null ) return false;
if ( this == o ) return true; // equals if same object
if ( o.getClass() == SetTransition.class ) {
return IntervalSet.of(label).equals(o);
}
return label!=((AtomTransition)o).label;
}
// public boolean intersect(Label other) {
// if ( other.getClass() == AtomTransition.class ) {
// return label==((AtomTransition)other).label;
// }
// return ((SetLabel)other).label.member(this.label);
// }
public int compareTo(Object o) {
return this.label-((AtomTransition)o).label;
}
public String toString(Grammar g) {
return g.getTokenDisplayName(label);
}
public String toString() {
return String.valueOf(label);
}
}

37
tool/src/org/antlr/v4/automata/BasicState.java
View File

@ -1,37 +0,0 @@
package org.antlr.v4.automata;
/** */
public class BasicState extends NFAState {
public Transition transition;
/** For o-A->o type NFA tranitions, record the label that leads to this
* state. Useful for creating rich error messages when we find
* insufficiently (with preds) covered states.
*/
public Transition incidentTransition;
public BasicState(NFA nfa) { super(nfa); }
@Override
public int getNumberOfTransitions() {
if ( transition!=null ) return 1;
return 0;
}
@Override
public void addTransition(Transition e) {
if ( transition!=null ) throw new IllegalArgumentException("only one transition");
transition = e;
}
@Override
public Transition transition(int i) {
if ( i>0 ) throw new IllegalArgumentException("only one transition");
return transition;
}
@Override
public boolean onlyHasEpsilonTransitions() {
return transition!=null && transition.isEpsilon();
}
}

5
tool/src/org/antlr/v4/automata/BlockEndState.java
View File

@ -1,5 +0,0 @@
package org.antlr.v4.automata;
public class BlockEndState extends BasicState {
public BlockEndState(NFA nfa) { super(nfa); }
}

25
tool/src/org/antlr/v4/automata/BlockStartState.java
View File

@ -1,25 +0,0 @@
package org.antlr.v4.automata;
import java.util.ArrayList;
import java.util.List;
/** */
public class BlockStartState extends DecisionState {
public static final int INITIAL_NUM_TRANSITIONS = 4;
public BlockEndState endState;
/** Track the transitions emanating from this NFA state. */
public List<Transition> transitions = new ArrayList<Transition>(INITIAL_NUM_TRANSITIONS);
public BlockStartState(NFA nfa) { super(nfa); }
@Override
public int getNumberOfTransitions() { return transitions.size(); }
@Override
public void addTransition(Transition e) { transitions.add(e); }
@Override
public Transition transition(int i) { return transitions.get(i); }
}

252
tool/src/org/antlr/v4/automata/DFA.java
View File

@ -1,252 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.analysis.PredictionDFAFactory;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.tool.Grammar;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/** A DFA (converted from a grammar's NFA).
* DFAs are used as prediction machine for alternative blocks in all kinds
* of recognizers (lexers, parsers, tree walkers).
*/
public class DFA {
public Grammar g;
/** What's the start state for this DFA? */
public DFAState startState;
public int decision;
/** From what NFAState did we create the DFA? */
public DecisionState decisionNFAStartState;
/** A set of all DFA states. Use Map so
* we can get old state back (Set only allows you to see if it's there).
* Not used during fixed k lookahead as it's a waste to fill it with
* a dup of states array.
*/
public Map<DFAState, DFAState> stateSet = new HashMap<DFAState, DFAState>();
/** Maps the state number to the actual DFAState.
*
* This is managed in parallel with stateSet and simply provides
* a way to go from state number to DFAState rather than via a
* hash lookup.
*/
public List<DFAState> states = new ArrayList<DFAState>();
public int nAlts = 0;
/** accept state(s) per predicted alt; track here */
public List<DFAState>[] altToAcceptStates;
/** Did DFA minimization do anything? */
public boolean minimized;
//public boolean cyclic;
/** Unique state numbers per DFA */
int stateCounter = 0;
public PredictionDFAFactory converter;
public DFA(Grammar g, DecisionState startState) {
this.g = g;
this.decisionNFAStartState = startState;
nAlts = startState.getNumberOfTransitions();
decision = startState.decision;
altToAcceptStates = new ArrayList[nAlts+1]; //(ArrayList<DFAState>[])Array.newInstance(ArrayList.class,nAlts+1);
}
public DFA(Grammar g, int nAlts) {
this.g = g;
this.nAlts = nAlts;
altToAcceptStates = new ArrayList[nAlts+1]; //(ArrayList<DFAState>[])Array.newInstance(ArrayList.class,nAlts+1);
}
/** Add a new DFA state to this DFA (doesn't check if already present). */
public void addState(DFAState d) {
stateSet.put(d,d);
d.stateNumber = stateCounter++;
states.add( d ); // index in states should be d.stateCounter
}
public void addAcceptState(int alt, DFAState acceptState) {
if ( stateSet.get(acceptState)==null ) addState(acceptState);
defineAcceptState(alt, acceptState);
}
public void defineAcceptState(int alt, DFAState acceptState) {
acceptState.isAcceptState = true;
acceptState.predictsAlt = alt;
if ( altToAcceptStates[alt]==null ) {
altToAcceptStates[alt] = new ArrayList<DFAState>();
}
altToAcceptStates[alt].add(acceptState);
}
public DFAState newState() {
DFAState n = new DFAState(this);
return n;
}
public LexerState newLexerState() {
LexerState n = new LexerState(this);
return n;
}
// // could imply converter.unreachableAlts.size()>0 too
// public boolean isAmbiguous() {
// boolean resolvedWithPredicates = true;
// // flip resolvedWithPredicates if we find an ambig state not resolve with pred
// for (DFAState d : converter.ambiguousStates) {
// if ( !d.resolvedWithPredicates ) resolvedWithPredicates = false;
// }
// return converter.ambiguousStates.size()>0 && !resolvedWithPredicates;
// }
public boolean valid() {
return
converter.resolver.danglingStates.size()==0;
// converter.abortedDueToMultipleRecursiveAltsAt ==null &&
// converter.recursionOverflowState ==null;
}
public String toString() {
if ( startState==null ) return "";
DFASerializer serializer = new DFASerializer(g, startState);
return serializer.toString();
}
public static OrderedHashSet<IntervalSet> getReachableLabels(DFAState d) {
OrderedHashSet<IntervalSet> reachableLabels = new OrderedHashSet<IntervalSet>();
for (NFAState s : d.getUniqueNFAStates()) { // for each state
int n = s.getNumberOfTransitions();
for (int i=0; i<n; i++) { // for each transition
Transition t = s.transition(i);
IntervalSet label = t.label();
// if ( t instanceof AtomTransition ) {
// label = IntervalSet.of(((AtomTransition)t).label);
// }
// else if ( t instanceof RangeTransition ) {
// label = ((RangeTransition)t).label();
// }
// else if ( t instanceof SetTransition ) {
// label = ((SetTransition)t).label;
// }
if ( label!=null ) {
addReachableLabel(reachableLabels, label);
}
}
}
//System.out.println("reachable labels for "+d+"="+reachableLabels);
return reachableLabels;
}
/** Add label uniquely and disjointly; intersection with
* another set or int/char forces breaking up the set(s).
*
* Example, if reachable list of labels is [a..z, {k,9}, 0..9],
* the disjoint list will be [{a..j,l..z}, k, 9, 0..8].
*
* As we add NFA configurations to a DFA state, we might as well track
* the set of all possible transition labels to make the DFA conversion
* more efficient. W/o the reachable labels, we'd need to check the
* whole vocabulary space (could be 0..\uFFFE)! The problem is that
* labels can be sets, which may overlap with int labels or other sets.
* As we need a deterministic set of transitions from any
* state in the DFA, we must make the reachable labels set disjoint.
* This operation amounts to finding the character classes for this
* DFA state whereas with tools like flex, that need to generate a
* homogeneous DFA, must compute char classes across all states.
* We are going to generate DFAs with heterogeneous states so we
* only care that the set of transitions out of a single state is
* unique. :)
*
* The idea for adding a new set, t, is to look for overlap with the
* elements of existing list s. Upon overlap, replace
* existing set s[i] with two new disjoint sets, s[i]-t and s[i]&t.
* (if s[i]-t is nil, don't add). The remainder is t-s[i], which is
* what you want to add to the set minus what was already there. The
* remainder must then be compared against the i+1..n elements in s
* looking for another collision. Each collision results in a smaller
* and smaller remainder. Stop when you run out of s elements or
* remainder goes to nil. If remainder is non nil when you run out of
* s elements, then add remainder to the end.
*/
public static void addReachableLabel(OrderedHashSet<IntervalSet> reachableLabels,
IntervalSet label)
{
/*
System.out.println("addReachableLabel to state "+dfa.decisionNumber+"."+stateNumber+": "+label.getSet().toString(dfa.nfa.grammar));
System.out.println("start of add to state "+dfa.decisionNumber+"."+stateNumber+": " +
"reachableLabels="+reachableLabels.toString());
*/
if ( reachableLabels.contains(label) ) { // exact label present
return;
}
IntervalSet remainder = label; // remainder starts out as whole set to add
int n = reachableLabels.size(); // only look at initial elements
// walk the existing list looking for the collision
for (int i=0; i<n; i++) {
IntervalSet rl = reachableLabels.get(i);
/*
System.out.println("comparing ["+i+"]: "+label.toString(dfa.nfa.grammar)+" & "+
rl.toString(dfa.nfa.grammar)+"="+
intersection.toString(dfa.nfa.grammar));
*/
IntervalSet intersection = (IntervalSet)label.and(rl);
if ( intersection.isNil() ) {
continue;
}
//System.out.println(label+" collides with "+rl);
// For any (s_i, t) with s_i&t!=nil replace with (s_i-t, s_i&t)
// (ignoring s_i-t if nil; don't put in list)
// Replace existing s_i with intersection since we
// know that will always be a non nil character class
IntervalSet s_i = rl;
reachableLabels.set(i, intersection);
// Compute s_i-t to see what is in current set and not in incoming
IntervalSet existingMinusNewElements = (IntervalSet)s_i.subtract(label);
//System.out.println(s_i+"-"+t+"="+existingMinusNewElements);
if ( !existingMinusNewElements.isNil() ) {
// found a new character class, add to the end (doesn't affect
// outer loop duration due to n computation a priori.
reachableLabels.add(existingMinusNewElements);
}
/*
System.out.println("after collision, " +
"reachableLabels="+reachableLabels.toString());
*/
// anything left to add to the reachableLabels?
remainder = (IntervalSet)label.subtract(s_i);
if ( remainder.isNil() ) {
break; // nothing left to add to set. done!
}
label = remainder;
}
if ( !remainder.isNil() ) {
/*
System.out.println("before add remainder to state "+dfa.decisionNumber+"."+stateNumber+": " +
"reachableLabels="+reachableLabels.toString());
System.out.println("remainder state "+dfa.decisionNumber+"."+stateNumber+": "+remainder.toString(dfa.nfa.grammar));
*/
reachableLabels.add(remainder);
}
/*
System.out.println("#END of add to state "+dfa.decisionNumber+"."+stateNumber+": " +
"reachableLabels="+reachableLabels.toString());
*/
}
}

71
tool/src/org/antlr/v4/automata/DFASerializer.java
View File

@ -1,71 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.analysis.SemanticContext;
import org.antlr.v4.tool.Grammar;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/** A DFA walker that knows how to dump them to serialized strings. */
public class DFASerializer {
Grammar g;
DFAState start;
public DFASerializer(Grammar g, DFAState start) {
this.g = g;
this.start = start;
}
public String toString() {
if ( start==null ) return null;
Set<Integer> marked = new HashSet<Integer>();
List<DFAState> work = new ArrayList<DFAState>();
work.add(start);
StringBuilder buf = new StringBuilder();
DFAState s = null;
while ( work.size()>0 ) {
s = work.remove(0);
if ( marked.contains(s.stateNumber) ) continue;
marked.add(s.stateNumber);
int n = s.getNumberOfEdges();
//System.out.println("visit "+getStateString(s)+"; edges="+n);
for (int i=0; i<n; i++) {
buf.append(getStateString(s));
Edge t = s.edge(i);
work.add( t.target );
String label = t.toString(g);
SemanticContext preds = t.semanticContext; //t.target.getGatedPredicatesInNFAConfigurations();
if ( preds!=null ) {
String predsStr = "";
predsStr = "&&"+preds.toString();
label += predsStr;
}
buf.append("-"+label+"->"+ getStateString(t.target)+'\n');
}
}
String output = buf.toString();
//return Utils.sortLinesInString(output);
return output;
}
String getStateString(DFAState s) {
int n = s.stateNumber;
String stateStr = "s"+n;
if ( s.isAcceptState ) {
if ( s instanceof LexerState ) {
stateStr = ":s"+n+"=>";
stateStr += ((LexerState)s).predictsRule.name;
}
else {
stateStr = ":s"+n+"=>"+s.getUniquelyPredictedAlt();
}
}
return stateStr;
}
}

278
tool/src/org/antlr/v4/automata/DFAState.java
View File

@ -1,278 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.analysis.NFAConfig;
import org.antlr.v4.analysis.Resolver;
import org.antlr.v4.analysis.SemanticContext;
import org.antlr.v4.misc.IntSet;
import org.antlr.v4.misc.OrderedHashSet;
import java.util.*;
/** A DFA state represents a set of possible NFA configurations.
* As Aho, Sethi, Ullman p. 117 says "The DFA uses its state
* to keep track of all possible states the NFA can be in after
* reading each input symbol. That is to say, after reading
* input a1a2..an, the DFA is in a state that represents the
* subset T of the states of the NFA that are reachable from the
* NFA's start state along some path labeled a1a2..an."
* In conventional NFA->DFA conversion, therefore, the subset T
* would be a bitset representing the set of states the
* NFA could be in. We need to track the alt predicted by each
* state as well, however. More importantly, we need to maintain
* a stack of states, tracking the closure operations as they
* jump from rule to rule, emulating rule invocations (method calls).
* Recall that NFAs do not normally have a stack like a pushdown-machine
* so I have to add one to simulate the proper lookahead sequences for
* the underlying LL grammar from which the NFA was derived.
*
* I use a list of NFAConfig objects. An NFAConfiguration
* is both a state (ala normal conversion) and an NFAContext describing
* the chain of rules (if any) followed to arrive at that state. There
* is also the semantic context, which is the "set" of predicates found
* on the path to this configuration.
*
* A DFA state may have multiple references to a particular state,
* but with different NFAContexts (with same or different alts)
* meaning that state was reached via a different set of rule invocations.
*/
public class DFAState {
public static final int INITIAL_NUM_TRANSITIONS = 4;
public static final int INVALID_STATE_NUMBER = -1;
public int stateNumber = INVALID_STATE_NUMBER;
public boolean isAcceptState = false;
/** If accept, which alt does it predict? */
public int predictsAlt = NFA.INVALID_ALT_NUMBER;
/** State in which DFA? */
public DFA dfa;
/** Track the transitions emanating from this DFA state. */
public List<Edge> edges =
new ArrayList<Edge>(INITIAL_NUM_TRANSITIONS);
/** The set of NFA configurations (state,alt,context) for this DFA state */
public OrderedHashSet<NFAConfig> nfaConfigs = new OrderedHashSet<NFAConfig>();
/** Rather than recheck every NFA configuration in a DFA state (after
* resolving) in reach just check this boolean. Saves a linear walk
* perhaps DFA state creation. Every little bit helps.
*
* This indicates that at least 2 alts were resolved, but not necessarily
* all alts in DFA state configs.
*/
public boolean resolvedWithPredicates = false;
//int cachedUniquelyPredicatedAlt = NFA.INVALID_ALT_NUMBER;
public DFAState() {; }
public DFAState(DFA dfa) {
this.dfa = dfa;
}
public void addNFAConfig(NFAConfig c) {
if ( nfaConfigs.contains(c) ) return;
nfaConfigs.add(c);
}
/** Walk each configuration and if they are all the same alt,
* even the resolved configs.
*/
public int getUniquelyPredictedAlt() {
if ( predictsAlt!=NFA.INVALID_ALT_NUMBER ) return predictsAlt;
predictsAlt = Resolver.getUniqueAlt(nfaConfigs, false);
return predictsAlt;
}
/** Return the uniquely mentioned alt from the NFA configurations, ignoring
* resolved configs
*/
public int getUniqueAlt() { return Resolver.getUniqueAlt(nfaConfigs, true); }
/** Get the set of all alts mentioned by all NFA configurations in this
* DFA state.
*/
public Set<Integer> getAltSet() {
Set<Integer> alts = new HashSet<Integer>();
for (NFAConfig c : nfaConfigs) {
alts.add(c.alt);
}
if ( alts.size()==0 ) return null;
return alts;
}
public int getMinAlt() {
int min = Integer.MAX_VALUE;
for (NFAConfig c : nfaConfigs) {
if ( c.alt < min ) min = c.alt;
}
return min;
}
public Set<NFAState> getUniqueNFAStates() {
return getUniqueNFAStates(NFA.INVALID_ALT_NUMBER);
}
public Set<NFAState> getUniqueNFAStates(int alt) {
OrderedHashSet<NFAState> alts = new OrderedHashSet<NFAState>();
for (NFAConfig c : nfaConfigs) {
if ( alt==NFA.INVALID_ALT_NUMBER || c.alt==alt ) alts.add(c.state);
}
if ( alts.size()==0 ) return null;
return alts;
}
public Map<Integer, SemanticContext> getPredicatesForAlts() {
// map alt to combined SemanticContext
Map<Integer, SemanticContext> altToPredicateContextMap =
new HashMap<Integer, SemanticContext>();
Set<Integer> alts = getAltSet();
for (Integer alt : alts) {
SemanticContext ctx = getPredicatesForAlt(alt);
altToPredicateContextMap.put(alt, ctx);
}
return altToPredicateContextMap;
}
public SemanticContext getPredicatesForAlt(int alt) {
SemanticContext preds = null;
for (NFAConfig c : nfaConfigs) {
if ( c.alt == alt &&
c.semanticContext!=SemanticContext.EMPTY_SEMANTIC_CONTEXT )
{
if ( preds == null ) preds = c.semanticContext;
else preds = SemanticContext.or(preds, c.semanticContext);
}
}
return preds;
}
public List<NFAConfig> getNFAConfigsForAlt(int alt) {
List<NFAConfig> configs = new ArrayList<NFAConfig>();
for (NFAConfig c : nfaConfigs) {
if ( c.alt == alt ) configs.add(c);
}
return configs;
}
/** For gated productions, we need an OR'd list of all predicates for the
* target of an edge so we can gate the edge based upon the predicates
* associated with taking that path (if any).
*
* For syntactic predicates, we only want to generate predicate
* evaluations as we transitions to an accept state; it's a waste to
* do it earlier. So, only add gated preds derived from manually-
* specified syntactic predicates if this is an accept state.
*
* Also, since configurations w/o gated predicates are like true
* gated predicates, finding a configuration whose alt has no gated
* predicate implies we should evaluate the predicate to true. This
* means the whole edge has to be ungated. Consider:
*
* X : ('a' | {p}?=> 'a')
* | 'a' 'b'
* ;
*
* Here, you 'a' gets you from s0 to s1 but you can't test p because
* plain 'a' is ok. It's also ok for starting alt 2. Hence, you can't
* test p. Even on the edge going to accept state for alt 1 of X, you
* can't test p. You can get to the same place with and w/o the context.
* Therefore, it is never ok to test p in this situation.
*/
public SemanticContext getGatedPredicatesInNFAConfigurations() {
SemanticContext unionOfPredicatesFromAllAlts = null;
for (NFAConfig c : nfaConfigs) {
SemanticContext gatedPredExpr =
c.semanticContext.getGatedPredicateContext();
if ( gatedPredExpr==null ) {
// if we ever find a configuration w/o a gated predicate
// (even if it's a nongated predicate), we cannot gate
// the indident edges.
return null;
}
else if ( isAcceptState || !c.semanticContext.isSyntacticPredicate() ) {
// at this point we have a gated predicate and, due to elseif,
// we know it's an accept and not a syn pred. In this case,
// it's safe to add the gated predicate to the union. We
// only want to add syn preds if it's an accept state. Other
// gated preds can be used with edges leading to accept states.
if ( unionOfPredicatesFromAllAlts==null ) {
unionOfPredicatesFromAllAlts = gatedPredExpr;
}
else {
unionOfPredicatesFromAllAlts =
SemanticContext.or(unionOfPredicatesFromAllAlts,gatedPredExpr);
}
}
}
if ( unionOfPredicatesFromAllAlts instanceof SemanticContext.TruePredicate ) {
return null;
}
return unionOfPredicatesFromAllAlts;
}
public int getNumberOfEdges() { return edges.size(); }
public void addEdge(Edge e) { edges.add(e); }
public Edge edge(int i) { return edges.get(i); }
public DFAState target(IntSet label) {
for (Edge e : edges) {
if ( !(e instanceof PredicateEdge) &&
!e.label.and(label).isNil() )
{
return e.target;
}
}
return null;
}
/** A decent hash for a DFA state is the sum of the NFA state/alt pairs. */
public int hashCode() {
int h = 0;
for (NFAConfig c : nfaConfigs) {
h += c.state.stateNumber + c.alt;
}
return h;
}
/** Two DFAStates are equal if their NFA configuration sets are the
* same. This method is used to see if a DFA state already exists.
*
* Because the number of alternatives and number of NFA configurations are
* finite, there is a finite number of DFA states that can be processed.
* This is necessary to show that the algorithm terminates.
*
* Cannot test the DFA state numbers here because in DFA.addState we need
* to know if any other state exists that has this exact set of NFA
* configurations. The DFAState state number is irrelevant.
*/
public boolean equals(Object o) {
// compare set of NFA configurations in this set with other
if ( this==o ) return true;
DFAState other = (DFAState)o;
boolean sameSet = this.nfaConfigs.equals(other.nfaConfigs);
//System.out.println("DFAState.equals: "+nfaConfigs+(sameSet?"==":"!=")+other.nfaConfigs);
return sameSet;
}
/** Print all NFA states plus what alts they predict */
public String toString() {
StringBuffer buf = new StringBuffer();
buf.append(stateNumber+":{");
for (int i = 0; i < nfaConfigs.size(); i++) {
NFAConfig c = (NFAConfig)nfaConfigs.get(i);
if ( i>0 ) {
buf.append(", ");
}
buf.append(c);
}
buf.append("}");
return buf.toString();
}
}

6
tool/src/org/antlr/v4/automata/DecisionState.java
View File

@ -1,6 +0,0 @@
package org.antlr.v4.automata;
public class DecisionState extends BasicState {
public int decision;
public DecisionState(NFA nfa) { super(nfa); }
}

28
tool/src/org/antlr/v4/automata/Edge.java
View File

@ -1,28 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.analysis.SemanticContext;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.tool.Grammar;
/** A DFA edge (NFA edges are called transitions) */
public class Edge {
public IntervalSet label;
public SemanticContext semanticContext; // predicated edge?
public DFAState target;
public Edge(DFAState target) {
this.target = target;
}
public Edge(DFAState target, IntervalSet label) {
this(target);
semanticContext = target.getGatedPredicatesInNFAConfigurations();
this.label = label;
}
public String toString() { return label.toString(); }
public String toString(Grammar g) {
return label!=null?label.toString(g):"";
}
}

11
tool/src/org/antlr/v4/automata/EpsilonTransition.java
View File

@ -1,11 +0,0 @@
package org.antlr.v4.automata;
public class EpsilonTransition extends Transition {
public EpsilonTransition(NFAState target) { super(target); }
public boolean isEpsilon() { return true; }
public int compareTo(Object o) {
return 0;
}
}

84
tool/src/org/antlr/v4/automata/Label.java
View File

@ -1,84 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.runtime.Token;
import org.antlr.v4.misc.IntervalSet;
/** A state machine transition label. A label can be either a simple
* label such as a token or character. A label can be a set of char or
* tokens. It can be an epsilon transition. It can be a semantic predicate
* (which assumes an epsilon transition) or a tree of predicates (in a DFA).
* Special label types have to be < 0 to avoid conflict with char.
*/
public abstract class Label implements /*Comparable, */ Cloneable {
public static final int INVALID = -7;
// public static final int ACTION = -6;
//public static final int EPSILON = -5;
//public static final String EPSILON_STR = "<EPSILON>";
/** label is a semantic predicate; implies label is epsilon also */
// public static final int SEMPRED = -4;
/** label is a set of tokens or char */
// public static final int SET = -3;
/** End of Token is like EOF for lexer rules. It implies that no more
* characters are available and that NFA conversion should terminate
* for this path. For example
*
* A : 'a' 'b' | 'a' ;
*
* yields a DFA predictor:
*
* o-a->o-b->1 predict alt 1
* |
* |-EOT->o predict alt 2
*
* To generate code for EOT, treat it as the "default" path, which
* implies there is no way to mismatch a char for the state from
* which the EOT emanates.
*/
public static final int EOT = -2;
public static final int EOF = -1;
/** We have labels like EPSILON that are below 0; it's hard to
* store them in an array with negative index so use this
* constant as an index shift when accessing arrays based upon
* token type. If real token type is i, then array index would be
* NUM_FAUX_LABELS + i.
*/
public static final int NUM_FAUX_LABELS = -INVALID;
/** Anything at this value or larger can be considered a simple atom int
* for easy comparison during analysis only; faux labels are not used
* during parse time for real token types or char values.
*/
public static final int MIN_ATOM_VALUE = EOT;
public static final int MIN_CHAR_VALUE = '\u0000';
public static final int MAX_CHAR_VALUE = '\uFFFE';
/** End of rule token type; imaginary token type used only for
* local, partial FOLLOW sets to indicate that the local FOLLOW
* hit the end of rule. During error recovery, the local FOLLOW
* of a token reference may go beyond the end of the rule and have
* to use FOLLOW(rule). I have to just shift the token types to 2..n
* rather than 1..n to accommodate this imaginary token in my bitsets.
* If I didn't use a bitset implementation for runtime sets, I wouldn't
* need this. EOF is another candidate for a run time token type for
* parsers. Follow sets are not computed for lexers so we do not have
* this issue.
*/
public static final int EOR_TOKEN_TYPE = Token.EOR_TOKEN_TYPE;
public int atom = Label.INVALID;
public IntervalSet set;
public int compareTo(Object o) {
return 0; // TODO: impl
}
}

10
tool/src/org/antlr/v4/automata/LexerDFA.java
View File

@ -1,10 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.tool.Grammar;
/** TODO: do we need? */
public class LexerDFA extends DFA {
public LexerDFA(Grammar g, DecisionState startState) {
super(g, startState);
}
}

80
tool/src/org/antlr/v4/automata/LexerNFAFactory.java
View File

@ -1,80 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.misc.CharSupport;
import org.antlr.v4.tool.GrammarAST;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.TerminalAST;
import java.util.List;
public class LexerNFAFactory extends ParserNFAFactory {
public LexerNFAFactory(LexerGrammar g) { super(g); }
public NFA createNFA() {
// BUILD ALL START STATES (ONE PER MODE)
for (String modeName : ((LexerGrammar)g).modes.keySet()) {
// create s0, start state; implied Tokens rule node
TokensStartState startState =
(TokensStartState)newState(TokensStartState.class, null);
nfa.modeToStartState.put(modeName, startState);
nfa.defineDecisionState(startState);
}
// CREATE NFA FOR EACH RULE
_createNFA(g.rules.values());
// LINK MODE START STATE TO EACH TOKEN RULE
for (String modeName : ((LexerGrammar)g).modes.keySet()) {
List<Rule> rules = ((LexerGrammar)g).modes.get(modeName);
TokensStartState startState = nfa.modeToStartState.get(modeName);
for (Rule r : rules) {
if ( !r.isFragment() ) {
RuleStartState s = nfa.ruleToStartState.get(r);
epsilon(startState, s);
}
}
}
return nfa;
}
@Override
public Handle range(GrammarAST a, GrammarAST b) {
BasicState left = newState(a);
BasicState right = newState(b);
int t1 = CharSupport.getCharValueFromGrammarCharLiteral(a.getText());
int t2 = CharSupport.getCharValueFromGrammarCharLiteral(b.getText());
left.transition = new RangeTransition(t1, t2, right);
a.nfaState = left;
b.nfaState = left;
return new Handle(left, right);
}
/** For a lexer, a string is a sequence of char to match. That is,
* "fog" is treated as 'f' 'o' 'g' not as a single transition in
* the DFA. Machine== o-'f'->o-'o'->o-'g'->o and has n+1 states
* for n characters.
*/
@Override
public Handle stringLiteral(TerminalAST stringLiteralAST) {
String chars = stringLiteralAST.getText();
chars = CharSupport.getStringFromGrammarStringLiteral(chars);
int n = chars.length();
BasicState left = newState(stringLiteralAST);
BasicState prev = left;
BasicState right = null;
for (int i=0; i<n; i++) {
right = newState(stringLiteralAST);
prev.transition = new AtomTransition(chars.charAt(i), right);
prev = right;
}
stringLiteralAST.nfaState = left;
return new Handle(left, right);
}
@Override
public Handle tokenRef(TerminalAST node) {
return ruleRef(node);
}
}

57
tool/src/org/antlr/v4/automata/LexerState.java
View File

@ -1,57 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.tool.ActionAST;
import org.antlr.v4.tool.Rule;
public class LexerState extends DFAState {
/** For ambiguous lexer rules, the accept state matches a set of rules,
* not just one. So, gives precedence to keywords vs IDs if keywords are first.
public List<Rule> matchesRules = new ArrayList<Rule>();
*/
public Rule predictsRule;
/** Single action sitting at extreme right edge of lexer rule */
public ActionAST action;
public LexerState(DFA dfa) {
super(dfa);
}
// public Set<NFAState> getUniqueNFAStates() { return nfaStates; }
//
// public Set<Integer> getAltSet() { return null; }
//
// /** Two LexerStates are equal if their NFA state lists are the
// * same. Don't test the DFA state numbers here because
// * we use to know if any other state exists that has this exact set
// * of states. The DFAState state number is irrelevant.
// */
// public boolean equals(Object o) {
// // compare set of NFA configurations in this set with other
// if ( this==o ) return true;
// LexerState other = (LexerState)o;
// return this.nfaStates.equals(other.nfaStates);
// }
//
// public int hashCode() {
// int h = 0;
// for (NFAState s : nfaStates) h += s.stateNumber;
// return h;
// }
//
// /** Print all NFA states plus what alts they predict */
// public String toString() {
// StringBuffer buf = new StringBuffer();
// buf.append(stateNumber+":{");
// for (int i = 0; i < nfaStates.size(); i++) {
// NFAState s = nfaStates.get(i);
// if ( i>0 ) {
// buf.append(", ");
// }
// buf.append(s);
// }
// buf.append("}");
// return buf.toString();
// }
}

33
tool/src/org/antlr/v4/automata/LoopbackState.java
View File

@ -1,33 +0,0 @@
package org.antlr.v4.automata;
/** */
public class LoopbackState extends DecisionState {
EpsilonTransition loopBack; // edge 2 (transition is edge 1)
public LoopbackState(NFA nfa) { super(nfa); }
@Override
public int getNumberOfTransitions() {
int n = 0;
if ( transition!=null ) n++;
if ( loopBack!=null ) n++;
return n;
}
@Override
public void addTransition(Transition e) {
if ( getNumberOfTransitions()>=2 ) throw new IllegalArgumentException("only two transitions");
if ( transition==null ) transition = e;
else loopBack = (EpsilonTransition)e;
}
@Override
public Transition transition(int i) {
if ( i>=2 ) throw new IllegalArgumentException("only two transitions");
if ( i==1 ) return transition;
return loopBack;
}
@Override
public boolean onlyHasEpsilonTransitions() { return true; }
}

44
tool/src/org/antlr/v4/automata/NFA.java
View File

@ -1,44 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/** */
public class NFA {
public static final int INVALID_ALT_NUMBER = -1;
public static final int INVALID_DECISION_NUMBER = -1;
public Grammar g;
public List<NFAState> states = new ArrayList<NFAState>();
/** Each subrule/rule is a decision point and we must track them so we
* can go back later and build DFA predictors for them. This includes
* all the rules, subrules, optional blocks, ()+, ()* etc...
*/
public List<DecisionState> decisionToNFAState = new ArrayList<DecisionState>();
public Map<Rule, RuleStartState> ruleToStartState = new LinkedHashMap<Rule, RuleStartState>();
public Map<Rule, RuleStopState> ruleToStopState = new LinkedHashMap<Rule, RuleStopState>();
public Map<String, TokensStartState> modeToStartState =
new LinkedHashMap<String, TokensStartState>();
int stateNumber = 0;
public NFA(Grammar g) { this.g = g; }
public void addState(NFAState state) {
states.add(state);
state.stateNumber = stateNumber++;
}
public int defineDecisionState(DecisionState s) {
decisionToNFAState.add(s);
s.decision = decisionToNFAState.size()-1;
return s.decision;
}
}

193
tool/src/org/antlr/v4/automata/NFAFactory.java
View File

@ -1,193 +0,0 @@
package org.antlr.v4.automata;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.tool.GrammarAST;
import org.antlr.v4.tool.TerminalAST;
import java.util.List;
public interface NFAFactory {
/** A pair of states pointing to the left/right (start and end) states of a
* state submachine. Used to build NFAs.
*/
public static class Handle {
public NFAState left;
public NFAState right;
public Handle(NFAState left, NFAState right) {
this.left = left;
this.right = right;
}
@Override
public String toString() {
return "("+left+","+right+")";
}
}
NFA createNFA();
void setCurrentRuleName(String name);
Handle rule(GrammarAST ruleAST, String name, Handle blk);
NFAState newState();
Handle label(Handle t);
Handle listLabel(Handle t);
Handle tokenRef(TerminalAST node);
/** From set build single edge graph o->o-set->o. To conform to
* what an alt block looks like, must have extra state on left.
*/
Handle set(IntervalSet set, GrammarAST associatedAST);
Handle tree(List<Handle> els);
Handle range(GrammarAST a, GrammarAST b);
Handle not(GrammarAST a, Handle A);
/** For a non-lexer, just build a simple token reference atom.
* For a lexer, a string is a sequence of char to match. That is,
* "fog" is treated as 'f' 'o' 'g' not as a single transition in
* the DFA. Machine== o-'f'->o-'o'->o-'g'->o and has n+1 states
* for n characters.
*/
Handle stringLiteral(TerminalAST stringLiteralAST);
/** For reference to rule r, build
*
* o-e->(r) o
*
* where (r) is the start of rule r and the trailing o is not linked
* to from rule ref state directly (it's done thru the transition(0)
* RuleClosureTransition.
*
* If the rule r is just a list of tokens, it's block will be just
* a set on an edge o->o->o-set->o->o->o, could inline it rather than doing
* the rule reference, but i'm not doing this yet as I'm not sure
* it would help much in the NFA->DFA construction.
*
* TODO add to codegen: collapse alt blks that are sets into single matchSet
* @param node
*/
Handle ruleRef(GrammarAST node);
/** From an empty alternative build Grip o-e->o */
Handle epsilon(GrammarAST node);
/** Build what amounts to an epsilon transition with a semantic
* predicate action. The pred is a pointer into the AST of
* the SEMPRED token.
*/
Handle sempred(GrammarAST pred);
Handle gated_sempred(GrammarAST pred);
/** Build what amounts to an epsilon transition with an action.
* The action goes into NFA though it is ignored during analysis.
* It slows things down a bit, but I must ignore predicates after
* having seen an action (5-5-2008).
*/
Handle action(GrammarAST action);
/** From a set ('a'|'b') build
*
* o->o-'a'..'b'->o->o (last NFAState is blockEndNFAState pointed to by all alts)
*/
Handle blockFromSet(Handle set);
Handle alt(List<Handle> els);
/** From A|B|..|Z alternative block build
*
* o->o-A->o->o (last NFAState is blockEndNFAState pointed to by all alts)
* | ^
* o->o-B->o--|
* | |
* ... |
* | |
* o->o-Z->o--|
*
* So every alternative gets begin NFAState connected by epsilon
* and every alt right side points at a block end NFAState. There is a
* new NFAState in the NFAState in the Grip for each alt plus one for the
* end NFAState.
*
* Special case: only one alternative: don't make a block with alt
* begin/end.
*
* Special case: if just a list of tokens/chars/sets, then collapse
* to a single edge'd o-set->o graph.
*
* Set alt number (1..n) in the left-Transition NFAState.
*/
Handle block(GrammarAST blockAST, List<Handle> alternativeGrips);
/** From (A)? build either:
*
* o--A->o
* | ^
* o---->|
*
* or, if A is a block, just add an empty alt to the end of the block
*/
Handle optional(GrammarAST optAST, Handle blk);
/** From (A)+ build
*
* |---| (Transition 2 from A.right points at alt 1)
* v | (follow of loop is Transition 1)
* o->o-A-o->o
*
* Meaning that the last NFAState in A points back to A's left Transition NFAState
* and we add a new begin/end NFAState. A can be single alternative or
* multiple.
*
* During analysis we'll call the follow link (transition 1) alt n+1 for
* an n-alt A block.
*/
Handle plus(GrammarAST plusAST, Handle blk);
/** From (A)* build
*
* |---|
* v |
* o->o-A-o--o (Transition 2 from block end points at alt 1; follow is Transition 1)
* | ^
* o---------| (optional branch is 2nd alt of optional block containing A+)
*
* Meaning that the last (end) NFAState in A points back to A's
* left side NFAState and we add 3 new NFAStates (the
* optional branch is built just like an optional subrule).
* See the Aplus() method for more on the loop back Transition.
* The new node on right edge is set to RIGHT_EDGE_OF_CLOSURE so we
* can detect nested (A*)* loops and insert an extra node. Previously,
* two blocks shared same EOB node.
*
* There are 2 or 3 decision points in a A*. If A is not a block (i.e.,
* it only has one alt), then there are two decisions: the optional bypass
* and then loopback. If A is a block of alts, then there are three
* decisions: bypass, loopback, and A's decision point.
*
* Note that the optional bypass must be outside the loop as (A|B)* is
* not the same thing as (A|B|)+.
*
* This is an accurate NFA representation of the meaning of (A)*, but
* for generating code, I don't need a DFA for the optional branch by
* virtue of how I generate code. The exit-loopback-branch decision
* is sufficient to let me make an appropriate enter, exit, loop
* determination. See codegen.g
*/
Handle star(GrammarAST starAST, Handle blk);
/** Build an atom with all possible values in its label */
Handle wildcard(GrammarAST associatedAST);
/** Build a subrule matching ^(. .*) (any tree or node). Let's use
* (^(. .+) | .) to be safe.
*/
Handle wildcardTree(GrammarAST associatedAST);
}

Some files were not shown because too many files have changed in this diff Show More