jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge pull request #1546 from sharwell/fix-1545 

Fix multiple problems with optional block bypass at end of rule
This commit is contained in:
Terence Parr 2016-12-27 21:47:12 -08:00 committed by GitHub
parent 644478a015 24b20e236f
commit b57843d983
18 changed files with 103 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserErrorsDescriptors.java
View File

@ -135,7 +135,7 @@ public class ParserErrorsDescriptors {
public static class InvalidEmptyInput extends BaseParserTestDescriptor {
public String input = "";
public String output = null;
public String errors = "line 1:0 missing ID at '<EOF>'\n";
public String errors = "line 1:0 mismatched input '<EOF>' expecting ID\n";
public String startRule = "start";
public String grammarName = "T";
@ -414,7 +414,7 @@ public class ParserErrorsDescriptors {
/**
grammar T;
a : 'a' 'b'* ;
a : 'a' 'b'* EOF ;
*/
@CommentHasStringValue
public String grammar;
@ -436,7 +436,7 @@ public class ParserErrorsDescriptors {
/**
grammar T;
a : 'a' ('b'|'z'{<Pass()>})*;
a : 'a' ('b'|'z'{<Pass()>})* EOF ;
*/
@CommentHasStringValue
public String grammar;

77
runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java
View File

@ -753,6 +753,83 @@ public class ParserExecDescriptors {
*/
@CommentHasStringValue
public String grammar;
}
/**
* This is a regression test for antlr/antlr4#1545, case 1.
*/
public static class OpenDeviceStatement_Case1 extends BaseParserTestDescriptor {
public String input = "OPEN DEVICE DEVICE";
public String output = "OPEN DEVICE DEVICE\n";
public String errors = null;
public String startRule = "statement";
public String grammarName = "OpenDeviceStatement";
/**
grammar OpenDeviceStatement;
program : statement+ '.' ;
statement : 'OPEN' ( 'DEVICE' ( OPT1 | OPT2 | OPT3 )? )+ {<writeln("$text")>} ;
OPT1 : 'OPT-1';
OPT2 : 'OPT-2';
OPT3 : 'OPT-3';
WS : (' '|'\n')+ -> channel(HIDDEN);
*/
@CommentHasStringValue
public String grammar;
}
/**
* This is a regression test for antlr/antlr4#1545, case 2.
*/
public static class OpenDeviceStatement_Case2 extends BaseParserTestDescriptor {
public String input = "OPEN DEVICE DEVICE";
public String output = "OPEN DEVICE DEVICE\n";
public String errors = null;
public String startRule = "statement";
public String grammarName = "OpenDeviceStatement";
/**
grammar OpenDeviceStatement;
program : statement+ '.' ;
statement : 'OPEN' ( 'DEVICE' ( (OPT1) | OPT2 | OPT3 )? )+ {<writeln("$text")>} ;
OPT1 : 'OPT-1';
OPT2 : 'OPT-2';
OPT3 : 'OPT-3';
WS : (' '|'\n')+ -> channel(HIDDEN);
*/
@CommentHasStringValue
public String grammar;
}
/**
* This is a regression test for antlr/antlr4#1545, case 3.
*/
public static class OpenDeviceStatement_Case3 extends BaseParserTestDescriptor {
public String input = "OPEN DEVICE DEVICE.";
public String output = "OPEN DEVICE DEVICE\n";
public String errors = null;
public String startRule = "statement";
public String grammarName = "OpenDeviceStatement";
/**
grammar OpenDeviceStatement;
program : statement+ '.' ;
statement : 'OPEN' ( 'DEVICE' ( (OPT1) | OPT2 | OPT3 )? )+ {<writeln("$text")>} ;
OPT1 : 'OPT-1';
OPT2 : 'OPT-2';
OPT3 : 'OPT-3';
WS : (' '|'\n')+ -> channel(HIDDEN);
*/
@CommentHasStringValue
public String grammar;
}
}

8
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/DefaultErrorStrategy.cs
View File

@ -265,12 +265,8 @@ namespace Antlr4.Runtime
ITokenStream tokens = ((ITokenStream)recognizer.InputStream);
int la = tokens.LA(1);
// try cheaper subset first; might get lucky. seems to shave a wee bit off
if (recognizer.Atn.NextTokens(s).Contains(la) || la == TokenConstants.EOF)
{
return;
}
// Return but don't end recovery. only do that upon valid token match
if (recognizer.IsExpectedToken(la))
var nextTokens = recognizer.Atn.NextTokens(s);
if (nextTokens.Contains(TokenConstants.EPSILON) || nextTokens.Contains(la))
{
return;
}

8
runtime/Cpp/runtime/src/DefaultErrorStrategy.cpp
View File

@ -100,12 +100,8 @@ void DefaultErrorStrategy::sync(Parser *recognizer) {
size_t la = tokens->LA(1);
// try cheaper subset first; might get lucky. seems to shave a wee bit off
if (recognizer->getATN().nextTokens(s).contains(la) || la == Token::EOF) {
return;
}
// Return but don't end recovery. only do that upon valid token match
if (recognizer->isExpectedToken((int)la)) {
auto nextTokens = recognizer->getATN().nextTokens(s);
if (nextTokens.contains(Token::EPSILON) || nextTokens.contains(la)) {
return;
}

7
runtime/Go/antlr/error_strategy.go
View File

@ -218,11 +218,8 @@ func (d *DefaultErrorStrategy) Sync(recognizer Parser) {
la := recognizer.GetTokenStream().LA(1)
// try cheaper subset first might get lucky. seems to shave a wee bit off
if la == TokenEOF || recognizer.GetATN().NextTokens(s, nil).contains(la) {
return
}
// Return but don't end recovery. only do that upon valid token Match
if recognizer.IsExpectedToken(la) {
nextTokens := recognizer.GetATN().NextTokens(s, nil)
if nextTokens.contains(TokenEpsilon) || nextTokens.contains(la) {
return
}

6
runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java
View File

@ -224,10 +224,8 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
int la = tokens.LA(1);
// try cheaper subset first; might get lucky. seems to shave a wee bit off
if ( recognizer.getATN().nextTokens(s).contains(la) || la==Token.EOF ) return;
// Return but don't end recovery. only do that upon valid token match
if (recognizer.isExpectedToken(la)) {
IntervalSet nextTokens = recognizer.getATN().nextTokens(s);
if (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la)) {
return;
}

7
runtime/JavaScript/src/antlr4/error/ErrorStrategy.js
View File

@ -223,11 +223,8 @@ DefaultErrorStrategy.prototype.sync = function(recognizer) {
var s = recognizer._interp.atn.states[recognizer.state];
var la = recognizer.getTokenStream().LA(1);
// try cheaper subset first; might get lucky. seems to shave a wee bit off
if (la===Token.EOF || recognizer.atn.nextTokens(s).contains(la)) {
return;
}
// Return but don't end recovery. only do that upon valid token match
if(recognizer.isExpectedToken(la)) {
var nextTokens = recognizer.atn.nextTokens(s);
if (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la)) {
return;
}
switch (s.stateType) {

7
runtime/Python2/src/antlr4/error/ErrorStrategy.py
View File

@ -202,11 +202,8 @@ class DefaultErrorStrategy(ErrorStrategy):
s = recognizer._interp.atn.states[recognizer.state]
la = recognizer.getTokenStream().LA(1)
# try cheaper subset first; might get lucky. seems to shave a wee bit off
if la==Token.EOF or la in recognizer.atn.nextTokens(s):
return
# Return but don't end recovery. only do that upon valid token match
if recognizer.isExpectedToken(la):
nextTokens = recognizer.atn.nextTokens(s)
if Token.EPSILON in nextTokens or la in nextTokens:
return
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,

7
runtime/Python3/src/antlr4/error/ErrorStrategy.py
View File

@ -207,11 +207,8 @@ class DefaultErrorStrategy(ErrorStrategy):
s = recognizer._interp.atn.states[recognizer.state]
la = recognizer.getTokenStream().LA(1)
# try cheaper subset first; might get lucky. seems to shave a wee bit off
if la==Token.EOF or la in recognizer.atn.nextTokens(s):
return
# Return but don't end recovery. only do that upon valid token match
if recognizer.isExpectedToken(la):
nextTokens = recognizer.atn.nextTokens(s)
if Token.EPSILON in nextTokens or la in nextTokens:
return
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,

5
runtime/Swift/Antlr4/org/antlr/v4/runtime/DefaultErrorStrategy.swift
View File

@ -226,12 +226,11 @@ public class DefaultErrorStrategy: ANTLRErrorStrategy {
// try cheaper subset first; might get lucky. seems to shave a wee bit off
//let set : IntervalSet = recognizer.getATN().nextTokens(s)
if try recognizer.getATN().nextTokens(s).contains(la) || la == CommonToken.EOF {
if try recognizer.getATN().nextTokens(s).contains(CommonToken.EPSILON) {
return
}
// Return but don't end recovery. only do that upon valid token match
if try recognizer.isExpectedToken(la) {
if try recognizer.getATN().nextTokens(s).contains(la) {
return
}

2
tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg
View File

@ -566,7 +566,7 @@ switch (TokenStream.LA(1)) {
<alt>
break;}; separator="\n">
default:
<error>
break;
}
>>

2
tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
View File

@ -644,7 +644,7 @@ switch (_input->LA(1)) {
\}
}; separator="\n">
default:
<error>
break;
}
>>

3
tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg
View File

@ -563,9 +563,6 @@ switch p.GetTokenStream().LA(1) {
<endif>
default:
<if(error)>
<error>
<endif>
}
>>

2
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -489,7 +489,7 @@ switch (_input.LA(1)) {
<alt>
break;}; separator="\n">
default:
<error>
break;
}
>>

2
tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg
View File

@ -360,7 +360,7 @@ switch (this._input.LA(1)) {
<alt>
break;}; separator="\n">
default:
<error>
break;
}
>>

2
tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
View File

@ -334,7 +334,7 @@ token = self._input.LA(1)
<alt>
pass}; separator="\nel">
else:
<error>
pass
>>
LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= <<

2
tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
View File

@ -342,7 +342,7 @@ token = self._input.LA(1)
<alt>
pass}; separator="\nel">
else:
<error>
pass
>>
LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= <<

2
tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg
View File

@ -510,7 +510,7 @@ switch (<parser.name>.Tokens(rawValue: try _input.LA(1))!) {
<alt>
break}; separator="\n">
default:
<error>
break
}
>>