From 814a1821beee549a1352f7877d1260890811c7f3 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Mon, 30 May 2016 16:27:12 +0200 Subject: [PATCH] TestParserErrors passes now. Fixed a bug in the Cpp.stg file where more input was consumed than should be. --- .../antlr/v4/test/runtime/cpp/Cpp.test.stg | 141 +++++++++++------- .../v4/test/runtime/cpp/TestParserErrors.java | 31 ++-- .../v4/tool/templates/codegen/Cpp/Cpp.stg | 7 +- 3 files changed, 105 insertions(+), 74 deletions(-) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/cpp/Cpp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/cpp/Cpp.test.stg index f0c0e643b..c836754c1 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/cpp/Cpp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/cpp/Cpp.test.stg @@ -224,13 +224,13 @@ ToStringTree(s) ::= <%->toStringTree(this)%> Column() ::= "column" -Text() ::= "text" +Text() ::= "getText()" ValEquals(a,b) ::= <% == %> -TextEquals(a) ::= <%text == ""%> +TextEquals(a) ::= <%getText() == ""%> -PlusText(a) ::= <%"" + text%> +PlusText(a) ::= <%"" + getText()%> InputText() ::= "_input->getText()" @@ -242,7 +242,7 @@ TokenStartColumnEquals(i) ::= <%_tokenStartColumn == %> ImportListener(X) ::= "" -GetExpectedTokenNames() ::= "getExpectedTokens()->toString(literalNames, symbolicNames)" +GetExpectedTokenNames() ::= "getExpectedTokens().toString(_tokenNames)" RuleInvocationStack() ::= "str_list(getRuleInvocationStack())" @@ -258,70 +258,96 @@ bool Parser::Property() { >> PositionAdjustingLexer() ::= << +protected: + class PositionAdjustingLexerATNSimulator : public atn::LexerATNSimulator { + public: + PositionAdjustingLexerATNSimulator(Lexer *recog, const atn::ATN &atn, std::vector\ &decisionToDFA, + Ref\ sharedContextCache) + : atn::LexerATNSimulator(recog, atn, decisionToDFA, sharedContextCache) { + } -def resetAcceptPosition(self, index, line, column): - _input->seek(index) - this->line = line - this->column = column - _interp.consume(_input) + void resetAcceptPosition(CharStream *input, int index, int line, int charPositionInLine) { + input->seek(index); + _line = line; + _charPositionInLine = charPositionInLine; + consume(input); + } -def nextToken(self): - if _interp.__dict__.get("resetAcceptPosition", None) is None: - _interp.__dict__["resetAcceptPosition"] = resetAcceptPosition - return super(type(self),self).nextToken() + }; + +public: + virtual Ref\ nextToken() override { + if (dynamic_cast\(_interpreter) == nullptr) { + delete _interpreter; + _interpreter = new PositionAdjustingLexerATNSimulator(this, _atn, _decisionToDFA, _sharedContextCache); + } -def emit(self): - if _type==PositionAdjustingLexer.TOKENS: - handleAcceptPositionForKeyword("tokens") - elif _type==PositionAdjustingLexer.LABEL: - handleAcceptPositionForIdentifier() - return super(type(self),self).emit() + return Lexer::nextToken(); + } -def handleAcceptPositionForIdentifier(self): - tokenText = text - identifierLength = 0 - while identifierLength \< len(tokenText) and isIdentifierChar(tokenText[identifierLength]): - identifierLength += 1 + virtual Ref\ emit() override { + switch (type) { + case TOKENS: + handleAcceptPositionForKeyword("tokens"); + break; - if _input->index > _tokenStartCharIndex + identifierLength: - offset = identifierLength - 1 - _interp.resetAcceptPosition(_tokenStartCharIndex + offset, - _tokenStartLine, _tokenStartColumn + offset) - return true - else: - return false + case LABEL: + handleAcceptPositionForIdentifier(); + break; + default: + break; + } + return Lexer::emit(); + } -def handleAcceptPositionForKeyword(self, keyword): - if _input->index > _tokenStartCharIndex + len(keyword): - offset = len(keyword) - 1 - _interp.resetAcceptPosition(_tokenStartCharIndex + offset, - _tokenStartLine, _tokenStartColumn + offset) - return true - else: - return false +private: + bool handleAcceptPositionForIdentifier() { + std::string tokenText = getText(); + int identifierLength = 0; + while (identifierLength \< tokenText.length() && isIdentifierChar(tokenText[identifierLength])) { + identifierLength++; + } -@staticmethod -def isIdentifierChar(c): - return c.isalnum() or c == '_' + if (getInputStream()->index() > tokenStartCharIndex + identifierLength) { + int offset = identifierLength - 1; + getInterpreter\()->resetAcceptPosition(getInputStream(), + tokenStartCharIndex + offset, tokenStartLine, tokenStartCharPositionInLine + offset); + return true; + } + return false; + } + + bool handleAcceptPositionForKeyword(const std::string &keyword) { + if (getInputStream()->index() > tokenStartCharIndex + keyword.length()) { + long offset = keyword.size() - 1; + getInterpreter\()->resetAcceptPosition(getInputStream(), + tokenStartCharIndex + offset, tokenStartLine, tokenStartCharPositionInLine + offset); + return true; + } + + return false; + } + + static bool isIdentifierChar(char c) { + return std::isalnum(c) || c == '_'; + } + +public: >> BasicListener(X) ::= << -if __name__ is not None and "." in __name__: - from .Listener import Listener -else: - from Listener import Listener - -class LeafListener(TListener): - def visitTerminal(self, node): - std::cout \<\< node.symbol.text; - +class LeafListener : public TBaseListener { +public: + virtual void visitTerminal(Ref\ node) override { + std::cout \<\< node->getSymbol()->getText() \<\< std::endl; + } +}; >> WalkListener(s) ::= << -walker = ParseTreeWalker() -walker.walk(TParser.LeafListener(), ) +tree::ParseTreeWalker::DEFAULT->walk(std::make_shared\(), ); >> TreeNodeWithAltNumField(X) ::= << @@ -400,10 +426,11 @@ class LeafListener(TListener): >> DeclareContextListGettersFunction() ::= << -def foo(): - s = SContext() - a = s.a() - b = s.b() +void foo() { + Ref\ s; + std::vector\> a = s->a(); + std::vector\> b = s->b(); +} >> Declare_foo() ::= <toString() << std::endl;} 'c' ;"); String grammar = grammarBuilder.toString(); @@ -33,9 +33,9 @@ public class TestParserErrors extends BaseCppTest { public void testConjuringUpTokenFromSet() throws Exception { mkdir(tmpdir); - StringBuilder grammarBuilder = new StringBuilder(82); + StringBuilder grammarBuilder = new StringBuilder(94); grammarBuilder.append("grammar T;\n"); - grammarBuilder.append("a : 'a' x=('b'|'c') {std::cout << \"conjured=\" + str($x) << \"\\n\";} 'd' ;"); + grammarBuilder.append("a : 'a' x=('b'|'c') {std::cout << \"conjured=\" + $x->toString() << std::endl;} 'd' ;"); String grammar = grammarBuilder.toString(); @@ -53,13 +53,14 @@ public class TestParserErrors extends BaseCppTest { public void testContextListGetters() throws Exception { mkdir(tmpdir); - StringBuilder grammarBuilder = new StringBuilder(160); + StringBuilder grammarBuilder = new StringBuilder(218); grammarBuilder.append("grammar T;\n"); grammarBuilder.append("@parser::members{\n"); - grammarBuilder.append("def foo():\n"); - grammarBuilder.append(" s = SContext()\n"); - grammarBuilder.append(" a = s.a()\n"); - grammarBuilder.append(" b = s.b()\n"); + grammarBuilder.append("void foo() {\n"); + grammarBuilder.append(" Ref s;\n"); + grammarBuilder.append(" std::vector> a = s->a();\n"); + grammarBuilder.append(" std::vector> b = s->b();\n"); + grammarBuilder.append("}\n"); grammarBuilder.append("}\n"); grammarBuilder.append("s : (a | b)+;\n"); grammarBuilder.append("a : 'a' {std::cout << 'a';};\n"); @@ -211,7 +212,7 @@ public class TestParserErrors extends BaseCppTest { public void testLL1ErrorInfo() throws Exception { mkdir(tmpdir); - StringBuilder grammarBuilder = new StringBuilder(314); + StringBuilder grammarBuilder = new StringBuilder(303); grammarBuilder.append("grammar T;\n"); grammarBuilder.append("start : animal (AND acClass)? service EOF;\n"); grammarBuilder.append("animal : (DOG | CAT );\n"); @@ -224,7 +225,7 @@ public class TestParserErrors extends BaseCppTest { grammarBuilder.append("WS : ' ' -> skip ;\n"); grammarBuilder.append("acClass\n"); grammarBuilder.append("@init\n"); - grammarBuilder.append("{std::cout << getExpectedTokens().toString(literalNames, symbolicNames) << \"\\n\";}\n"); + grammarBuilder.append("{std::cout << getExpectedTokens().toString(_tokenNames) << std::endl;}\n"); grammarBuilder.append(" : ;"); String grammar = grammarBuilder.toString(); @@ -440,10 +441,10 @@ public class TestParserErrors extends BaseCppTest { public void testSingleSetInsertionConsumption() throws Exception { mkdir(tmpdir); - StringBuilder grammarBuilder = new StringBuilder(95); + StringBuilder grammarBuilder = new StringBuilder(107); grammarBuilder.append("grammar T;\n"); grammarBuilder.append("myset: ('b'|'c') ;\n"); - grammarBuilder.append("a: 'a' myset 'd' {std::cout << \"\" + str($myset.stop) << \"\\n\";} ; "); + grammarBuilder.append("a: 'a' myset 'd' {std::cout << \"\" + $myset.stop->toString() << std::endl;} ; "); String grammar = grammarBuilder.toString(); @@ -571,10 +572,10 @@ public class TestParserErrors extends BaseCppTest { public void testSingleTokenDeletionConsumption() throws Exception { mkdir(tmpdir); - StringBuilder grammarBuilder = new StringBuilder(95); + StringBuilder grammarBuilder = new StringBuilder(107); grammarBuilder.append("grammar T;\n"); grammarBuilder.append("myset: ('b'|'c') ;\n"); - grammarBuilder.append("a: 'a' myset 'd' {std::cout << \"\" + str($myset.stop) << \"\\n\";} ; "); + grammarBuilder.append("a: 'a' myset 'd' {std::cout << \"\" + $myset.stop->toString() << std::endl;} ; "); String grammar = grammarBuilder.toString(); diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg index 8e9808528..dbf68b400 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg @@ -809,8 +809,9 @@ setState(); if ( \<= 0 || !()) { = }>_errHandler->recoverInline(this); +} else { + consume(); } -consume(); >> WildcardHeader(w) ::= "" @@ -907,8 +908,10 @@ TokenPropertyRef_index(t) ::= "(-> != nullptr ? ->