forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Fixed token toString() so it produces the output expected by the tests.

This commit is contained in:
Mike Lischke 2016-05-26 12:20:26 +02:00
parent f103e05bc3
commit 27b76b5a33
12 changed files with 115 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
runtime/Cpp/demo/Mac/antlr4-cpp-demo/main.cpp
View File

@ -19,17 +19,16 @@ int main(int , const char **) {
ANTLRInputStream input(u8"🍴 = 🍐 + \"😎\";(((x * π))) * µ + ∰; a + (x * (y ? 0 : 1) + z);");
TLexer lexer(&input);
CommonTokenStream tokens(&lexer);
/*
tokens.fill();
for (auto token : tokens.getTokens()) {
std::cout << token->toString() << std::endl;
}
*/
TParser parser(&tokens);
Ref<tree::ParseTree> tree = parser.main();
std::cout << tree->toStringTree(&parser) << std::endl;
std::fstream("test.txt") << tree->toStringTree(&parser);
std::cout << tree->toStringTree(&parser) << std::endl << std::endl;
return 0;
}

4
runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj
View File

@ -988,7 +988,7 @@
276E5C9E1CDB57AA003FF4B4 /* BufferedTokenStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BufferedTokenStream.h; sourceTree = "<group>"; };
276E5C9F1CDB57AA003FF4B4 /* CharStream.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CharStream.cpp; sourceTree = "<group>"; };
276E5CA01CDB57AA003FF4B4 /* CharStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CharStream.h; sourceTree = "<group>"; };
276E5CA11CDB57AA003FF4B4 /* CommonToken.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CommonToken.cpp; sourceTree = "<group>"; };
276E5CA11CDB57AA003FF4B4 /* CommonToken.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CommonToken.cpp; sourceTree = "<group>"; wrapsLines = 0; };
276E5CA21CDB57AA003FF4B4 /* CommonToken.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CommonToken.h; sourceTree = "<group>"; };
276E5CA31CDB57AA003FF4B4 /* CommonTokenFactory.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CommonTokenFactory.cpp; sourceTree = "<group>"; };
276E5CA41CDB57AA003FF4B4 /* CommonTokenFactory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CommonTokenFactory.h; sourceTree = "<group>"; };
@ -1062,7 +1062,7 @@
276E5CEC1CDB57AA003FF4B4 /* guid.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = guid.h; sourceTree = "<group>"; };
276E5CED1CDB57AA003FF4B4 /* StringUtils.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = StringUtils.cpp; sourceTree = "<group>"; };
276E5CEE1CDB57AA003FF4B4 /* StringUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StringUtils.h; sourceTree = "<group>"; };
276E5CEF1CDB57AA003FF4B4 /* Token.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Token.cpp; sourceTree = "<group>"; };
276E5CEF1CDB57AA003FF4B4 /* Token.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Token.cpp; sourceTree = "<group>"; wrapsLines = 0; };
276E5CF01CDB57AA003FF4B4 /* Token.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Token.h; sourceTree = "<group>"; };
276E5CF21CDB57AA003FF4B4 /* TokenFactory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = TokenFactory.h; sourceTree = "<group>"; };
276E5CF41CDB57AA003FF4B4 /* TokenSource.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = TokenSource.h; sourceTree = "<group>"; };

40
runtime/Cpp/runtime/src/CommonToken.cpp
View File

@ -95,7 +95,7 @@ void CommonToken::setLine(int line) {
_line = line;
}
std::string CommonToken::getText() {
std::string CommonToken::getText() const {
if (!_text.empty()) {
return _text;
}
@ -116,11 +116,11 @@ void CommonToken::setText(const std::string &text) {
_text = text;
}
int CommonToken::getLine() {
int CommonToken::getLine() const {
return _line;
}
int CommonToken::getCharPositionInLine() {
int CommonToken::getCharPositionInLine() const {
return _charPositionInLine;
}
@ -128,7 +128,7 @@ void CommonToken::setCharPositionInLine(int charPositionInLine) {
_charPositionInLine = charPositionInLine;
}
size_t CommonToken::getChannel() {
size_t CommonToken::getChannel() const {
return _channel;
}
@ -140,7 +140,7 @@ void CommonToken::setType(int type) {
_type = type;
}
int CommonToken::getStartIndex() {
int CommonToken::getStartIndex() const {
return _start;
}
@ -148,7 +148,7 @@ void CommonToken::setStartIndex(int start) {
_start = start;
}
int CommonToken::getStopIndex() {
int CommonToken::getStopIndex() const {
return _stop;
}
@ -156,7 +156,7 @@ void CommonToken::setStopIndex(int stop) {
_stop = stop;
}
int CommonToken::getTokenIndex() {
int CommonToken::getTokenIndex() const {
return _index;
}
@ -164,14 +164,36 @@ void CommonToken::setTokenIndex(int index) {
_index = index;
}
org::antlr::v4::runtime::TokenSource *CommonToken::getTokenSource() {
org::antlr::v4::runtime::TokenSource *CommonToken::getTokenSource() const {
return _source.first;
}
org::antlr::v4::runtime::CharStream *CommonToken::getInputStream() {
org::antlr::v4::runtime::CharStream *CommonToken::getInputStream() const {
return _source.second;
}
std::string CommonToken::toString() const {
std::stringstream ss;
std::string channelStr;
if (_channel > 0) {
channelStr = ",channel=" + std::to_string(_channel);
}
std::string txt = getText();
if (!txt.empty()) {
antlrcpp::replaceAll(txt, "\n", "\\n");
antlrcpp::replaceAll(txt, "\r", "\\r");
antlrcpp::replaceAll(txt, "\t", "\\t");
} else {
txt = "<no text>";
}
ss << "[@" << getTokenIndex() << "," << _start << ":" << _stop << "='" << txt << "',<" << _type << ">" << channelStr
<< "," << _line << ":" << getCharPositionInLine() << "]";
return ss.str();
}
void CommonToken::InitializeInstanceFields() {
_type = 0;
_line = 0;

20
runtime/Cpp/runtime/src/CommonToken.h
View File

@ -152,31 +152,33 @@ namespace runtime {
* of the token.
*/
virtual void setText(const std::string &text) override;
virtual std::string getText() override;
virtual std::string getText() const override;
virtual void setLine(int line) override;
virtual int getLine() override;
virtual int getLine() const override;
virtual int getCharPositionInLine() override;
virtual int getCharPositionInLine() const override;
virtual void setCharPositionInLine(int charPositionInLine) override;
virtual size_t getChannel() override;
virtual size_t getChannel() const override;
virtual void setChannel(int channel) override;
virtual void setType(int type) override;
virtual int getStartIndex() override;
virtual int getStartIndex() const override;
virtual void setStartIndex(int start);
virtual int getStopIndex() override;
virtual int getStopIndex() const override;
virtual void setStopIndex(int stop);
virtual int getTokenIndex() override;
virtual int getTokenIndex() const override;
virtual void setTokenIndex(int index) override;
virtual TokenSource *getTokenSource() override;
virtual CharStream *getInputStream() override;
virtual TokenSource *getTokenSource() const override;
virtual CharStream *getInputStream() const override;
virtual std::string toString() const override;
private:
void InitializeInstanceFields();
};

19
runtime/Cpp/runtime/src/Token.cpp
View File

@ -35,22 +35,3 @@
using namespace org::antlr::v4::runtime;
std::string Token::toString() {
std::stringstream ss;
std::string txt = getText();
if (!txt.empty()) {
antlrcpp::replaceAll(txt, "\n", "\\n");
antlrcpp::replaceAll(txt, "\r", "\\r");
antlrcpp::replaceAll(txt, "\t", "\\t");
} else {
txt = "<no text>";
}
ss << "{Token: " << txt << ", channel: " << getChannel() << ", type: " << getType() << ", start: " << getStartIndex() <<
", stop: " << getStopIndex() << ", index: " << getTokenIndex() << ", line: " << getLine() << ", offset: " <<
getCharPositionInLine() << "}";
return ss.str();
}

38
runtime/Cpp/runtime/src/Token.h
View File

@ -38,34 +38,28 @@ namespace antlr {
namespace v4 {
namespace runtime {
/// <summary>
/// A token has properties: text, type, line, character position in the line
/// (so we can ignore tabs), token channel, index, and source from which
/// we obtained this token.
/// </summary>
/// (so we can ignore tabs), token channel, index, and source from which
/// we obtained this token.
class ANTLR4CPP_PUBLIC Token {
public:
static const size_t INVALID_TYPE = 0;
/// During lookahead operations, this "token" signifies we hit rule end ATN state
/// and did not follow it despite needing to.
/// and did not follow it despite needing to.
static const ssize_t EPSILON = -2;
static const size_t MIN_USER_TOKEN_TYPE = 1;
static const ssize_t EOF = IntStream::EOF;
virtual ~Token() {};
/// <summary>
/// All tokens go to the parser (unless skip() is called in that rule)
/// on a particular "channel". The parser tunes to a particular channel
/// so that whitespace etc... can go to the parser on a "hidden" channel.
/// </summary>
/// on a particular "channel". The parser tunes to a particular channel
/// so that whitespace etc... can go to the parser on a "hidden" channel.
static const size_t DEFAULT_CHANNEL = 0;
/// <summary>
/// Anything on different channel than DEFAULT_CHANNEL is not parsed
/// by parser.
/// </summary>
/// by parser.
static const size_t HIDDEN_CHANNEL = 1;
/**
@ -84,7 +78,7 @@ namespace runtime {
/// <summary>
/// Get the text of the token.
/// </summary>
virtual std::string getText() = 0;
virtual std::string getText() const = 0;
/// <summary>
/// Get the token type of the token </summary>
@ -94,20 +88,20 @@ namespace runtime {
/// The line number on which the 1st character of this token was matched,
/// line=1..n
/// </summary>
virtual int getLine() = 0;
virtual int getLine() const = 0;
/// <summary>
/// The index of the first character of this token relative to the
/// beginning of the line at which it occurs, 0..n-1
/// </summary>
virtual int getCharPositionInLine() = 0;
virtual int getCharPositionInLine() const = 0;
/// <summary>
/// Return the channel this token. Each token can arrive at the parser
/// on a different channel, but the parser only "tunes" to a single channel.
/// The parser ignores everything not on DEFAULT_CHANNEL.
/// </summary>
virtual size_t getChannel() = 0;
virtual size_t getChannel() const = 0;
/// <summary>
/// An index from 0..n-1 of the token object in the input stream.
@ -117,31 +111,31 @@ namespace runtime {
/// Return -1 to indicate that this token was conjured up since
/// it doesn't have a valid index.
/// </summary>
virtual int getTokenIndex() = 0;
virtual int getTokenIndex() const = 0;
/// <summary>
/// The starting character index of the token
/// This method is optional; return -1 if not implemented.
/// </summary>
virtual int getStartIndex() = 0;
virtual int getStartIndex() const = 0;
/// <summary>
/// The last character index of the token.
/// This method is optional; return -1 if not implemented.
/// </summary>
virtual int getStopIndex() = 0;
virtual int getStopIndex() const = 0;
/// <summary>
/// Gets the <seealso cref="TokenSource"/> which created this token.
/// </summary>
virtual TokenSource *getTokenSource() = 0;
virtual TokenSource *getTokenSource() const = 0;
/// <summary>
/// Gets the <seealso cref="CharStream"/> from which this token was derived.
/// </summary>
virtual CharStream *getInputStream() = 0;
virtual CharStream *getInputStream() const = 0;
virtual std::string toString();
virtual std::string toString() const = 0;
};
} // namespace runtime

18
runtime/Cpp/runtime/src/tree/Trees.cpp
View File

@ -90,18 +90,22 @@ std::string Trees::toStringTree(Ref<Tree> t, const std::vector<std::string> &rul
std::string temp = antlrcpp::escapeWhitespace(Trees::getNodeText(child, ruleNames), false);
if (child->getChildCount() > 0) {
// Go deeper one level.
stack.push(childIndex + 1);
stack.push(childIndex);
run = child;
childIndex = 0;
ss << "(" << temp << " ";
} else {
ss << temp;
if (++childIndex == run->getChildCount() && stack.size() > 0) {
// Reached the end of the current level. See if we can step up from here.
childIndex = stack.top();
stack.pop();
run = run->getParent().lock();
ss << ")";
while (++childIndex == run->getChildCount()) {
if (stack.size() > 0) {
// Reached the end of the current level. See if we can step up from here.
childIndex = stack.top();
stack.pop();
run = run->getParent().lock();
ss << ")";
} else {
break;
}
}
}
}

26
runtime/Cpp/runtime/src/tree/pattern/RuleTagToken.cpp
View File

@ -46,19 +46,19 @@ RuleTagToken::RuleTagToken(const std::string &ruleName, int bypassTokenType, con
}
std::string RuleTagToken::getRuleName() {
std::string RuleTagToken::getRuleName() const {
return ruleName;
}
std::string RuleTagToken::getLabel() {
std::string RuleTagToken::getLabel() const {
return label;
}
size_t RuleTagToken::getChannel() {
size_t RuleTagToken::getChannel() const {
return DEFAULT_CHANNEL;
}
std::string RuleTagToken::getText() {
std::string RuleTagToken::getText() const {
if (label != "") {
return std::string("<") + label + std::string(":") + ruleName + std::string(">");
}
@ -70,34 +70,34 @@ int RuleTagToken::getType() const {
return bypassTokenType;
}
int RuleTagToken::getLine() {
int RuleTagToken::getLine() const {
return 0;
}
int RuleTagToken::getCharPositionInLine() {
int RuleTagToken::getCharPositionInLine() const {
return -1;
}
int RuleTagToken::getTokenIndex() {
int RuleTagToken::getTokenIndex() const {
return -1;
}
int RuleTagToken::getStartIndex() {
int RuleTagToken::getStartIndex() const {
return -1;
}
int RuleTagToken::getStopIndex() {
int RuleTagToken::getStopIndex() const {
return -1;
}
org::antlr::v4::runtime::TokenSource *RuleTagToken::getTokenSource() {
org::antlr::v4::runtime::TokenSource *RuleTagToken::getTokenSource() const {
return nullptr;
}
org::antlr::v4::runtime::CharStream *RuleTagToken::getInputStream() {
org::antlr::v4::runtime::CharStream *RuleTagToken::getInputStream() const {
return nullptr;
}
std::string RuleTagToken::toString() {
return ruleName + std::string(":") + std::to_string(bypassTokenType);
std::string RuleTagToken::toString() const {
return ruleName + ":" + std::to_string(bypassTokenType);
}

24
runtime/Cpp/runtime/src/tree/pattern/RuleTagToken.h
View File

@ -91,21 +91,21 @@ namespace pattern {
/// Gets the name of the rule associated with this rule tag.
/// </summary>
/// <returns> The name of the parser rule associated with this rule tag. </returns>
std::string getRuleName();
std::string getRuleName() const;
/// <summary>
/// Gets the label associated with the rule tag.
/// </summary>
/// <returns> The name of the label associated with the rule tag, or
/// {@code null} if this is an unlabeled rule tag. </returns>
std::string getLabel();
std::string getLabel() const;
/// <summary>
/// {@inheritDoc}
/// <p/>
/// Rule tag tokens are always placed on the <seealso cref="#DEFAULT_CHANNE"/>.
/// </summary>
virtual size_t getChannel() override;
virtual size_t getChannel() const override;
/// <summary>
/// {@inheritDoc}
@ -113,7 +113,7 @@ namespace pattern {
/// This method returns the rule tag formatted with {@code <} and {@code >}
/// delimiters.
/// </summary>
virtual std::string getText() override;
virtual std::string getText() const override;
/// <summary>
/// {@inheritDoc}
@ -128,49 +128,49 @@ namespace pattern {
/// <p/>
/// The implementation for <seealso cref="RuleTagToken"/> always returns 0.
/// </summary>
virtual int getLine() override;
virtual int getLine() const override;
/// <summary>
/// {@inheritDoc}
/// <p/>
/// The implementation for <seealso cref="RuleTagToken"/> always returns -1.
/// </summary>
virtual int getCharPositionInLine() override;
virtual int getCharPositionInLine() const override;
/// <summary>
/// {@inheritDoc}
/// <p/>
/// The implementation for <seealso cref="RuleTagToken"/> always returns -1.
/// </summary>
virtual int getTokenIndex() override;
virtual int getTokenIndex() const override;
/// <summary>
/// {@inheritDoc}
/// <p/>
/// The implementation for <seealso cref="RuleTagToken"/> always returns -1.
/// </summary>
virtual int getStartIndex() override;
virtual int getStartIndex() const override;
/// <summary>
/// {@inheritDoc}
/// <p/>
/// The implementation for <seealso cref="RuleTagToken"/> always returns -1.
/// </summary>
virtual int getStopIndex() override;
virtual int getStopIndex() const override;
/// <summary>
/// {@inheritDoc}
/// <p/>
/// The implementation for <seealso cref="RuleTagToken"/> always returns {@code null}.
/// </summary>
virtual TokenSource *getTokenSource() override;
virtual TokenSource *getTokenSource() const override;
/// <summary>
/// {@inheritDoc}
/// <p/>
/// The implementation for <seealso cref="RuleTagToken"/> always returns {@code null}.
/// </summary>
virtual CharStream *getInputStream() override;
virtual CharStream *getInputStream() const override;
/// <summary>
/// {@inheritDoc}
@ -178,7 +178,7 @@ namespace pattern {
/// The implementation for <seealso cref="RuleTagToken"/> returns a string of the form
/// {@code ruleName:bypassTokenType}.
/// </summary>
virtual std::string toString() override;
virtual std::string toString() const override;
};
} // namespace pattern

16
runtime/Cpp/runtime/src/tree/pattern/TokenTagToken.cpp
View File

@ -41,22 +41,22 @@ TokenTagToken::TokenTagToken(const std::string &tokenName, int type, const std::
: CommonToken(type), tokenName(tokenName), label(label) {
}
std::string TokenTagToken::getTokenName() {
std::string TokenTagToken::getTokenName() const {
return tokenName;
}
std::string TokenTagToken::getLabel() {
std::string TokenTagToken::getLabel() const {
return label;
}
std::string TokenTagToken::getText() {
if (label != "") {
return std::string("<") + label + std::string(":") + tokenName + std::string(">");
std::string TokenTagToken::getText() const {
if (!label.empty()) {
return "<" + label + ":" + tokenName + ">";
}
return std::string("<") + tokenName + std::string(">");
return "<" + tokenName + ">";
}
std::string TokenTagToken::toString() {
return tokenName + std::string(":") + std::to_string(_type);
std::string TokenTagToken::toString() const {
return tokenName + ":" + std::to_string(_type);
}

8
runtime/Cpp/runtime/src/tree/pattern/TokenTagToken.h
View File

@ -78,14 +78,14 @@ namespace pattern {
/// <summary>
/// Gets the token name. </summary>
/// <returns> The token name. </returns>
std::string getTokenName();
std::string getTokenName() const;
/// <summary>
/// Gets the label associated with the rule tag.
/// </summary>
/// <returns> The name of the label associated with the rule tag, or
/// {@code null} if this is an unlabeled rule tag. </returns>
std::string getLabel();
std::string getLabel() const;
/// <summary>
/// {@inheritDoc}
@ -93,7 +93,7 @@ namespace pattern {
/// The implementation for <seealso cref="TokenTagToken"/> returns the token tag
/// formatted with {@code <} and {@code >} delimiters.
/// </summary>
virtual std::string getText() override;
virtual std::string getText() const override;
/// <summary>
/// {@inheritDoc}
@ -101,7 +101,7 @@ namespace pattern {
/// The implementation for <seealso cref="TokenTagToken"/> returns a string of the form
/// {@code tokenName:type}.
/// </summary>
virtual std::string toString() override;
virtual std::string toString() const override;
};
} // namespace pattern

5
tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
View File

@ -908,7 +908,10 @@ RulePropertyRef_ctx(r) ::= "RulePropertyRef_ctx(r)<ctx(r)>.<r.label>"
ThisRulePropertyRef_start(r) ::= "ThisRulePropertyRef_start(r) _localctx->start"
ThisRulePropertyRef_stop(r) ::= "ThisRulePropertyRef_stop(r) _localctx->stop"
ThisRulePropertyRef_text(r) ::= "ThisRulePropertyRef_text(r) input->getText(_localctx->start, _input->LT(-1))"
ThisRulePropertyRef_textHeader(r) ::= "<! Required but unused. !>"
ThisRulePropertyRef_text(r) ::= "input->getText(_localctx->start, _input->LT(-1))"
ThisRulePropertyRef_ctx(r) ::= "ThisRulePropertyRef_ctx(r) _localctx"
NonLocalAttrRef(s) ::= "NonLocalAttrRef(s) ((<s.ruleName; format=\"cap\">Context)getInvokingContext(<s.ruleIndex>)).<s.name>"