jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge pull request #3113 from mike-lischke/master 

Fixed ANTLRInputStream and ANTLRFileStream
This commit is contained in:
Terence Parr 2021-03-10 09:03:03 -08:00 committed by GitHub
parent 0f8bddcbc1 4431f1ff69
commit d889ba8668
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 86 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java
View File

@ -394,7 +394,8 @@ public class BaseCppTest extends BaseRuntimeTestSupport implements RuntimeTestSu
+ "\n"
+ "\n"
+ "int main(int argc, const char* argv[]) {\n"
+ " ANTLRFileStream input(argv[1]);\n"
+ " ANTLRFileStream input;\n"
+ " input.loadFromFile(argv[1]);\n"
+ " <lexerName> lexer(&input);\n"
+ " CommonTokenStream tokens(&lexer);\n"
+ "<createParser>"
@ -436,7 +437,8 @@ public class BaseCppTest extends BaseRuntimeTestSupport implements RuntimeTestSu
+ "using namespace antlr4;\n"
+ "\n"
+ "int main(int argc, const char* argv[]) {\n"
+ " ANTLRFileStream input(argv[1]);\n"
+ " ANTLRFileStream input;\n"
+ " input.loadFromFile(argv[1]);\n"
+ " <lexerName> lexer(&input);\n"
+ " CommonTokenStream tokens(&lexer);\n"
+ " tokens.fill();\n"

46
runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj
View File

@ -3,7 +3,7 @@
archiveVersion = 1;
classes = {
};
objectVersion = 46;
objectVersion = 54;
objects = {
/* Begin PBXBuildFile section */
@ -2226,7 +2226,7 @@
37D727A21867AF1E007B6D10 /* Project object */ = {
isa = PBXProject;
attributes = {
LastUpgradeCheck = 1030;
LastUpgradeCheck = 1240;
ORGANIZATIONNAME = ANTLR;
TargetAttributes = {
270C67EF1CDB4F1E00116E17 = {
@ -2238,7 +2238,7 @@
};
};
buildConfigurationList = 37D727A51867AF1E007B6D10 /* Build configuration list for PBXProject "antlrcpp" */;
compatibilityVersion = "Xcode 3.2";
compatibilityVersion = "Xcode 12.0";
developmentRegion = en;
hasScannedForEncodings = 0;
knownRegions = (
@ -2751,8 +2751,12 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
INFOPLIST_FILE = "antlrcpp-ios/Info.plist";
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
IPHONEOS_DEPLOYMENT_TARGET = 9.3;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
IPHONEOS_DEPLOYMENT_TARGET = 12.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
MTL_ENABLE_DEBUG_INFO = YES;
PRODUCT_BUNDLE_IDENTIFIER = "org.antlr.v4.runtime.antlrcpp-ios";
PRODUCT_NAME = "$(TARGET_NAME)";
@ -2782,8 +2786,12 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
INFOPLIST_FILE = "antlrcpp-ios/Info.plist";
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
IPHONEOS_DEPLOYMENT_TARGET = 9.3;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
IPHONEOS_DEPLOYMENT_TARGET = 12.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
MTL_ENABLE_DEBUG_INFO = NO;
PRODUCT_BUNDLE_IDENTIFIER = "org.antlr.v4.runtime.antlrcpp-ios";
PRODUCT_NAME = "$(TARGET_NAME)";
@ -2841,8 +2849,7 @@
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES;
CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_CXX_LANGUAGE_STANDARD = "c++17";
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_ASSIGN_ENUM = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
@ -2857,6 +2864,7 @@
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_IMPLICIT_CONVERSION = YES;
@ -2889,8 +2897,12 @@
GCC_WARN_UNUSED_LABEL = YES;
GCC_WARN_UNUSED_PARAMETER = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = "src/ thirdparty/utfcpp/source/ thirdparty/utfcpp/source/utf8/";
MACOSX_DEPLOYMENT_TARGET = 10.9;
HEADER_SEARCH_PATHS = (
src/,
thirdparty/utfcpp/source/,
thirdparty/utfcpp/source/utf8/,
);
MACOSX_DEPLOYMENT_TARGET = 11.1;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = macosx;
};
@ -2901,8 +2913,7 @@
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES;
CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_CXX_LANGUAGE_STANDARD = "c++17";
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_ASSIGN_ENUM = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
@ -2917,6 +2928,7 @@
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_IMPLICIT_CONVERSION = YES;
@ -2945,8 +2957,12 @@
GCC_WARN_UNUSED_LABEL = YES;
GCC_WARN_UNUSED_PARAMETER = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = "src/ thirdparty/utfcpp/source/ thirdparty/utfcpp/source/utf8/";
MACOSX_DEPLOYMENT_TARGET = 10.9;
HEADER_SEARCH_PATHS = (
src/,
thirdparty/utfcpp/source/,
thirdparty/utfcpp/source/utf8/,
);
MACOSX_DEPLOYMENT_TARGET = 11.1;
SDKROOT = macosx;
};
name = Release;

2
runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "1030"
LastUpgradeVersion = "1240"
version = "1.3">
<BuildAction
parallelizeBuildables = "YES"

6
runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4_ios.xcscheme
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "1030"
LastUpgradeVersion = "1240"
version = "1.3">
<BuildAction
parallelizeBuildables = "YES"
@ -29,8 +29,6 @@
shouldUseLaunchSchemeArgsEnv = "YES">
<Testables>
</Testables>
<AdditionalOptions>
</AdditionalOptions>
</TestAction>
<LaunchAction
buildConfiguration = "Debug"
@ -51,8 +49,6 @@
ReferencedContainer = "container:antlrcpp.xcodeproj">
</BuildableReference>
</MacroExpansion>
<AdditionalOptions>
</AdditionalOptions>
</LaunchAction>
<ProfileAction
buildConfiguration = "Release"

6
runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4_static.xcscheme
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "1030"
LastUpgradeVersion = "1240"
version = "1.3">
<BuildAction
parallelizeBuildables = "YES"
@ -29,8 +29,6 @@
shouldUseLaunchSchemeArgsEnv = "YES">
<Testables>
</Testables>
<AdditionalOptions>
</AdditionalOptions>
</TestAction>
<LaunchAction
buildConfiguration = "Debug"
@ -51,8 +49,6 @@
ReferencedContainer = "container:antlrcpp.xcodeproj">
</BuildableReference>
</MacroExpansion>
<AdditionalOptions>
</AdditionalOptions>
</LaunchAction>
<ProfileAction
buildConfiguration = "Release"

7
runtime/Cpp/runtime/src/ANTLRFileStream.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
@ -9,11 +9,6 @@
using namespace antlr4;
ANTLRFileStream::ANTLRFileStream(const std::string &fileName) {
_fileName = fileName;
loadFromFile(fileName);
}
void ANTLRFileStream::loadFromFile(const std::string &fileName) {
_fileName = fileName;
if (_fileName.empty()) {

15
runtime/Cpp/runtime/src/ANTLRFileStream.h
View File

@ -1,4 +1,4 @@
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
@ -13,15 +13,18 @@ namespace antlr4 {
/// when you construct the object (or call load()).
// TODO: this class needs testing.
class ANTLR4CPP_PUBLIC ANTLRFileStream : public ANTLRInputStream {
protected:
std::string _fileName; // UTF-8 encoded file name.
public:
// Assumes a file name encoded in UTF-8 and file content in the same encoding (with or w/o BOM).
ANTLRFileStream(const std::string &fileName);
ANTLRFileStream() = default;
ANTLRFileStream(const std::string &) = delete;
ANTLRFileStream(const char *data, size_t length) = delete;
ANTLRFileStream(std::istream &stream) = delete;
// Assumes a file name encoded in UTF-8 and file content in the same encoding (with or w/o BOM).
virtual void loadFromFile(const std::string &fileName);
virtual std::string getSourceName() const override;
private:
std::string _fileName; // UTF-8 encoded file name.
};
} // namespace antlr4

51
runtime/Cpp/runtime/src/ANTLRInputStream.cpp
View File

@ -1,8 +1,10 @@
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
#include <string.h>
#include "Exceptions.h"
#include "misc/Interval.h"
#include "IntStream.h"
@ -17,44 +19,41 @@ using namespace antlrcpp;
using misc::Interval;
ANTLRInputStream::ANTLRInputStream() {
InitializeInstanceFields();
}
#if __cplusplus >= 201703L
ANTLRInputStream::ANTLRInputStream(std::string_view input) {
#else
ANTLRInputStream::ANTLRInputStream(const std::string &input) {
ANTLRInputStream::ANTLRInputStream(const std::string_view &input): ANTLRInputStream() {
load(input.data(), input.length());
}
#endif
InitializeInstanceFields();
load(input);
ANTLRInputStream::ANTLRInputStream(const std::string &input): ANTLRInputStream() {
load(input.data(), input.size());
}
ANTLRInputStream::ANTLRInputStream(const char data_[], size_t numberOfActualCharsInArray)
: ANTLRInputStream(std::string(data_, numberOfActualCharsInArray)) {
ANTLRInputStream::ANTLRInputStream(const char *data, size_t length) {
load(data, length);
}
ANTLRInputStream::ANTLRInputStream(std::istream &stream) {
InitializeInstanceFields();
ANTLRInputStream::ANTLRInputStream(std::istream &stream): ANTLRInputStream() {
load(stream);
}
#if __cplusplus >= 201703L
void ANTLRInputStream::load(std::string_view input) {
// Remove the UTF-8 BOM if present.
constexpr std::string_view bom = "\xef\xbb\xbf";
if (input.compare(0, 3, bom) == 0)
input.remove_prefix(3);
_data = antlrcpp::utf8_to_utf32(input.data(), input.data() + input.size());
p = 0;
}
#else
void ANTLRInputStream::load(const std::string &input) {
load(input.data(), input.size());
}
void ANTLRInputStream::load(const char *data, size_t length) {
// Remove the UTF-8 BOM if present.
const char bom[4] = "\xef\xbb\xbf";
if (input.compare(0, 3, bom, 3) == 0)
_data = antlrcpp::utf8_to_utf32(input.data() + 3, input.data() + input.size());
const char *bom = "\xef\xbb\xbf";
if (length >= 3 && strncmp(data, bom, 3) == 0)
_data = antlrcpp::utf8_to_utf32(data + 3, data + length);
else
_data = antlrcpp::utf8_to_utf32(input.data(), input.data() + input.size());
_data = antlrcpp::utf8_to_utf32(data, data + length);
p = 0;
}
#endif
void ANTLRInputStream::load(std::istream &stream) {
if (!stream.good() || stream.eof()) // No fail, bad or EOF.
@ -63,7 +62,7 @@ void ANTLRInputStream::load(std::istream &stream) {
_data.clear();
std::string s((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>());
load(s);
load(s.data(), s.length());
}
void ANTLRInputStream::reset() {

21
runtime/Cpp/runtime/src/ANTLRInputStream.h
View File

@ -1,4 +1,4 @@
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
@ -25,19 +25,18 @@ namespace antlr4 {
/// What is name or source of this char stream?
std::string name;
#if __cplusplus >= 201703L
ANTLRInputStream(std::string_view input = "");
#else
ANTLRInputStream(const std::string &input = "");
#endif
ANTLRInputStream(const char data_[], size_t numberOfActualCharsInArray);
ANTLRInputStream(std::istream &stream);
ANTLRInputStream();
#if __cplusplus >= 201703L
virtual void load(std::string_view input);
#else
virtual void load(const std::string &input);
ANTLRInputStream(const std::string_view &input);
#endif
ANTLRInputStream(const std::string &input);
ANTLRInputStream(const char *data, size_t length);
ANTLRInputStream(std::istream &stream);
virtual void load(const std::string &input);
virtual void load(const char *data, size_t length);
virtual void load(std::istream &stream);
/// Reset the stream so that it's in the same state it was

4
runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp
View File

@ -1348,7 +1348,9 @@ Parser* ParserATNSimulator::getParser() {
return parser;
}
#ifdef _MSC_VER
#pragma warning (disable:4996) // 'getenv': This function or variable may be unsafe. Consider using _dupenv_s instead.
#endif
bool ParserATNSimulator::getLrLoopSetting() {
char *var = std::getenv("TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT");
@ -1358,7 +1360,9 @@ bool ParserATNSimulator::getLrLoopSetting() {
return value == "true" || value == "1";
}
#ifdef _MSC_VER
#pragma warning (default:4996)
#endif
void ParserATNSimulator::InitializeInstanceFields() {
_mode = PredictionMode::LL;