forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #1630 from bhamiltoncx/python-runtime-test-utf-8 

Use UTF-8 for Python runtime tests, allow specifying error handling
This commit is contained in:
Terence Parr 2017-01-30 11:32:55 -08:00 committed by GitHub
parent 444d35ecde d9d4d8c6c4
commit e8f45783cd
5 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
runtime-testsuite/test/org/antlr/v4/test/runtime/python2/BasePython2Test.java
View File

@ -32,7 +32,7 @@ public class BasePython2Test extends BasePythonTest {
+ "from <lexerName> import <lexerName>\n"
+ "\n"
+ "def main(argv):\n"
+ " input = FileStream(argv[1])\n"
+ " input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
+ " lexer = <lexerName>(input)\n"
+ " stream = CommonTokenStream(lexer)\n"
+ " stream.fill()\n"
@ -76,7 +76,7 @@ public class BasePython2Test extends BasePythonTest {
+ " raise IllegalStateException(\"Invalid parse tree shape detected.\")\n"
+ "\n"
+ "def main(argv):\n"
+ " input = FileStream(argv[1])\n"
+ " input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
+ " lexer = <lexerName>(input)\n"
+ " stream = CommonTokenStream(lexer)\n"
+ "<createParser>"

4
runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java
View File

@ -30,7 +30,7 @@ public class BasePython3Test extends BasePythonTest {
+ "from <lexerName> import <lexerName>\n"
+ "\n"
+ "def main(argv):\n"
+ " input = FileStream(argv[1])\n"
+ " input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
+ " lexer = <lexerName>(input)\n"
+ " stream = CommonTokenStream(lexer)\n"
+ " stream.fill()\n"
@ -74,7 +74,7 @@ public class BasePython3Test extends BasePythonTest {
+ " raise IllegalStateException(\"Invalid parse tree shape detected.\")\n"
+ "\n"
+ "def main(argv):\n"
+ " input = FileStream(argv[1])\n"
+ " input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
+ " lexer = <lexerName>(input)\n"
+ " stream = CommonTokenStream(lexer)\n"
+ "<createParser>"

4
runtime/Python2/src/antlr4/FileStream.py
View File

@ -16,12 +16,12 @@ from antlr4.InputStream import InputStream
class FileStream(InputStream):
def __init__(self, fileName, encoding='ascii'):
def __init__(self, fileName, encoding='ascii', errors='strict'):
self.fileName = fileName
# read binary to avoid line ending conversion
with open(fileName, 'rb') as file:
bytes = file.read()
data = codecs.decode(bytes, encoding)
data = codecs.decode(bytes, encoding, errors)
super(type(self), self).__init__(data)

6
runtime/Python2/src/antlr4/StdinStream.py
View File

@ -15,7 +15,7 @@ from antlr4.InputStream import InputStream
class StdinStream(InputStream):
def __init__(self, encoding='ascii'):
def __init__(self, encoding='ascii', errors='strict'):
bytes = sys.stdin.read()
data = codecs.decode(bytes, encoding)
super(type(self), self).__init__(data)
data = codecs.decode(bytes, encoding, errors)
super(type(self), self).__init__(data)

10
runtime/Python3/src/antlr4/FileStream.py
View File

@ -16,19 +16,19 @@ from antlr4.InputStream import InputStream
class FileStream(InputStream):
def __init__(self, fileName:str, encoding:str='ascii'):
super().__init__(self.readDataFrom(fileName, encoding))
def __init__(self, fileName:str, encoding:str='ascii', errors:str='strict'):
super().__init__(self.readDataFrom(fileName, encoding, errors))
self.fileName = fileName
def readDataFrom(self, fileName:str, encoding:str):
def readDataFrom(self, fileName:str, encoding:str, errors:str='strict'):
# read binary to avoid line ending conversion
with open(fileName, 'rb') as file:
bytes = file.read()
return codecs.decode(bytes, encoding)
return codecs.decode(bytes, encoding, errors)
class TestFileStream(unittest.TestCase):
def testStream(self):
stream = FileStream(__file__)
self.assertTrue(stream.size>0)
self.assertTrue(stream.size>0)