From ed977513ec6af2e9015e6f6eee6e6b2a0b48e2c5 Mon Sep 17 00:00:00 2001 From: Manuel Krebber Date: Tue, 29 Nov 2016 11:51:56 +0100 Subject: [PATCH 1/8] Added a console option to specify the encoding to use for doctest files. Defaults to UTF-8. --- _pytest/doctest.py | 6 +++++- doc/en/doctest.rst | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/_pytest/doctest.py b/_pytest/doctest.py index f4782dded..05acc7c59 100644 --- a/_pytest/doctest.py +++ b/_pytest/doctest.py @@ -43,6 +43,10 @@ def pytest_addoption(parser): action="store_true", default=False, help="ignore doctest ImportErrors", dest="doctest_ignore_import_errors") + group.addoption("--doctest-encoding", + type=str.lower, default="utf-8", + help="choose the encoding to use for doctest files", + dest="doctestencoding") def pytest_collect_file(path, parent): @@ -171,7 +175,7 @@ class DoctestTextfile(pytest.Module): # inspired by doctest.testfile; ideally we would use it directly, # but it doesn't support passing a custom checker - text = self.fspath.read() + text = self.fspath.read_text(self.config.getoption("doctestencoding")) filename = str(self.fspath) name = self.fspath.basename globs = {'__name__': '__main__'} diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 5a1122515..93d7c30c4 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -11,6 +11,13 @@ can change the pattern by issuing:: on the command line. Since version ``2.9``, ``--doctest-glob`` can be given multiple times in the command-line. +You can specify the encoding that will be used for those doctest files +using the ``--doctest-encoding`` command line option:: + + pytest --doctest-encoding='ascii' + +The default encoding is UTF-8. + You can also trigger running of doctests from docstrings in all python modules (including regular python test modules):: @@ -52,9 +59,9 @@ then you can just invoke ``pytest`` without command line options:: platform linux -- Python 3.5.2, pytest-3.0.4, py-1.4.31, pluggy-0.4.0 rootdir: $REGENDOC_TMPDIR, inifile: pytest.ini collected 1 items - + mymodule.py . - + ======= 1 passed in 0.12 seconds ======== It is possible to use fixtures using the ``getfixture`` helper:: From d254c6b0aeb872a791889d5e3497fa3ca679c482 Mon Sep 17 00:00:00 2001 From: Manuel Krebber Date: Tue, 29 Nov 2016 12:18:41 +0100 Subject: [PATCH 2/8] Added some tests for --docstring-encoding option. Added option to specify encoding for internal testdir._makefile() for the tests. --- _pytest/pytester.py | 4 ++-- testing/test_doctest.py | 46 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/_pytest/pytester.py b/_pytest/pytester.py index d19ff1ec6..37348c26e 100644 --- a/_pytest/pytester.py +++ b/_pytest/pytester.py @@ -471,7 +471,7 @@ class Testdir: if not hasattr(self, '_olddir'): self._olddir = old - def _makefile(self, ext, args, kwargs): + def _makefile(self, ext, args, kwargs, encoding="utf-8"): items = list(kwargs.items()) if args: source = py.builtin._totext("\n").join( @@ -490,7 +490,7 @@ class Testdir: source_unicode = "\n".join([my_totext(line) for line in source.lines]) source = py.builtin._totext(source_unicode) - content = source.strip().encode("utf-8") # + "\n" + content = source.strip().encode(encoding) # + "\n" #content = content.rstrip() + "\n" p.write(content, "wb") if ret is None: diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 4ea2cc58e..4ecd5744b 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -129,6 +129,52 @@ class TestDoctests: '*1 passed*', ]) + def test_encoding_ascii(self, testdir): + """Test support for --doctest-encoding option. + """ + testdir._makefile(".txt", [""" + >>> 1 + 1 + """], {}, encoding='ascii') + + result = testdir.runpytest("--doctest-encoding=ascii") + + result.stdout.fnmatch_lines([ + '*1 passed*', + ]) + + def test_encoding_latin1(self, testdir): + """Test support for --doctest-encoding option. + """ + testdir._makefile(".txt", [""" + >>> 'üäö' + 'üäö' + """], {}, encoding='latin1') + + result = testdir.runpytest("--doctest-encoding=latin1") + + result.stdout.fnmatch_lines([ + '*1 passed*', + ]) + + def test_encoding_utf8(self, testdir): + """Test support for --doctest-encoding option. + """ + testdir.maketxtfile(""" + >>> 'üäö' + 'üäö' + """) + + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + '*1 passed*', + ]) + + result = testdir.runpytest("--doctest-encoding=utf-8") + result.stdout.fnmatch_lines([ + '*1 passed*', + ]) + def test_doctest_unexpected_exception(self, testdir): testdir.maketxtfile(""" >>> i = 0 From 929912de2920fdf82a2b5303e9585803368234cd Mon Sep 17 00:00:00 2001 From: Manuel Krebber Date: Tue, 29 Nov 2016 14:51:17 +0100 Subject: [PATCH 3/8] Changed the tests to pass on python 2 as well. --- testing/test_doctest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 4ecd5744b..d48e58042 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -146,9 +146,9 @@ class TestDoctests: def test_encoding_latin1(self, testdir): """Test support for --doctest-encoding option. """ - testdir._makefile(".txt", [""" - >>> 'üäö' - 'üäö' + testdir._makefile(".txt", [u""" + >>> len(u'üäö') + 3 """], {}, encoding='latin1') result = testdir.runpytest("--doctest-encoding=latin1") @@ -160,9 +160,9 @@ class TestDoctests: def test_encoding_utf8(self, testdir): """Test support for --doctest-encoding option. """ - testdir.maketxtfile(""" - >>> 'üäö' - 'üäö' + testdir.maketxtfile(u""" + >>> len(u'üäö') + 3 """) result = testdir.runpytest() From c043bbb8545eecf991cda9bfab95b78bae9ecd08 Mon Sep 17 00:00:00 2001 From: Manuel Krebber Date: Wed, 30 Nov 2016 11:43:33 +0100 Subject: [PATCH 4/8] Changed the doctest_encoding option to an ini option. Parametrized the tests for it. --- _pytest/doctest.py | 9 +++---- testing/test_doctest.py | 57 ++++++++++++++--------------------------- 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/_pytest/doctest.py b/_pytest/doctest.py index 05acc7c59..4ee21b12d 100644 --- a/_pytest/doctest.py +++ b/_pytest/doctest.py @@ -25,6 +25,7 @@ DOCTEST_REPORT_CHOICES = ( def pytest_addoption(parser): parser.addini('doctest_optionflags', 'option flags for doctests', type="args", default=["ELLIPSIS"]) + parser.addini("doctest_encoding", 'encoding used for doctest files', default="utf-8") group = parser.getgroup("collect") group.addoption("--doctest-modules", action="store_true", default=False, @@ -43,10 +44,6 @@ def pytest_addoption(parser): action="store_true", default=False, help="ignore doctest ImportErrors", dest="doctest_ignore_import_errors") - group.addoption("--doctest-encoding", - type=str.lower, default="utf-8", - help="choose the encoding to use for doctest files", - dest="doctestencoding") def pytest_collect_file(path, parent): @@ -166,7 +163,6 @@ def get_optionflags(parent): flag_acc |= flag_lookup_table[flag] return flag_acc - class DoctestTextfile(pytest.Module): obj = None @@ -175,7 +171,8 @@ class DoctestTextfile(pytest.Module): # inspired by doctest.testfile; ideally we would use it directly, # but it doesn't support passing a custom checker - text = self.fspath.read_text(self.config.getoption("doctestencoding")) + encoding = self.config.getini("doctest_encoding") + text = self.fspath.read_text(encoding) filename = str(self.fspath) name = self.fspath.basename globs = {'__name__': '__main__'} diff --git a/testing/test_doctest.py b/testing/test_doctest.py index d48e58042..02f4c3b26 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -129,48 +129,29 @@ class TestDoctests: '*1 passed*', ]) - def test_encoding_ascii(self, testdir): - """Test support for --doctest-encoding option. + @pytest.mark.parametrize( + ' test_string, encoding', + [ + (u'foo', 'ascii'), + (u'öäü', 'latin1'), + (u'öäü', 'utf-8') + ] + ) + def test_encoding(self, testdir, test_string, encoding): + """Test support for doctest_encoding ini option. """ - testdir._makefile(".txt", [""" - >>> 1 - 1 - """], {}, encoding='ascii') - - result = testdir.runpytest("--doctest-encoding=ascii") - - result.stdout.fnmatch_lines([ - '*1 passed*', - ]) - - def test_encoding_latin1(self, testdir): - """Test support for --doctest-encoding option. - """ - testdir._makefile(".txt", [u""" - >>> len(u'üäö') - 3 - """], {}, encoding='latin1') - - result = testdir.runpytest("--doctest-encoding=latin1") - - result.stdout.fnmatch_lines([ - '*1 passed*', - ]) - - def test_encoding_utf8(self, testdir): - """Test support for --doctest-encoding option. - """ - testdir.maketxtfile(u""" - >>> len(u'üäö') - 3 - """) + testdir.makeini(""" + [pytest] + doctest_encoding={0} + """.format(encoding)) + doctest = u""" + >>> u"{}" + {} + """.format(test_string, repr(test_string)) + testdir._makefile(".txt", [doctest], {}, encoding=encoding) result = testdir.runpytest() - result.stdout.fnmatch_lines([ - '*1 passed*', - ]) - result = testdir.runpytest("--doctest-encoding=utf-8") result.stdout.fnmatch_lines([ '*1 passed*', ]) From 1f62e5b5a0fd0ec8ab13e409eb2cd5e198ec88a0 Mon Sep 17 00:00:00 2001 From: Manuel Krebber Date: Wed, 30 Nov 2016 11:50:11 +0100 Subject: [PATCH 5/8] Added CHANGELOG entry and myself to AUTHORS. --- AUTHORS | 1 + CHANGELOG.rst | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 8b07e8bf2..609eb3c77 100644 --- a/AUTHORS +++ b/AUTHORS @@ -86,6 +86,7 @@ Lukas Bednar Luke Murphy Maciek Fijalkowski Maho +Manuel Krebber Marc Schlaich Marcin Bachry Mark Abramowitz diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dd5c68123..baecaf80e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,8 @@ New Features ------------ -* +* Added an ini option ``doctest_encoding`` to specify which encoding to use for doctest files. + Thanks `@wheerd`_ for the PR (`#2101`_). * @@ -21,10 +22,10 @@ Changes assertion messages if verbosity < 2 (`#1512`_). Thanks `@mattduck`_ for the PR -* ``--pdbcls`` no longer implies ``--pdb``. This makes it possible to use +* ``--pdbcls`` no longer implies ``--pdb``. This makes it possible to use ``addopts=--pdbcls=module.SomeClass`` on ``pytest.ini``. Thanks `@davidszotten`_ for the PR (`#1952`_). -* Change exception raised by ``capture.DontReadFromInput.fileno()`` from ``ValueError`` +* Change exception raised by ``capture.DontReadFromInput.fileno()`` from ``ValueError`` to ``io.UnsupportedOperation``. Thanks `@vlad-dragos`_ for the PR. @@ -34,11 +35,13 @@ Changes .. _@davidszotten: https://github.com/davidszotten .. _@fushi: https://github.com/fushi .. _@mattduck: https://github.com/mattduck +.. _@wheerd: https://github.com/wheerd .. _#1512: https://github.com/pytest-dev/pytest/issues/1512 .. _#1874: https://github.com/pytest-dev/pytest/pull/1874 .. _#1952: https://github.com/pytest-dev/pytest/pull/1952 .. _#2013: https://github.com/pytest-dev/pytest/issues/2013 +.. _#2101: https://github.com/pytest-dev/pytest/pull/2101 3.0.5.dev0 From b7fb9fac91069d2d193b78265567f0d3c5673669 Mon Sep 17 00:00:00 2001 From: Manuel Krebber Date: Wed, 30 Nov 2016 14:17:54 +0100 Subject: [PATCH 6/8] Fixed the documentation for doctest_encoding. --- doc/en/doctest.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 93d7c30c4..5967ba047 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -12,9 +12,13 @@ on the command line. Since version ``2.9``, ``--doctest-glob`` can be given multiple times in the command-line. You can specify the encoding that will be used for those doctest files -using the ``--doctest-encoding`` command line option:: +using the ``doctest_encoding`` ini option: - pytest --doctest-encoding='ascii' +.. code-block:: ini + + # content of pytest.ini + [pytest] + doctest_encoding = latin1 The default encoding is UTF-8. From f8fef07b4c1399ff84b41a3fa4d28d93cc578fc4 Mon Sep 17 00:00:00 2001 From: Manuel Krebber Date: Wed, 30 Nov 2016 14:19:07 +0100 Subject: [PATCH 7/8] Fixed the tests for python 2.6 --- testing/test_doctest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 02f4c3b26..72ba0a622 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -145,8 +145,8 @@ class TestDoctests: doctest_encoding={0} """.format(encoding)) doctest = u""" - >>> u"{}" - {} + >>> u"{0}" + {1} """.format(test_string, repr(test_string)) testdir._makefile(".txt", [doctest], {}, encoding=encoding) From 2edfc805afefbbabdeccdb9b4e9fd6ab89335008 Mon Sep 17 00:00:00 2001 From: Wheerd Date: Wed, 30 Nov 2016 18:01:00 +0100 Subject: [PATCH 8/8] Added "versionadded" for doctest_encoding ini option to docs. --- doc/en/doctest.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 5967ba047..a82e3ed9b 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -11,16 +11,18 @@ can change the pattern by issuing:: on the command line. Since version ``2.9``, ``--doctest-glob`` can be given multiple times in the command-line. -You can specify the encoding that will be used for those doctest files -using the ``doctest_encoding`` ini option: +.. versionadded:: 3.1 -.. code-block:: ini + You can specify the encoding that will be used for those doctest files + using the ``doctest_encoding`` ini option: - # content of pytest.ini - [pytest] - doctest_encoding = latin1 + .. code-block:: ini -The default encoding is UTF-8. + # content of pytest.ini + [pytest] + doctest_encoding = latin1 + + The default encoding is UTF-8. You can also trigger running of doctests from docstrings in all python modules (including regular