From 32b85e4cccaf2981837063a316021e48ab73e958 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 12:57:27 +0100 Subject: [PATCH 01/40] ci: Travis: use xdist with pluggymaster jobs --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba2ba37f2..9ec72187f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,10 @@ env: # Specialized factors for py27. - TOXENV=py27-nobyte - TOXENV=py27-xdist - - TOXENV=py27-pluggymaster + - TOXENV=py27-pluggymaster-xdist # Specialized factors for py37. - TOXENV=py37-pexpect,py37-trial,py37-numpy - - TOXENV=py37-pluggymaster + - TOXENV=py37-pluggymaster-xdist - TOXENV=py37-freeze PYTEST_NO_COVERAGE=1 matrix: From 904f1ca1ceef479ae549e2e61cc60358c83453da Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 12:31:31 +0100 Subject: [PATCH 02/40] ci: Travis: macos: use existing py37 `brew update` and `brew upgrade python` is very slow. This uses the existing `/usr/local/bin/python3` (3.7.0). --- .travis.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba2ba37f2..14d4e166a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,20 +43,23 @@ jobs: python: '3.6' - env: TOXENV=py37 - &test-macos - language: generic os: osx - osx_image: xcode9.4 - sudo: required + osx_image: xcode10.1 + language: generic install: - python -m pip install --pre tox + before_install: + - python -V + - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 env: TOXENV=py27-xdist - <<: *test-macos env: TOXENV=py37-xdist before_install: - - brew update - - brew upgrade python - - brew unlink python - - brew link python + - which python3 + - python3 -V + - ln -sfn "$(which python3)" /usr/local/bin/python + - python -V + - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 # Jobs only run via Travis cron jobs (currently daily). - env: TOXENV=py38-xdist From f85f36ed038618350409941fa5c38aa93d99b3b1 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 13 Feb 2019 18:35:38 +0100 Subject: [PATCH 03/40] AppVeyor: drop pluggymaster --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 773e16b1b..da4def8f1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,14 +6,13 @@ environment: - TOXENV: "py34-xdist" - TOXENV: "py35-xdist" - TOXENV: "py36-xdist" + # NOTE: pypy-xdist is buggy currently (https://github.com/pytest-dev/pytest-xdist/issues/142). - TOXENV: "pypy" PYTEST_NO_COVERAGE: "1" # Specialized factors for py27. - TOXENV: "py27-trial,py27-numpy,py27-nobyte" - - TOXENV: "py27-pluggymaster" # Specialized factors for py37. - TOXENV: "py37-trial,py37-numpy" - - TOXENV: "py37-pluggymaster" - TOXENV: "py37-freeze" PYTEST_NO_COVERAGE: "1" From 4eb40ef283b6177f02cffc68b4fc9b332d4a6a34 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 14:03:15 +0100 Subject: [PATCH 04/40] ci: azure: use xdist with pluggymaster --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f375bbc9..049980efb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,9 +27,9 @@ jobs: python.version: '2.7' tox.env: 'py27-trial' python.needs_vc: True - py27-pluggymaster: + py27-pluggymaster-xdist: python.version: '2.7' - tox.env: 'pluggymaster' + tox.env: 'py27-pluggymaster-xdist' pypy: python.version: 'pypy' tox.env: 'pypy' @@ -55,9 +55,9 @@ jobs: py37-trial/numpy: python.version: '3.7' tox.env: 'py37-trial,py37-numpy' - py37-pluggymaster: + py37-pluggymaster-xdist: python.version: '3.7' - tox.env: 'py37-pluggymaster' + tox.env: 'py37-pluggymaster-xdist' maxParallel: 10 steps: From d5d190335c66e5a1212766524b233f5a409e5948 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 25 Feb 2019 20:27:45 -0300 Subject: [PATCH 05/40] Upload code coverage from azure --- appveyor.yml | 8 +++++--- azure-pipelines.yml | 12 ++++++++++++ scripts/{appveyor-retry.cmd => retry.cmd} | 0 scripts/upload-coverage.bat | 15 ++++++++++----- 4 files changed, 27 insertions(+), 8 deletions(-) rename scripts/{appveyor-retry.cmd => retry.cmd} (100%) diff --git a/appveyor.yml b/appveyor.yml index da4def8f1..d00cd143f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,10 +23,12 @@ install: - echo Installed Pythons - dir c:\Python* + - set PATH=C:\Python36;C:\Python36\Scripts;%PATH% + - if "%TOXENV%" == "pypy" call scripts\install-pypy.bat - - C:\Python36\python -m pip install --upgrade pip - - C:\Python36\python -m pip install --upgrade --pre tox + - python -m pip install --upgrade pip + - python -m pip install --upgrade --pre tox build: false # Not a C# project, build stuff at the test step instead. @@ -34,7 +36,7 @@ before_test: - call scripts\prepare-coverage.bat test_script: - - C:\Python36\python -m tox + - python -m tox on_success: - call scripts\upload-coverage.bat diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 049980efb..881f7dc22 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,6 +6,10 @@ variables: PYTEST_ADDOPTS: "--junitxml=build/test-results/$(tox.env).xml" python.needs_vc: False python.exe: "python" + COVERAGE_FILE: "$(Build.Repository.LocalPath)/.coverage" + COVERAGE_PROCESS_START: "$(Build.Repository.LocalPath)/.coveragerc" + _PYTEST_TOX_COVERAGE_RUN: "coverage run -m" + _PYTEST_TOX_EXTRA_DEP: "coverage-enable-subprocess" jobs: @@ -34,6 +38,9 @@ jobs: python.version: 'pypy' tox.env: 'pypy' python.exe: 'pypy' + _PYTEST_TOX_COVERAGE_RUN: "" + _PYTEST_TOX_EXTRA_DEP: "" + PYTEST_NO_COVERAGE: "1" py34: python.version: '3.4' tox.env: 'py34' @@ -99,3 +106,8 @@ jobs: testResultsFiles: 'build/test-results/$(tox.env).xml' testRunTitle: '$(tox.env)' condition: succeededOrFailed() + + - script: call scripts\upload-coverage.bat + displayName: 'Upload coverage' + env: + CODECOV_TOKEN: $(CODECOV_TOKEN) diff --git a/scripts/appveyor-retry.cmd b/scripts/retry.cmd similarity index 100% rename from scripts/appveyor-retry.cmd rename to scripts/retry.cmd diff --git a/scripts/upload-coverage.bat b/scripts/upload-coverage.bat index 7391f817c..b51d4060f 100644 --- a/scripts/upload-coverage.bat +++ b/scripts/upload-coverage.bat @@ -1,11 +1,16 @@ REM script called by AppVeyor to combine and upload coverage information to codecov if not defined PYTEST_NO_COVERAGE ( echo Prepare to upload coverage information - C:\Python36\Scripts\pip install codecov - C:\Python36\Scripts\coverage combine - C:\Python36\Scripts\coverage xml --ignore-errors - C:\Python36\Scripts\coverage report -m --ignore-errors - scripts\appveyor-retry C:\Python36\Scripts\codecov --required -X gcov pycov search -f coverage.xml --flags windows + if defined CODECOV_TOKEN ( + echo CODECOV_TOKEN defined + ) else ( + echo CODECOV_TOKEN NOT defined + ) + python -m pip install codecov + coverage combine + coverage xml --ignore-errors + coverage report -m --ignore-errors + scripts\retry codecov --required -X gcov pycov search -f coverage.xml --flags windows ) else ( echo Skipping coverage upload, PYTEST_NO_COVERAGE is set ) From 4d9296c71f2e2330c823ab2ecd20ef3779423a46 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 17:06:03 +0100 Subject: [PATCH 06/40] ci: Travis: use single install section --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c18ac9e9..83429f355 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ stages: python: - '3.7' install: - - pip install --upgrade --pre tox + - python -m pip install --upgrade --pre tox env: matrix: - TOXENV=py27 @@ -46,8 +46,6 @@ jobs: os: osx osx_image: xcode10.1 language: generic - install: - - python -m pip install --pre tox before_install: - python -V - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 From 3fbe100a02b6bddd39354890bd3fe135559a7d9e Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 21:14:54 +0100 Subject: [PATCH 07/40] ci: Travis: remove coverage from some jobs --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 83429f355..17087bf95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,10 @@ env: # Specialized factors for py27. - TOXENV=py27-nobyte - TOXENV=py27-xdist - - TOXENV=py27-pluggymaster-xdist + - TOXENV=py27-pluggymaster-xdist PYTEST_NO_COVERAGE=1 # Specialized factors for py37. - TOXENV=py37-pexpect,py37-trial,py37-numpy - - TOXENV=py37-pluggymaster-xdist + - TOXENV=py37-pluggymaster-xdist PYTEST_NO_COVERAGE=1 - TOXENV=py37-freeze PYTEST_NO_COVERAGE=1 matrix: @@ -60,7 +60,7 @@ jobs: - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 # Jobs only run via Travis cron jobs (currently daily). - - env: TOXENV=py38-xdist + - env: TOXENV=py38-xdist PYTEST_NO_COVERAGE=1 python: '3.8-dev' if: type = cron From 86070f0b7dfa0308c818c9a23051f6a9d33dc19c Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 21:16:01 +0100 Subject: [PATCH 08/40] ci: AppVeyor: no coverage reporting Should be covered by Azure. --- appveyor.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d00cd143f..230fc6b9f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,6 @@ environment: + global: + PYTEST_NO_COVERAGE: "1" matrix: - TOXENV: "py37-xdist" - TOXENV: "py27-xdist" @@ -8,13 +10,11 @@ environment: - TOXENV: "py36-xdist" # NOTE: pypy-xdist is buggy currently (https://github.com/pytest-dev/pytest-xdist/issues/142). - TOXENV: "pypy" - PYTEST_NO_COVERAGE: "1" # Specialized factors for py27. - TOXENV: "py27-trial,py27-numpy,py27-nobyte" # Specialized factors for py37. - TOXENV: "py37-trial,py37-numpy" - TOXENV: "py37-freeze" - PYTEST_NO_COVERAGE: "1" matrix: fast_finish: true @@ -38,9 +38,6 @@ before_test: test_script: - python -m tox -on_success: - - call scripts\upload-coverage.bat - cache: - '%LOCALAPPDATA%\pip\cache' - '%USERPROFILE%\.cache\pre-commit' From e3eb26f91affb9cc1534886470e231e723741c0e Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 21:24:03 +0100 Subject: [PATCH 09/40] ci: Travis: no coverage by default [skip appveyor] --- .travis.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17087bf95..9616a0818 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,15 +12,15 @@ install: - python -m pip install --upgrade --pre tox env: matrix: - - TOXENV=py27 + - TOXENV=py27 PYTEST_COVERAGE=1 # Specialized factors for py27. - - TOXENV=py27-nobyte - - TOXENV=py27-xdist - - TOXENV=py27-pluggymaster-xdist PYTEST_NO_COVERAGE=1 + - TOXENV=py27-nobyte PYTEST_COVERAGE=1 + - TOXENV=py27-xdist PYTEST_COVERAGE=1 + - TOXENV=py27-pluggymaster-xdist # Specialized factors for py37. - TOXENV=py37-pexpect,py37-trial,py37-numpy - - TOXENV=py37-pluggymaster-xdist PYTEST_NO_COVERAGE=1 - - TOXENV=py37-freeze PYTEST_NO_COVERAGE=1 + - TOXENV=py37-pluggymaster-xdist + - TOXENV=py37-freeze matrix: allow_failures: @@ -30,9 +30,9 @@ matrix: jobs: include: # Coverage tracking is slow with pypy, skip it. - - env: TOXENV=pypy-xdist PYTEST_NO_COVERAGE=1 + - env: TOXENV=pypy-xdist python: 'pypy2.7-6.0' - - env: TOXENV=pypy3-xdist PYTEST_NO_COVERAGE=1 + - env: TOXENV=pypy3-xdist python: 'pypy3.5-6.0' - env: TOXENV=py34-xdist @@ -60,7 +60,7 @@ jobs: - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 # Jobs only run via Travis cron jobs (currently daily). - - env: TOXENV=py38-xdist PYTEST_NO_COVERAGE=1 + - env: TOXENV=py38-xdist python: '3.8-dev' if: type = cron @@ -72,7 +72,6 @@ jobs: - stage: deploy python: '3.6' - env: PYTEST_NO_COVERAGE=1 install: pip install -U setuptools setuptools_scm script: skip deploy: @@ -88,7 +87,7 @@ jobs: before_script: - | - if [[ "$PYTEST_NO_COVERAGE" != 1 ]]; then + if [[ "$PYTEST_COVERAGE" = 1 ]]; then export COVERAGE_FILE="$PWD/.coverage" export COVERAGE_PROCESS_START="$PWD/.coveragerc" export _PYTEST_TOX_COVERAGE_RUN="coverage run -m" @@ -99,7 +98,7 @@ script: tox --recreate after_success: - | - if [[ "$PYTEST_NO_COVERAGE" != 1 ]]; then + if [[ "$PYTEST_COVERAGE" = 1 ]]; then set -e # Add last TOXENV to $PATH. PATH="$PWD/.tox/${TOXENV##*,}/bin:$PATH" From fb7ee7f42ca4b451b4e85bb82033ce358274bc34 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 26 Feb 2019 19:01:59 -0300 Subject: [PATCH 10/40] Add badge for azure pipelines --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index bd7f1daa6..81b9d1ed8 100644 --- a/README.rst +++ b/README.rst @@ -22,6 +22,9 @@ .. image:: https://travis-ci.org/pytest-dev/pytest.svg?branch=master :target: https://travis-ci.org/pytest-dev/pytest +.. image:: https://dev.azure.com/pytest-dev/pytest/_apis/build/status/pytest-CI?branchName=master + :target: https://dev.azure.com/pytest-dev/pytest + .. image:: https://ci.appveyor.com/api/projects/status/mrgbjaua7t33pg6b?svg=true :target: https://ci.appveyor.com/project/pytestbot/pytest From 6bd77c0abd3bb203a5c39b67a89d6ff6af079cc5 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 26 Feb 2019 19:03:02 -0300 Subject: [PATCH 11/40] Remove appveyor.yml and related scripts --- README.rst | 3 --- appveyor.yml | 47 ------------------------------------ scripts/install-pypy.bat | 6 ----- scripts/prepare-coverage.bat | 10 -------- 4 files changed, 66 deletions(-) delete mode 100644 appveyor.yml delete mode 100644 scripts/install-pypy.bat delete mode 100644 scripts/prepare-coverage.bat diff --git a/README.rst b/README.rst index 81b9d1ed8..1955d3503 100644 --- a/README.rst +++ b/README.rst @@ -25,9 +25,6 @@ .. image:: https://dev.azure.com/pytest-dev/pytest/_apis/build/status/pytest-CI?branchName=master :target: https://dev.azure.com/pytest-dev/pytest -.. image:: https://ci.appveyor.com/api/projects/status/mrgbjaua7t33pg6b?svg=true - :target: https://ci.appveyor.com/project/pytestbot/pytest - .. image:: https://img.shields.io/badge/code%20style-black-000000.svg :target: https://github.com/ambv/black diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 230fc6b9f..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,47 +0,0 @@ -environment: - global: - PYTEST_NO_COVERAGE: "1" - matrix: - - TOXENV: "py37-xdist" - - TOXENV: "py27-xdist" - - TOXENV: "linting,docs,doctesting" - - TOXENV: "py34-xdist" - - TOXENV: "py35-xdist" - - TOXENV: "py36-xdist" - # NOTE: pypy-xdist is buggy currently (https://github.com/pytest-dev/pytest-xdist/issues/142). - - TOXENV: "pypy" - # Specialized factors for py27. - - TOXENV: "py27-trial,py27-numpy,py27-nobyte" - # Specialized factors for py37. - - TOXENV: "py37-trial,py37-numpy" - - TOXENV: "py37-freeze" - -matrix: - fast_finish: true - -install: - - echo Installed Pythons - - dir c:\Python* - - - set PATH=C:\Python36;C:\Python36\Scripts;%PATH% - - - if "%TOXENV%" == "pypy" call scripts\install-pypy.bat - - - python -m pip install --upgrade pip - - python -m pip install --upgrade --pre tox - -build: false # Not a C# project, build stuff at the test step instead. - -before_test: - - call scripts\prepare-coverage.bat - -test_script: - - python -m tox - -cache: - - '%LOCALAPPDATA%\pip\cache' - - '%USERPROFILE%\.cache\pre-commit' - -# We don't deploy anything on tags with AppVeyor, we use Travis instead, so we -# might as well save resources -skip_tags: true diff --git a/scripts/install-pypy.bat b/scripts/install-pypy.bat deleted file mode 100644 index 8012ea46a..000000000 --- a/scripts/install-pypy.bat +++ /dev/null @@ -1,6 +0,0 @@ -REM install pypy using choco -REM redirect to a file because choco install python.pypy is too noisy. If the command fails, write output to console -choco install python.pypy > pypy-inst.log 2>&1 || (type pypy-inst.log & exit /b 1) -set PATH=C:\tools\pypy\pypy;%PATH% # so tox can find pypy -echo PyPy installed -pypy --version diff --git a/scripts/prepare-coverage.bat b/scripts/prepare-coverage.bat deleted file mode 100644 index bff1e62b0..000000000 --- a/scripts/prepare-coverage.bat +++ /dev/null @@ -1,10 +0,0 @@ -REM scripts called by AppVeyor to setup the environment variables to enable coverage -if not defined PYTEST_NO_COVERAGE ( - set "COVERAGE_FILE=%CD%\.coverage" - set "COVERAGE_PROCESS_START=%CD%\.coveragerc" - set "_PYTEST_TOX_COVERAGE_RUN=coverage run -m" - set "_PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess" - echo Coverage setup completed -) else ( - echo Skipping coverage setup, PYTEST_NO_COVERAGE is set -) From cd9415baf240b81b63c5f5c783d59c2021295559 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 26 Feb 2019 19:22:23 -0300 Subject: [PATCH 12/40] Configure azure to enable coverage on PYTEST_COVERAGE=1, similar to Travis --- azure-pipelines.yml | 21 +++++++++++++++------ scripts/setup-coverage-vars.bat | 7 +++++++ scripts/upload-coverage.bat | 6 +++--- 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 scripts/setup-coverage-vars.bat diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 881f7dc22..949c39fc4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,8 +8,7 @@ variables: python.exe: "python" COVERAGE_FILE: "$(Build.Repository.LocalPath)/.coverage" COVERAGE_PROCESS_START: "$(Build.Repository.LocalPath)/.coveragerc" - _PYTEST_TOX_COVERAGE_RUN: "coverage run -m" - _PYTEST_TOX_EXTRA_DEP: "coverage-enable-subprocess" + PYTEST_COVERAGE: '0' jobs: @@ -21,16 +20,20 @@ jobs: py27: python.version: '2.7' tox.env: 'py27' + PYTEST_COVERAGE: '1' py27-xdist: python.version: '2.7' tox.env: 'py27-xdist' + PYTEST_COVERAGE: '1' py27-numpy/nobyte: python.version: '2.7' tox.env: 'py27-numpy,py27-nobyte' + PYTEST_COVERAGE: '1' py27-trial: python.version: '2.7' tox.env: 'py27-trial' python.needs_vc: True + PYTEST_COVERAGE: '1' py27-pluggymaster-xdist: python.version: '2.7' tox.env: 'py27-pluggymaster-xdist' @@ -38,30 +41,33 @@ jobs: python.version: 'pypy' tox.env: 'pypy' python.exe: 'pypy' - _PYTEST_TOX_COVERAGE_RUN: "" - _PYTEST_TOX_EXTRA_DEP: "" - PYTEST_NO_COVERAGE: "1" py34: python.version: '3.4' tox.env: 'py34' + PYTEST_COVERAGE: '1' py35: python.version: '3.5' tox.env: 'py35' + PYTEST_COVERAGE: '1' py36: python.version: '3.6' tox.env: 'py36' + PYTEST_COVERAGE: '1' py37: python.version: '3.7' tox.env: 'py37' + PYTEST_COVERAGE: '1' py37-linting/docs/doctesting: python.version: '3.7' tox.env: 'linting,docs,doctesting' py37-xdist: python.version: '3.7' tox.env: 'py37-xdist' + PYTEST_COVERAGE: '1' py37-trial/numpy: python.version: '3.7' tox.env: 'py37-trial,py37-numpy' + PYTEST_COVERAGE: '1' py37-pluggymaster-xdist: python.version: '3.7' tox.env: 'py37-pluggymaster-xdist' @@ -98,7 +104,9 @@ jobs: - script: $(python.exe) -m pip install --upgrade pip && $(python.exe) -m pip install tox displayName: 'Install tox' - - script: $(python.exe) -m tox -e $(tox.env) + - script: | + call scripts/setup-coverage-vars.bat || goto :eof + $(python.exe) -m tox -e $(tox.env) displayName: 'Run tests' - task: PublishTestResults@2 @@ -109,5 +117,6 @@ jobs: - script: call scripts\upload-coverage.bat displayName: 'Upload coverage' + condition: eq(variables['PYTEST_COVERAGE'], '1') env: CODECOV_TOKEN: $(CODECOV_TOKEN) diff --git a/scripts/setup-coverage-vars.bat b/scripts/setup-coverage-vars.bat new file mode 100644 index 000000000..7a4a6d4de --- /dev/null +++ b/scripts/setup-coverage-vars.bat @@ -0,0 +1,7 @@ +if "%PYTEST_COVERAGE%" == "1" ( + set "_PYTEST_TOX_COVERAGE_RUN=coverage run -m" + set "_PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess" + echo Coverage vars configured, PYTEST_COVERAGE=%PYTEST_COVERAGE% +) else ( + echo Skipping coverage vars setup, PYTEST_COVERAGE=%PYTEST_COVERAGE% +) diff --git a/scripts/upload-coverage.bat b/scripts/upload-coverage.bat index b51d4060f..2e84f99dd 100644 --- a/scripts/upload-coverage.bat +++ b/scripts/upload-coverage.bat @@ -1,5 +1,5 @@ -REM script called by AppVeyor to combine and upload coverage information to codecov -if not defined PYTEST_NO_COVERAGE ( +REM script called by Azure to combine and upload coverage information to codecov +if "%PYTEST_COVERAGE%" == "1" ( echo Prepare to upload coverage information if defined CODECOV_TOKEN ( echo CODECOV_TOKEN defined @@ -12,5 +12,5 @@ if not defined PYTEST_NO_COVERAGE ( coverage report -m --ignore-errors scripts\retry codecov --required -X gcov pycov search -f coverage.xml --flags windows ) else ( - echo Skipping coverage upload, PYTEST_NO_COVERAGE is set + echo Skipping coverage upload, PYTEST_COVERAGE=%PYTEST_COVERAGE% ) From 16cbb3196cc376014ebbf53af45928d65021cb88 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 26 Feb 2019 20:39:06 -0300 Subject: [PATCH 13/40] Document how to disable caching rewritten .pyc files to disk Also changed how the section is presented: instead of "Note" blocks, use proper sections as those contain enough information to exist on their own. Fix #1680 --- doc/en/assert.rst | 60 ++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/doc/en/assert.rst b/doc/en/assert.rst index b119adcf0..adefe89bb 100644 --- a/doc/en/assert.rst +++ b/doc/en/assert.rst @@ -252,8 +252,8 @@ the conftest file: .. _assert-details: .. _`assert introspection`: -Advanced assertion introspection ----------------------------------- +Assertion introspection details +------------------------------- .. versionadded:: 2.1 @@ -266,28 +266,46 @@ supporting modules which are not themselves test modules will not be rewritten** You can manually enable assertion rewriting for an imported module by calling `register_assert_rewrite `_ -before you import it (a good place to do that is in ``conftest.py``). - -.. note:: - - ``pytest`` rewrites test modules on import by using an import - hook to write new ``pyc`` files. Most of the time this works transparently. - However, if you are messing with import yourself, the import hook may - interfere. - - If this is the case you have two options: - - * Disable rewriting for a specific module by adding the string - ``PYTEST_DONT_REWRITE`` to its docstring. - - * Disable rewriting for all modules by using ``--assert=plain``. - - Additionally, rewriting will fail silently if it cannot write new ``.pyc`` files, - i.e. in a read-only filesystem or a zipfile. - +before you import it (a good place to do that is in your root ``conftest.py``). For further information, Benjamin Peterson wrote up `Behind the scenes of pytest's new assertion rewriting `_. +Assertion rewriting caches files on disk +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``pytest`` will write back the rewritten modules to disk for caching. You can disable +this behavior (for example to avoid leaving stable ``.pyc`` files around in projects that +move files around a lot) by adding this to the top of your ``conftest.py`` file: + +.. code-block:: python + + import sys + + sys.dont_write_bytecode = True + +Note that you still get the benefits of assertion introspection, only change is that +the ``.pyc`` files won't be cached on disk. + +Additionally, rewriting will silently skip caching if it cannot write new ``.pyc`` files, +i.e. in a read-only filesystem or a zipfile. + + +Disabling assert rewriting +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``pytest`` rewrites test modules on import by using an import +hook to write new ``pyc`` files. Most of the time this works transparently. +However, if you are messing with import yourself, the import hook may +interfere. + +If this is the case you have two options: + +* Disable rewriting for a specific module by adding the string + ``PYTEST_DONT_REWRITE`` to its docstring. + +* Disable rewriting for all modules by using ``--assert=plain``. + + .. versionadded:: 2.1 Add assert rewriting as an alternate introspection technique. From b5cf61312b96ba304c7aa24420a36a8f908e9935 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 26 Feb 2019 23:30:48 +0100 Subject: [PATCH 14/40] coverage: use source=. and report.include This appears to improve performance - ~4s with `tox -e py37-coverage -- testing/test_collection.py`. --- .coveragerc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 97934dc3b..6cb6f4973 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,11 @@ [run] -source = pytest,_pytest,testing/ +source = . parallel = 1 branch = 1 +[report] +include = src/*, testing/* + [paths] source = src/ .tox/*/lib/python*/site-packages/ From 0138e9cbb001b6ed3aab026737c83642603e9ff1 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 08:44:11 +0100 Subject: [PATCH 15/40] ci: coverage: remove --ignore-errors This should not be necessary (anymore). --- .travis.yml | 4 ++-- scripts/upload-coverage.bat | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9616a0818..7cfc4068a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,8 +103,8 @@ after_success: # Add last TOXENV to $PATH. PATH="$PWD/.tox/${TOXENV##*,}/bin:$PATH" coverage combine - coverage xml --ignore-errors - coverage report -m --ignore-errors + coverage xml + coverage report -m bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -F $TRAVIS_OS_NAME fi diff --git a/scripts/upload-coverage.bat b/scripts/upload-coverage.bat index 2e84f99dd..3f9d423e7 100644 --- a/scripts/upload-coverage.bat +++ b/scripts/upload-coverage.bat @@ -8,8 +8,8 @@ if "%PYTEST_COVERAGE%" == "1" ( ) python -m pip install codecov coverage combine - coverage xml --ignore-errors - coverage report -m --ignore-errors + coverage xml + coverage report -m scripts\retry codecov --required -X gcov pycov search -f coverage.xml --flags windows ) else ( echo Skipping coverage upload, PYTEST_COVERAGE=%PYTEST_COVERAGE% From 5a0f379289dbcec8409ac17cf1ead414f33e0dcc Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 08:46:19 +0100 Subject: [PATCH 16/40] ci: codecov: set name for uploads/builds --- .travis.yml | 2 +- azure-pipelines.yml | 1 + scripts/upload-coverage.bat | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7cfc4068a..be30f0151 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ after_success: coverage combine coverage xml coverage report -m - bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -F $TRAVIS_OS_NAME + bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -F $TRAVIS_OS_NAME -n $TOXENV fi notifications: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 949c39fc4..e27f03ee8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -120,3 +120,4 @@ jobs: condition: eq(variables['PYTEST_COVERAGE'], '1') env: CODECOV_TOKEN: $(CODECOV_TOKEN) + PYTEST_CODECOV_NAME: $(tox.env) diff --git a/scripts/upload-coverage.bat b/scripts/upload-coverage.bat index 3f9d423e7..ad8092d16 100644 --- a/scripts/upload-coverage.bat +++ b/scripts/upload-coverage.bat @@ -10,7 +10,7 @@ if "%PYTEST_COVERAGE%" == "1" ( coverage combine coverage xml coverage report -m - scripts\retry codecov --required -X gcov pycov search -f coverage.xml --flags windows + scripts\retry codecov --required -X gcov pycov search -f coverage.xml --flags windows --name %PYTEST_CODECOV_NAME% ) else ( echo Skipping coverage upload, PYTEST_COVERAGE=%PYTEST_COVERAGE% ) From 31174f3f8306471dc41d266f1f2831f647b344c2 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 09:26:29 +0100 Subject: [PATCH 17/40] .coveragerc: fix/tighten paths --- .coveragerc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index 6cb6f4973..1c20833d1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,6 +7,9 @@ branch = 1 include = src/*, testing/* [paths] -source = src/ - .tox/*/lib/python*/site-packages/ - .tox\*\Lib\site-packages\ +source = src/_pytest + .tox/*/lib/python*/site-packages/_pytest/ + .tox\*\Lib\site-packages\pytest\ +source_pytest_py = src/pytest.py + .tox/*/lib/python*/site-packages/pytest.py + .tox\*\Lib\site-packages\pytest.py From fec656b3b1a62ccbc6e026924577d9c9955bd1a6 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 07:46:03 -0300 Subject: [PATCH 18/40] Apply suggestions from code review Co-Authored-By: nicoddemus --- doc/en/assert.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/en/assert.rst b/doc/en/assert.rst index adefe89bb..e7e78601b 100644 --- a/doc/en/assert.rst +++ b/doc/en/assert.rst @@ -274,7 +274,7 @@ Assertion rewriting caches files on disk ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``pytest`` will write back the rewritten modules to disk for caching. You can disable -this behavior (for example to avoid leaving stable ``.pyc`` files around in projects that +this behavior (for example to avoid leaving stale ``.pyc`` files around in projects that move files around a lot) by adding this to the top of your ``conftest.py`` file: .. code-block:: python @@ -283,7 +283,7 @@ move files around a lot) by adding this to the top of your ``conftest.py`` file: sys.dont_write_bytecode = True -Note that you still get the benefits of assertion introspection, only change is that +Note that you still get the benefits of assertion introspection, the only change is that the ``.pyc`` files won't be cached on disk. Additionally, rewriting will silently skip caching if it cannot write new ``.pyc`` files, @@ -295,7 +295,7 @@ Disabling assert rewriting ``pytest`` rewrites test modules on import by using an import hook to write new ``pyc`` files. Most of the time this works transparently. -However, if you are messing with import yourself, the import hook may +However, if you are working with the import machinery yourself, the import hook may interfere. If this is the case you have two options: From 1e2810e07dff6c3e7171ccba707676bae63beff9 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 09:15:04 +0100 Subject: [PATCH 19/40] tox: generic pexpect factor --- tox.ini | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tox.ini b/tox.ini index 87a56f842..16dbaed90 100644 --- a/tox.ini +++ b/tox.ini @@ -21,23 +21,30 @@ envlist = [testenv] commands = - {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {env:_PYTEST_TOX_ARGS:} {posargs} + {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {posargs:{env:_PYTEST_TOX_DEFAULT_POSARGS}} coverage: coverage combine coverage: coverage report passenv = USER USERNAME COVERAGE_* TRAVIS PYTEST_ADDOPTS setenv = - _PYTEST_TOX_ARGS=--lsof + _PYTEST_TOX_DEFAULT_POSARGS=--lsof + # Configuration to run with coverage similar to Travis/Appveyor, e.g. # "tox -e py37-coverage". coverage: _PYTEST_TOX_COVERAGE_RUN=coverage run -m coverage: _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess coverage: COVERAGE_FILE={toxinidir}/.coverage coverage: COVERAGE_PROCESS_START={toxinidir}/.coveragerc - xdist: _PYTEST_TOX_ARGS={env:_PYTEST_TOX_ARGS:-n auto} + + pexpect: _PYTEST_TOX_PLATFORM=linux|darwin + pexpect: _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_DEFAULT_POSARGS:testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py} + + xdist: _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_DEFAULT_POSARGS:-n auto} extras = testing deps = + pexpect: pexpect xdist: pytest-xdist>=1.13 {env:_PYTEST_TOX_EXTRA_DEP:} +platform = {env:_PYTEST_TOX_PLATFORM:.*} [testenv:py27-subprocess] deps = @@ -54,19 +61,6 @@ basepython = python3 deps = pre-commit>=1.11.0 commands = pre-commit run --all-files --show-diff-on-failure -[testenv:py27-pexpect] -platform = linux|darwin -deps = - {[testenv]deps} - pexpect -commands = - {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {posargs:testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py} - -[testenv:py37-pexpect] -platform = {[testenv:py27-pexpect]platform} -deps = {[testenv:py27-pexpect]deps} -commands = {[testenv:py27-pexpect]commands} - [testenv:py27-nobyte] extras = testing deps = From ee95d666f8c875a7b4d46b2f4e0ce08cb05a0007 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 11:32:23 +0100 Subject: [PATCH 20/40] coverage: run.include, drop --ignore-errors, codecov name --- .coveragerc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.coveragerc b/.coveragerc index 1c20833d1..5942e71c6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,15 +1,15 @@ [run] -source = . +include = + src/* + testing/* + .tox/*/lib/python*/site-packages/_pytest/* + .tox/*/lib/python*/site-packages/pytest.py + .tox\*\Lib\site-packages\_pytest\* + .tox\*\Lib\site-packages\pytest.py parallel = 1 branch = 1 -[report] -include = src/*, testing/* - [paths] -source = src/_pytest - .tox/*/lib/python*/site-packages/_pytest/ - .tox\*\Lib\site-packages\pytest\ -source_pytest_py = src/pytest.py - .tox/*/lib/python*/site-packages/pytest.py - .tox\*\Lib\site-packages\pytest.py +source = src/ + .tox/*/lib/python*/site-packages/ + .tox\*\Lib\site-packages\ From ccab469a0cc7784874424519033e275e804334b6 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 16:07:11 +0100 Subject: [PATCH 21/40] Fix test_argcomplete: use python -m pytest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously it was not run with a) xdist ("-c"), and b) "python -m pytest" ("…/pytest.py", not executable). --- setup.py | 1 + testing/test_parseopt.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index b286a4f20..f5150243a 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ def main(): # fmt: off extras_require={ "testing": [ + "argcomplete", "hypothesis>=3.56", "nose", "requests", diff --git a/testing/test_parseopt.py b/testing/test_parseopt.py index e25705d00..91f9a7d2b 100644 --- a/testing/test_parseopt.py +++ b/testing/test_parseopt.py @@ -299,15 +299,12 @@ def test_argcomplete(testdir, monkeypatch): if not distutils.spawn.find_executable("bash"): pytest.skip("bash not available") script = str(testdir.tmpdir.join("test_argcomplete")) - pytest_bin = sys.argv[0] - if "pytest" not in os.path.basename(pytest_bin): - pytest.skip("need to be run with pytest executable, not {}".format(pytest_bin)) with open(str(script), "w") as fp: # redirect output from argcomplete to stdin and stderr is not trivial # http://stackoverflow.com/q/12589419/1307905 # so we use bash - fp.write('COMP_WORDBREAKS="$COMP_WORDBREAKS" %s 8>&1 9>&2' % pytest_bin) + fp.write('COMP_WORDBREAKS="$COMP_WORDBREAKS" python -m pytest 8>&1 9>&2') # alternative would be exteneded Testdir.{run(),_run(),popen()} to be able # to handle a keyword argument env that replaces os.environ in popen or # extends the copy, advantage: could not forget to restore @@ -323,7 +320,11 @@ def test_argcomplete(testdir, monkeypatch): # argcomplete not found pytest.skip("argcomplete not available") elif not result.stdout.str(): - pytest.skip("bash provided no output, argcomplete not available?") + pytest.skip( + "bash provided no output on stdout, argcomplete not available? (stderr={!r})".format( + result.stderr.str() + ) + ) else: result.stdout.fnmatch_lines(["--funcargs", "--fulltrace"]) os.mkdir("test_argcomplete.d") From 9c590fa474259ad80fa0b42be43813a44f48dec1 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 28 Feb 2019 19:40:24 +0100 Subject: [PATCH 22/40] ci: Travis: skip coverage with cron runs The current commit on features has 50+ uploads already: https://codecov.io/gh/pytest-dev/pytest/commit/c7bbb2a78873a63b9df9df08f8dc77cde30debd3/build --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index be30f0151..d522fcace 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,6 +86,11 @@ jobs: repo: pytest-dev/pytest before_script: + - | + # Do not (re-)upload coverage with cron runs. + if [[ "$TRAVIS_EVENT_TYPE" = cron ]]; then + PYTEST_COVERAGE=0 + fi - | if [[ "$PYTEST_COVERAGE" = 1 ]]; then export COVERAGE_FILE="$PWD/.coverage" From 8b447878dc513342ba872fb902d7cbe03985e973 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 28 Feb 2019 21:24:26 +0100 Subject: [PATCH 23/40] ci: codecov: remove flags completely for now This appears to be one of the reasons for timeouts on their backend. --- .travis.yml | 2 +- scripts/upload-coverage.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index be30f0151..dc75e6c8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ after_success: coverage combine coverage xml coverage report -m - bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -F $TRAVIS_OS_NAME -n $TOXENV + bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV fi notifications: diff --git a/scripts/upload-coverage.bat b/scripts/upload-coverage.bat index ad8092d16..6a4773e18 100644 --- a/scripts/upload-coverage.bat +++ b/scripts/upload-coverage.bat @@ -10,7 +10,7 @@ if "%PYTEST_COVERAGE%" == "1" ( coverage combine coverage xml coverage report -m - scripts\retry codecov --required -X gcov pycov search -f coverage.xml --flags windows --name %PYTEST_CODECOV_NAME% + scripts\retry codecov --required -X gcov pycov search -f coverage.xml --name %PYTEST_CODECOV_NAME% ) else ( echo Skipping coverage upload, PYTEST_COVERAGE=%PYTEST_COVERAGE% ) From c30ab1014ea65ad032fba3c84b196381201c476c Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 28 Feb 2019 22:44:24 +0100 Subject: [PATCH 24/40] tox: add generic nobyte and numpy factors Remove `py27-nobyte` from tox.ini, which was using xdist already. Therefore this also removes `py27-xdist` from Travis. "nobyte" was added in 036557ac to test that test_assertrewrite.py works with a global PYTHONDONTWRITEBYTECODE=1 setting. "numpy" is only a special dependency, and can be run together with nobyte/xdist. --- .travis.yml | 9 ++++----- azure-pipelines.yml | 8 ++------ tox.ini | 28 ++++------------------------ 3 files changed, 10 insertions(+), 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc75e6c8b..69dc06582 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,11 +14,10 @@ env: matrix: - TOXENV=py27 PYTEST_COVERAGE=1 # Specialized factors for py27. - - TOXENV=py27-nobyte PYTEST_COVERAGE=1 - - TOXENV=py27-xdist PYTEST_COVERAGE=1 + - TOXENV=py27-nobyte-numpy-xdist PYTEST_COVERAGE=1 - TOXENV=py27-pluggymaster-xdist # Specialized factors for py37. - - TOXENV=py37-pexpect,py37-trial,py37-numpy + - TOXENV=py37-pexpect,py37-trial - TOXENV=py37-pluggymaster-xdist - TOXENV=py37-freeze @@ -65,8 +64,8 @@ jobs: if: type = cron - stage: baseline - env: TOXENV=py27-pexpect,py27-trial,py27-numpy - - env: TOXENV=py37-xdist + env: TOXENV=py27-pexpect,py27-trial + - env: TOXENV=py37-numpy-xdist - env: TOXENV=linting,docs,doctesting python: '3.7' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e27f03ee8..b648cd3d9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,13 +21,9 @@ jobs: python.version: '2.7' tox.env: 'py27' PYTEST_COVERAGE: '1' - py27-xdist: + py27-nobyte-numpy-xdist: python.version: '2.7' - tox.env: 'py27-xdist' - PYTEST_COVERAGE: '1' - py27-numpy/nobyte: - python.version: '2.7' - tox.env: 'py27-numpy,py27-nobyte' + tox.env: 'py27-nobyte-numpy-xdist' PYTEST_COVERAGE: '1' py27-trial: python.version: '2.7' diff --git a/tox.ini b/tox.ini index 16dbaed90..380dd2539 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ envlist = pypy pypy3 {py27,py37}-{pexpect,xdist,trial,numpy,pluggymaster} - py27-nobyte + py27-nobyte-xdist doctesting py37-freeze docs @@ -35,12 +35,15 @@ setenv = coverage: COVERAGE_FILE={toxinidir}/.coverage coverage: COVERAGE_PROCESS_START={toxinidir}/.coveragerc + nobyte: PYTHONDONTWRITEBYTECODE=1 + pexpect: _PYTEST_TOX_PLATFORM=linux|darwin pexpect: _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_DEFAULT_POSARGS:testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py} xdist: _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_DEFAULT_POSARGS:-n auto} extras = testing deps = + numpy: numpy pexpect: pexpect xdist: pytest-xdist>=1.13 {env:_PYTEST_TOX_EXTRA_DEP:} @@ -61,18 +64,6 @@ basepython = python3 deps = pre-commit>=1.11.0 commands = pre-commit run --all-files --show-diff-on-failure -[testenv:py27-nobyte] -extras = testing -deps = - {[testenv]deps} - pytest-xdist>=1.13 -distribute = true -setenv = - {[testenv]setenv} - PYTHONDONTWRITEBYTECODE=1 -commands = - {env:_PYTEST_TOX_COVERAGE_RUN:} pytest -n auto {posargs} - [testenv:py27-trial] deps = {[testenv]deps} @@ -85,17 +76,6 @@ commands = deps = {[testenv:py27-trial]deps} commands = {[testenv:py27-trial]commands} -[testenv:py27-numpy] -deps = - {[testenv]deps} - numpy -commands= - {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {posargs:testing/python/approx.py} - -[testenv:py37-numpy] -deps = {[testenv:py27-numpy]deps} -commands = {[testenv:py27-numpy]commands} - [testenv:py27-pluggymaster] setenv= {[testenv]setenv} From a51dc0c7ce3b4964dc78c52c47c8210ae8e57553 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 10:08:39 +0100 Subject: [PATCH 25/40] Validate type with writing to captured output like without Fixes https://github.com/pytest-dev/pytest/issues/4861. --- changelog/4861.bugfix.rst | 1 + src/_pytest/capture.py | 5 +++++ testing/test_capture.py | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 changelog/4861.bugfix.rst diff --git a/changelog/4861.bugfix.rst b/changelog/4861.bugfix.rst new file mode 100644 index 000000000..5b3a89aa2 --- /dev/null +++ b/changelog/4861.bugfix.rst @@ -0,0 +1 @@ +Validate type with writing to captured output like without. diff --git a/src/_pytest/capture.py b/src/_pytest/capture.py index 533690949..7bd319b1a 100644 --- a/src/_pytest/capture.py +++ b/src/_pytest/capture.py @@ -17,6 +17,7 @@ from tempfile import TemporaryFile import six import pytest +from _pytest.compat import _PY3 from _pytest.compat import CaptureIO patchsysdict = {0: "stdin", 1: "stdout", 2: "stderr"} @@ -412,6 +413,10 @@ class EncodedFile(object): def write(self, obj): if isinstance(obj, six.text_type): obj = obj.encode(self.encoding, "replace") + elif _PY3: + raise TypeError( + "write() argument must be str, not {}".format(type(obj).__name__) + ) self.buffer.write(obj) def writelines(self, linelist): diff --git a/testing/test_capture.py b/testing/test_capture.py index 81ab4e8a8..546738f28 100644 --- a/testing/test_capture.py +++ b/testing/test_capture.py @@ -18,6 +18,7 @@ from six import text_type import pytest from _pytest import capture from _pytest.capture import CaptureManager +from _pytest.compat import _PY3 from _pytest.main import EXIT_NOTESTSCOLLECTED # note: py.io capture tests where copied from @@ -1526,3 +1527,26 @@ def test_capture_with_live_logging(testdir, capture_fixture): result = testdir.runpytest_subprocess("--log-cli-level=INFO") assert result.ret == 0 + + +def test_typeerror_encodedfile_write(testdir): + """It should behave the same with and without output capturing (#4861).""" + p = testdir.makepyfile( + """ + def test_fails(): + import sys + sys.stdout.write(b"foo") + """ + ) + result_without_capture = testdir.runpytest("-s", str(p)) + + result_with_capture = testdir.runpytest(str(p)) + + assert result_with_capture.ret == result_without_capture.ret + + if _PY3: + result_with_capture.stdout.fnmatch_lines( + ["E TypeError: write() argument must be str, not bytes"] + ) + else: + assert result_with_capture.ret == 0 From 33db5e081d5eedc9e64f3b501ac4c0ce2a497178 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 1 Mar 2019 10:09:29 -0300 Subject: [PATCH 26/40] Tweak changelog --- changelog/4861.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/4861.bugfix.rst b/changelog/4861.bugfix.rst index 5b3a89aa2..b4bf125d1 100644 --- a/changelog/4861.bugfix.rst +++ b/changelog/4861.bugfix.rst @@ -1 +1 @@ -Validate type with writing to captured output like without. +Improve validation of contents written to captured output so it behaves the same as when capture is disabled. From 75e1fde6681a41bfa3ea1649b4ba0888d1dc2299 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 14:24:09 +0100 Subject: [PATCH 27/40] tox.ini: add config for isort --- tox.ini | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tox.ini b/tox.ini index 16dbaed90..997e98c2c 100644 --- a/tox.ini +++ b/tox.ini @@ -205,6 +205,14 @@ filterwarnings = # Do not cause SyntaxError for invalid escape sequences in py37. default:invalid escape sequence:DeprecationWarning pytester_example_dir = testing/example_scripts + [flake8] max-line-length = 120 ignore = E203,W503 + +[isort] +; This config mimics what reorder-python-imports does. +force_single_line = 1 +known_localfolder = pytest,_pytest +known_third_party = test_source,test_excinfo +force_alphabetical_sort_within_sections = 1 From 0d31e852b1ec9f1c8a10483b94f88e508d2b46c2 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 14:23:52 +0100 Subject: [PATCH 28/40] Run isort --- bench/skip.py | 1 - setup.py | 1 - src/_pytest/assertion/truncate.py | 1 - src/_pytest/deprecated.py | 1 - src/_pytest/hookspec.py | 1 - src/_pytest/logging.py | 1 - src/_pytest/main.py | 1 - src/_pytest/mark/structures.py | 1 - src/_pytest/pathlib.py | 1 - testing/code/test_source.py | 1 - .../perf_examples/collect_stats/generate_folders.py | 1 - testing/test_resultlog.py | 1 - testing/test_warnings.py | 1 - 13 files changed, 13 deletions(-) diff --git a/bench/skip.py b/bench/skip.py index 25559cc27..2fc8240e5 100644 --- a/bench/skip.py +++ b/bench/skip.py @@ -2,7 +2,6 @@ from six.moves import range import pytest - SKIP = True diff --git a/setup.py b/setup.py index f5150243a..89e48c6b5 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ import os from setuptools import setup - # TODO: if py gets upgrade to >=1.6, # remove _width_of_current_line in terminal.py INSTALL_REQUIRES = [ diff --git a/src/_pytest/assertion/truncate.py b/src/_pytest/assertion/truncate.py index d19c8b61e..69d6acdef 100644 --- a/src/_pytest/assertion/truncate.py +++ b/src/_pytest/assertion/truncate.py @@ -12,7 +12,6 @@ import os import six - DEFAULT_MAX_LINES = 8 DEFAULT_MAX_CHARS = 8 * 80 USAGE_MSG = "use '-vv' to show" diff --git a/src/_pytest/deprecated.py b/src/_pytest/deprecated.py index 4afde6902..6134ca77b 100644 --- a/src/_pytest/deprecated.py +++ b/src/_pytest/deprecated.py @@ -16,7 +16,6 @@ from _pytest.warning_types import PytestDeprecationWarning from _pytest.warning_types import RemovedInPytest4Warning from _pytest.warning_types import UnformattedWarning - YIELD_TESTS = "yield tests were removed in pytest 4.0 - {name} will be ignored" diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 3b6c9a7d1..0641e3bc5 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -3,7 +3,6 @@ from pluggy import HookspecMarker from _pytest.deprecated import PYTEST_LOGWARNING - hookspec = HookspecMarker("pytest") # ------------------------------------------------------------------------- diff --git a/src/_pytest/logging.py b/src/_pytest/logging.py index 7e906deab..22db44301 100644 --- a/src/_pytest/logging.py +++ b/src/_pytest/logging.py @@ -15,7 +15,6 @@ from _pytest.compat import dummy_context_manager from _pytest.config import create_terminal_writer from _pytest.pathlib import Path - DEFAULT_LOG_FORMAT = "%(filename)-25s %(lineno)4d %(levelname)-8s %(message)s" DEFAULT_LOG_DATE_FORMAT = "%H:%M:%S" diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 6e639d872..d8478d4fc 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -24,7 +24,6 @@ from _pytest.deprecated import PYTEST_CONFIG_GLOBAL from _pytest.outcomes import exit from _pytest.runner import collect_one_node - # exitcodes for the command line EXIT_OK = 0 EXIT_TESTSFAILED = 1 diff --git a/src/_pytest/mark/structures.py b/src/_pytest/mark/structures.py index c8a25d156..d65d8b9d8 100644 --- a/src/_pytest/mark/structures.py +++ b/src/_pytest/mark/structures.py @@ -12,7 +12,6 @@ from ..compat import MappingMixin from ..compat import NOTSET from _pytest.outcomes import fail - EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark" diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 5ecdd6026..5dd00e74f 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -19,7 +19,6 @@ from six.moves import map from .compat import PY36 - if PY36: from pathlib import Path, PurePath else: diff --git a/testing/code/test_source.py b/testing/code/test_source.py index fc5eaed04..965838dae 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -16,7 +16,6 @@ import _pytest._code import pytest from _pytest._code import Source - astonly = pytest.mark.nothing failsonjython = pytest.mark.xfail("sys.platform.startswith('java')") diff --git a/testing/example_scripts/perf_examples/collect_stats/generate_folders.py b/testing/example_scripts/perf_examples/collect_stats/generate_folders.py index 94b3f013f..ff1eaf7d6 100644 --- a/testing/example_scripts/perf_examples/collect_stats/generate_folders.py +++ b/testing/example_scripts/perf_examples/collect_stats/generate_folders.py @@ -1,7 +1,6 @@ import argparse import pathlib - HERE = pathlib.Path(__file__).parent TEST_CONTENT = (HERE / "template_test.py").read_bytes() diff --git a/testing/test_resultlog.py b/testing/test_resultlog.py index 2c5175246..aba5713af 100644 --- a/testing/test_resultlog.py +++ b/testing/test_resultlog.py @@ -12,7 +12,6 @@ from _pytest.resultlog import pytest_configure from _pytest.resultlog import pytest_unconfigure from _pytest.resultlog import ResultLog - pytestmark = pytest.mark.filterwarnings("ignore:--result-log is deprecated") diff --git a/testing/test_warnings.py b/testing/test_warnings.py index 984aae027..1b4231edc 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -8,7 +8,6 @@ import six import pytest - WARNINGS_SUMMARY_HEADER = "warnings summary" From f0d7773ffaf3ac64d863ae5643449c4408ceb460 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 27 Feb 2019 13:10:42 +0100 Subject: [PATCH 29/40] tox: split default posargs in multiple env vars This is required for combining. Use it for new lsof facor also. --- tox.ini | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 380dd2539..96c2aec0d 100644 --- a/tox.ini +++ b/tox.ini @@ -21,12 +21,12 @@ envlist = [testenv] commands = - {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {posargs:{env:_PYTEST_TOX_DEFAULT_POSARGS}} + {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {posargs:{env:_PYTEST_TOX_DEFAULT_POSARGS:}} coverage: coverage combine coverage: coverage report passenv = USER USERNAME COVERAGE_* TRAVIS PYTEST_ADDOPTS setenv = - _PYTEST_TOX_DEFAULT_POSARGS=--lsof + _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_PEXPECT:} {env:_PYTEST_TOX_POSARGS_XDIST:} # Configuration to run with coverage similar to Travis/Appveyor, e.g. # "tox -e py37-coverage". @@ -37,10 +37,12 @@ setenv = nobyte: PYTHONDONTWRITEBYTECODE=1 - pexpect: _PYTEST_TOX_PLATFORM=linux|darwin - pexpect: _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_DEFAULT_POSARGS:testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py} + lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof - xdist: _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_DEFAULT_POSARGS:-n auto} + pexpect: _PYTEST_TOX_PLATFORM=linux|darwin + pexpect: _PYTEST_TOX_POSARGS_PEXPECT=testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py + + xdist: _PYTEST_TOX_POSARGS_XDIST=-n auto extras = testing deps = numpy: numpy From f8160f7bc500451fbfc0000a544199481774fa78 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 16:50:31 +0100 Subject: [PATCH 30/40] ci: Travis: run py37-lsof-numpy-xdist with coverage --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fba5eaff7..58c7e6751 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,7 @@ jobs: - stage: baseline env: TOXENV=py27-pexpect,py27-trial - - env: TOXENV=py37-numpy-xdist + - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 - env: TOXENV=linting,docs,doctesting python: '3.7' From a96907a9db8ee18b94ab1ea78794c016b611a237 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 17:59:04 +0100 Subject: [PATCH 31/40] .coveragerc: use globs with includes Apparently this caused missing coverage with pdb/pexpect tests. --- .coveragerc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.coveragerc b/.coveragerc index 5942e71c6..def57d76a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,15 +1,15 @@ [run] include = - src/* + */src/* testing/* - .tox/*/lib/python*/site-packages/_pytest/* - .tox/*/lib/python*/site-packages/pytest.py - .tox\*\Lib\site-packages\_pytest\* - .tox\*\Lib\site-packages\pytest.py + */lib/python*/site-packages/_pytest/* + */lib/python*/site-packages/pytest.py + *\Lib\site-packages\_pytest\* + *\Lib\site-packages\pytest.py parallel = 1 branch = 1 [paths] source = src/ - .tox/*/lib/python*/site-packages/ - .tox\*\Lib\site-packages\ + */lib/python*/site-packages/ + *\Lib\site-packages\ From dac164cc9969c8484b21695bb70fc6735708ff26 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 1 Mar 2019 17:09:07 -0300 Subject: [PATCH 32/40] Add missing ref docs to tmp_path and tmp_path_factory --- doc/en/reference.rst | 26 ++++++++++++++++++++++++++ doc/en/tmpdir.rst | 5 +++++ 2 files changed, 31 insertions(+) diff --git a/doc/en/reference.rst b/doc/en/reference.rst index 1ed5d91b1..7ab734075 100644 --- a/doc/en/reference.rst +++ b/doc/en/reference.rst @@ -499,6 +499,32 @@ Each recorded warning is an instance of :class:`warnings.WarningMessage`. differently; see :ref:`ensuring_function_triggers`. +tmp_path +~~~~~~~~ + +**Tutorial**: :doc:`tmpdir` + +.. currentmodule:: _pytest.tmpdir + +.. autofunction:: tmp_path() + :no-auto-options: + + +tmp_path_factory +~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`tmp_path_factory example` + +.. _`tmp_path_factory factory api`: + +``tmp_path_factory`` instances have the following methods: + +.. currentmodule:: _pytest.tmpdir + +.. automethod:: TempPathFactory.mktemp +.. automethod:: TempPathFactory.getbasetemp + + tmpdir ~~~~~~ diff --git a/doc/en/tmpdir.rst b/doc/en/tmpdir.rst index df490d7fb..e2b8fc32b 100644 --- a/doc/en/tmpdir.rst +++ b/doc/en/tmpdir.rst @@ -66,6 +66,9 @@ Running this would result in a passed test except for the last test_tmp_path.py:13: AssertionError ========================= 1 failed in 0.12 seconds ========================= + +.. _`tmp_path_factory example`: + The ``tmp_path_factory`` fixture -------------------------------- @@ -77,6 +80,8 @@ to create arbitrary temporary directories from any other fixture or test. It is intended to replace ``tmpdir_factory``, and returns :class:`pathlib.Path` instances. +See :ref:`tmp_path_factory API ` for details. + The 'tmpdir' fixture -------------------- From 2970c1df24e8492a16fc4681cfb08aba4e7f7a36 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 3 Mar 2019 13:12:03 +0100 Subject: [PATCH 33/40] tox: generic pluggymaster factor --- tox.ini | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tox.ini b/tox.ini index 1fc20cb5f..3f954e1d4 100644 --- a/tox.ini +++ b/tox.ini @@ -42,11 +42,15 @@ setenv = pexpect: _PYTEST_TOX_PLATFORM=linux|darwin pexpect: _PYTEST_TOX_POSARGS_PEXPECT=testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py + # TODO: remove?! + pluggymaster: _PYTEST_SETUP_SKIP_PLUGGY_DEP=1 + xdist: _PYTEST_TOX_POSARGS_XDIST=-n auto extras = testing deps = numpy: numpy pexpect: pexpect + pluggymaster: git+https://github.com/pytest-dev/pluggy.git@master xdist: pytest-xdist>=1.13 {env:_PYTEST_TOX_EXTRA_DEP:} platform = {env:_PYTEST_TOX_PLATFORM:.*} @@ -78,18 +82,6 @@ commands = deps = {[testenv:py27-trial]deps} commands = {[testenv:py27-trial]commands} -[testenv:py27-pluggymaster] -setenv= - {[testenv]setenv} - _PYTEST_SETUP_SKIP_PLUGGY_DEP=1 -deps = - {[testenv]deps} - git+https://github.com/pytest-dev/pluggy.git@master - -[testenv:py37-pluggymaster] -setenv = {[testenv:py27-pluggymaster]setenv} -deps = {[testenv:py27-pluggymaster]deps} - [testenv:docs] basepython = python3 skipsdist = True From c4aa57bc4c3ffc71e9842f14251ee6c60b8953c3 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 3 Mar 2019 13:15:13 +0100 Subject: [PATCH 34/40] tox/setup.py: remove _PYTEST_SETUP_SKIP_PLUGGY_DEP=1 hack --- setup.py | 9 +-------- tox.ini | 3 --- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 89e48c6b5..cd4765440 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -import os - from setuptools import setup # TODO: if py gets upgrade to >=1.6, @@ -15,15 +13,10 @@ INSTALL_REQUIRES = [ 'funcsigs;python_version<"3.0"', 'pathlib2>=2.2.0;python_version<"3.6"', 'colorama;sys_platform=="win32"', + "pluggy>=0.7", ] -# if _PYTEST_SETUP_SKIP_PLUGGY_DEP is set, skip installing pluggy; -# used by tox.ini to test with pluggy master -if "_PYTEST_SETUP_SKIP_PLUGGY_DEP" not in os.environ: - INSTALL_REQUIRES.append("pluggy>=0.7") - - def main(): setup( use_scm_version={"write_to": "src/_pytest/_version.py"}, diff --git a/tox.ini b/tox.ini index 3f954e1d4..97cac6f6b 100644 --- a/tox.ini +++ b/tox.ini @@ -42,9 +42,6 @@ setenv = pexpect: _PYTEST_TOX_PLATFORM=linux|darwin pexpect: _PYTEST_TOX_POSARGS_PEXPECT=testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py - # TODO: remove?! - pluggymaster: _PYTEST_SETUP_SKIP_PLUGGY_DEP=1 - xdist: _PYTEST_TOX_POSARGS_XDIST=-n auto extras = testing deps = From 83558a0ba3a737752deb7d0b1c78a0ef0ec0843c Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 3 Mar 2019 14:06:55 +0100 Subject: [PATCH 35/40] tests: make test_crash_on_closing_tmpfile_py27 more reliable It fails reliable for me now without the fix from 9517c3a2a. Ref: #2370 --- testing/test_capture.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/testing/test_capture.py b/testing/test_capture.py index 546738f28..91cf8d8cf 100644 --- a/testing/test_capture.py +++ b/testing/test_capture.py @@ -1403,28 +1403,36 @@ def test_dontreadfrominput_has_encoding(testdir): def test_crash_on_closing_tmpfile_py27(testdir): - testdir.makepyfile( + p = testdir.makepyfile( """ from __future__ import print_function - import time import threading import sys + printing = threading.Event() + def spam(): f = sys.stderr - while True: - print('.', end='', file=f) + print('SPAMBEFORE', end='', file=f) + printing.set() - def test_silly(): + while True: + try: + f.flush() + except (OSError, ValueError): + break + + def test_spam_in_thread(): t = threading.Thread(target=spam) t.daemon = True t.start() - time.sleep(0.5) + printing.wait() """ ) - result = testdir.runpytest_subprocess() + result = testdir.runpytest_subprocess(str(p)) assert result.ret == 0 + assert result.stderr.str() == "" assert "IOError" not in result.stdout.str() From c6eb3413f302ff27f9d444171057ef021ae55bde Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Mon, 4 Mar 2019 17:35:14 +0100 Subject: [PATCH 36/40] .coveragerc: fix include for pypy PyPy uses "site-packages" directly. --- .coveragerc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.coveragerc b/.coveragerc index def57d76a..b5cfb9e7b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,6 +4,8 @@ include = testing/* */lib/python*/site-packages/_pytest/* */lib/python*/site-packages/pytest.py + */pypy*/site-packages/_pytest/* + */pypy*/site-packages/pytest.py *\Lib\site-packages\_pytest\* *\Lib\site-packages\pytest.py parallel = 1 @@ -12,4 +14,5 @@ branch = 1 [paths] source = src/ */lib/python*/site-packages/ + */pypy*/site-packages/ *\Lib\site-packages\ From fa94e3c1b2a62f8eb39f37cdb6737b88391221f2 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sat, 2 Mar 2019 00:50:31 +0100 Subject: [PATCH 37/40] ci: Azure: fix coverage generation with PyPy While this is not used currently, it is still good for having it covered (it failed when running coverage with all jobs unconditionally). --- azure-pipelines.yml | 5 +++++ scripts/upload-coverage.bat | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b648cd3d9..68f10d33f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,6 +37,10 @@ jobs: python.version: 'pypy' tox.env: 'pypy' python.exe: 'pypy' + pypy3: + python.version: 'pypy3' + tox.env: 'pypy3' + python.exe: 'pypy3' py34: python.version: '3.4' tox.env: 'py34' @@ -115,5 +119,6 @@ jobs: displayName: 'Upload coverage' condition: eq(variables['PYTEST_COVERAGE'], '1') env: + PYTHON: $(python.exe) CODECOV_TOKEN: $(CODECOV_TOKEN) PYTEST_CODECOV_NAME: $(tox.env) diff --git a/scripts/upload-coverage.bat b/scripts/upload-coverage.bat index 6a4773e18..b21e0126e 100644 --- a/scripts/upload-coverage.bat +++ b/scripts/upload-coverage.bat @@ -6,11 +6,11 @@ if "%PYTEST_COVERAGE%" == "1" ( ) else ( echo CODECOV_TOKEN NOT defined ) - python -m pip install codecov - coverage combine - coverage xml - coverage report -m - scripts\retry codecov --required -X gcov pycov search -f coverage.xml --name %PYTEST_CODECOV_NAME% + %PYTHON% -m pip install codecov + %PYTHON% -m coverage combine + %PYTHON% -m coverage xml + %PYTHON% -m coverage report -m + scripts\retry %PYTHON% -m codecov --required -X gcov pycov search -f coverage.xml --name %PYTEST_CODECOV_NAME% ) else ( echo Skipping coverage upload, PYTEST_COVERAGE=%PYTEST_COVERAGE% ) From 2e871f35f3c82f79b8cb94fef29273d4d0706f36 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Mon, 4 Mar 2019 17:51:25 +0100 Subject: [PATCH 38/40] ci: Azure: clarify name for coverage job --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 68f10d33f..b2ea65245 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -116,7 +116,7 @@ jobs: condition: succeededOrFailed() - script: call scripts\upload-coverage.bat - displayName: 'Upload coverage' + displayName: 'Report and upload coverage' condition: eq(variables['PYTEST_COVERAGE'], '1') env: PYTHON: $(python.exe) From 07fa69335cabed36dff7d9074573abf35fc9f8f0 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Mon, 4 Mar 2019 18:16:20 +0100 Subject: [PATCH 39/40] ci: Azure: remove pypy3 It was enabled accidentally. --- azure-pipelines.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b2ea65245..bec7a6d74 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,10 +37,11 @@ jobs: python.version: 'pypy' tox.env: 'pypy' python.exe: 'pypy' - pypy3: - python.version: 'pypy3' - tox.env: 'pypy3' - python.exe: 'pypy3' + # NOTE: pypy3 fails to install pip currently due to an interal error. + # pypy3: + # python.version: 'pypy3' + # tox.env: 'pypy3' + # python.exe: 'pypy3' py34: python.version: '3.4' tox.env: 'py34' From 8481e438bd3ec2a50a6818b2a660642dc411caaf Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 14:55:09 +0100 Subject: [PATCH 40/40] ci: revisit coverage reporting This brings coverage back that got missing with 9dcd6f2. Continuation of #4839 / #4846. --- .travis.yml | 101 +++++++++++++++++++++------------- azure-pipelines.yml | 42 ++++++++------ testing/test_assertrewrite.py | 3 +- 3 files changed, 89 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index 58c7e6751..03b502c80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,24 +10,38 @@ python: - '3.7' install: - python -m pip install --upgrade --pre tox -env: - matrix: - - TOXENV=py27 PYTEST_COVERAGE=1 - # Specialized factors for py27. - - TOXENV=py27-nobyte-numpy-xdist PYTEST_COVERAGE=1 - - TOXENV=py27-pluggymaster-xdist - # Specialized factors for py37. - - TOXENV=py37-pexpect,py37-trial - - TOXENV=py37-pluggymaster-xdist - - TOXENV=py37-freeze - -matrix: - allow_failures: - - python: '3.8-dev' - env: TOXENV=py38-xdist jobs: include: + # OSX tests - first (in test stage), since they are the slower ones. + - &test-macos + # NOTE: (tests with) pexpect appear to be buggy on Travis, + # at least with coverage. + # Log: https://travis-ci.org/pytest-dev/pytest/jobs/500358864 + os: osx + osx_image: xcode10.1 + language: generic + # Coverage for: + # - py2 with symlink in test_cmdline_python_package_symlink. + env: TOXENV=py27-xdist PYTEST_COVERAGE=1 + before_install: + - python -V + - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 + - <<: *test-macos + env: TOXENV=py37-xdist + before_install: + - which python3 + - python3 -V + - ln -sfn "$(which python3)" /usr/local/bin/python + - python -V + - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 + + # Full run of latest (major) supported versions, without xdist. + - env: TOXENV=py27 + python: '2.7' + - env: TOXENV=py37 + python: '3.7' + # Coverage tracking is slow with pypy, skip it. - env: TOXENV=pypy-xdist python: 'pypy2.7-6.0' @@ -38,25 +52,25 @@ jobs: python: '3.4' - env: TOXENV=py35-xdist python: '3.5' - - env: TOXENV=py36-xdist - python: '3.6' - - env: TOXENV=py37 - - &test-macos - os: osx - osx_image: xcode10.1 - language: generic - before_install: - - python -V - - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 - env: TOXENV=py27-xdist - - <<: *test-macos - env: TOXENV=py37-xdist - before_install: - - which python3 - - python3 -V - - ln -sfn "$(which python3)" /usr/local/bin/python - - python -V - - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 + + # Coverage for: + # - pytester's LsofFdLeakChecker + # - TestArgComplete (linux only) + # - numpy + - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 + + # Specialized factors for py27. + - env: TOXENV=py27-nobyte-numpy-xdist + python: '2.7' + - env: TOXENV=py27-pluggymaster-xdist + python: '2.7' + + # Specialized factors for py37. + # Coverage for: + # - test_sys_breakpoint_interception (via pexpect). + - env: TOXENV=py37-pexpect,py37-trial PYTEST_COVERAGE=1 + - env: TOXENV=py37-pluggymaster-xdist + - env: TOXENV=py37-freeze # Jobs only run via Travis cron jobs (currently daily). - env: TOXENV=py38-xdist @@ -64,10 +78,14 @@ jobs: if: type = cron - stage: baseline - env: TOXENV=py27-pexpect,py27-trial - - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 - - env: TOXENV=linting,docs,doctesting - python: '3.7' + # Coverage for: + # - _pytest.unittest._handle_skip (via pexpect). + env: TOXENV=py27-pexpect,py27-trial PYTEST_COVERAGE=1 + python: '2.7' + # Use py36 here for faster baseline. + - env: TOXENV=py36-xdist + python: '3.6' + - env: TOXENV=linting,docs,doctesting PYTEST_COVERAGE=1 - stage: deploy python: '3.6' @@ -84,6 +102,11 @@ jobs: tags: true repo: pytest-dev/pytest +matrix: + allow_failures: + - python: '3.8-dev' + env: TOXENV=py38-xdist + before_script: - | # Do not (re-)upload coverage with cron runs. @@ -109,7 +132,7 @@ after_success: coverage combine coverage xml coverage report -m - bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV + bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV-$TRAVIS_OS_NAME fi notifications: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bec7a6d74..712106c94 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,19 +20,28 @@ jobs: py27: python.version: '2.7' tox.env: 'py27' - PYTEST_COVERAGE: '1' - py27-nobyte-numpy-xdist: + py27-nobyte-lsof-numpy: python.version: '2.7' - tox.env: 'py27-nobyte-numpy-xdist' + tox.env: 'py27-lsof-nobyte-numpy' + # Coverage for: + # - test_supports_breakpoint_module_global + # - test_terminal_reporter_writer_attr (without xdist) + # - "if write" branch in _pytest.assertion.rewrite + # - numpy + # - pytester's LsofFdLeakChecker (being skipped) PYTEST_COVERAGE: '1' py27-trial: python.version: '2.7' tox.env: 'py27-trial' python.needs_vc: True - PYTEST_COVERAGE: '1' py27-pluggymaster-xdist: python.version: '2.7' tox.env: 'py27-pluggymaster-xdist' + # Coverage for: + # - except-IOError in _attempt_to_close_capture_file for py2. + # Also seen with py27-nobyte (using xdist), and py27-xdist. + # But no exception with py27-pexpect,py27-trial,py27-numpy. + PYTEST_COVERAGE: '1' pypy: python.version: 'pypy' tox.env: 'pypy' @@ -42,33 +51,34 @@ jobs: # python.version: 'pypy3' # tox.env: 'pypy3' # python.exe: 'pypy3' - py34: + py34-xdist: python.version: '3.4' - tox.env: 'py34' + tox.env: 'py34-xdist' + # Coverage for: + # - _pytest.compat._bytes_to_ascii PYTEST_COVERAGE: '1' - py35: + py35-xdist: python.version: '3.5' - tox.env: 'py35' + tox.env: 'py35-xdist' + # Coverage for: + # - test_supports_breakpoint_module_global PYTEST_COVERAGE: '1' - py36: + py36-xdist: python.version: '3.6' - tox.env: 'py36' - PYTEST_COVERAGE: '1' + tox.env: 'py36-xdist' py37: python.version: '3.7' tox.env: 'py37' + # Coverage for: + # - _py36_windowsconsoleio_workaround (with py36+) + # - test_request_garbage (no xdist) PYTEST_COVERAGE: '1' py37-linting/docs/doctesting: python.version: '3.7' tox.env: 'linting,docs,doctesting' - py37-xdist: - python.version: '3.7' - tox.env: 'py37-xdist' - PYTEST_COVERAGE: '1' py37-trial/numpy: python.version: '3.7' tox.env: 'py37-trial,py37-numpy' - PYTEST_COVERAGE: '1' py37-pluggymaster-xdist: python.version: '3.7' tox.env: 'py37-pluggymaster-xdist' diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index bfb6acfab..098e106f9 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -1313,8 +1313,7 @@ class TestEarlyRewriteBailout(object): # always (previously triggered via xdist only). # Ref: https://github.com/pytest-dev/py/pull/207 monkeypatch.setattr(sys, "path", [""] + sys.path) - if "pathlib" in sys.modules: - del sys.modules["pathlib"] + monkeypatch.delitem(sys.modules, "pathlib", raising=False) testdir.makepyfile( **{