From 8481e438bd3ec2a50a6818b2a660642dc411caaf Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 1 Mar 2019 14:55:09 +0100 Subject: [PATCH] ci: revisit coverage reporting This brings coverage back that got missing with 9dcd6f2. Continuation of #4839 / #4846. --- .travis.yml | 101 +++++++++++++++++++++------------- azure-pipelines.yml | 42 ++++++++------ testing/test_assertrewrite.py | 3 +- 3 files changed, 89 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index 58c7e6751..03b502c80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,24 +10,38 @@ python: - '3.7' install: - python -m pip install --upgrade --pre tox -env: - matrix: - - TOXENV=py27 PYTEST_COVERAGE=1 - # Specialized factors for py27. - - TOXENV=py27-nobyte-numpy-xdist PYTEST_COVERAGE=1 - - TOXENV=py27-pluggymaster-xdist - # Specialized factors for py37. - - TOXENV=py37-pexpect,py37-trial - - TOXENV=py37-pluggymaster-xdist - - TOXENV=py37-freeze - -matrix: - allow_failures: - - python: '3.8-dev' - env: TOXENV=py38-xdist jobs: include: + # OSX tests - first (in test stage), since they are the slower ones. + - &test-macos + # NOTE: (tests with) pexpect appear to be buggy on Travis, + # at least with coverage. + # Log: https://travis-ci.org/pytest-dev/pytest/jobs/500358864 + os: osx + osx_image: xcode10.1 + language: generic + # Coverage for: + # - py2 with symlink in test_cmdline_python_package_symlink. + env: TOXENV=py27-xdist PYTEST_COVERAGE=1 + before_install: + - python -V + - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 + - <<: *test-macos + env: TOXENV=py37-xdist + before_install: + - which python3 + - python3 -V + - ln -sfn "$(which python3)" /usr/local/bin/python + - python -V + - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 + + # Full run of latest (major) supported versions, without xdist. + - env: TOXENV=py27 + python: '2.7' + - env: TOXENV=py37 + python: '3.7' + # Coverage tracking is slow with pypy, skip it. - env: TOXENV=pypy-xdist python: 'pypy2.7-6.0' @@ -38,25 +52,25 @@ jobs: python: '3.4' - env: TOXENV=py35-xdist python: '3.5' - - env: TOXENV=py36-xdist - python: '3.6' - - env: TOXENV=py37 - - &test-macos - os: osx - osx_image: xcode10.1 - language: generic - before_install: - - python -V - - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 - env: TOXENV=py27-xdist - - <<: *test-macos - env: TOXENV=py37-xdist - before_install: - - which python3 - - python3 -V - - ln -sfn "$(which python3)" /usr/local/bin/python - - python -V - - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 + + # Coverage for: + # - pytester's LsofFdLeakChecker + # - TestArgComplete (linux only) + # - numpy + - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 + + # Specialized factors for py27. + - env: TOXENV=py27-nobyte-numpy-xdist + python: '2.7' + - env: TOXENV=py27-pluggymaster-xdist + python: '2.7' + + # Specialized factors for py37. + # Coverage for: + # - test_sys_breakpoint_interception (via pexpect). + - env: TOXENV=py37-pexpect,py37-trial PYTEST_COVERAGE=1 + - env: TOXENV=py37-pluggymaster-xdist + - env: TOXENV=py37-freeze # Jobs only run via Travis cron jobs (currently daily). - env: TOXENV=py38-xdist @@ -64,10 +78,14 @@ jobs: if: type = cron - stage: baseline - env: TOXENV=py27-pexpect,py27-trial - - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 - - env: TOXENV=linting,docs,doctesting - python: '3.7' + # Coverage for: + # - _pytest.unittest._handle_skip (via pexpect). + env: TOXENV=py27-pexpect,py27-trial PYTEST_COVERAGE=1 + python: '2.7' + # Use py36 here for faster baseline. + - env: TOXENV=py36-xdist + python: '3.6' + - env: TOXENV=linting,docs,doctesting PYTEST_COVERAGE=1 - stage: deploy python: '3.6' @@ -84,6 +102,11 @@ jobs: tags: true repo: pytest-dev/pytest +matrix: + allow_failures: + - python: '3.8-dev' + env: TOXENV=py38-xdist + before_script: - | # Do not (re-)upload coverage with cron runs. @@ -109,7 +132,7 @@ after_success: coverage combine coverage xml coverage report -m - bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV + bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV-$TRAVIS_OS_NAME fi notifications: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bec7a6d74..712106c94 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,19 +20,28 @@ jobs: py27: python.version: '2.7' tox.env: 'py27' - PYTEST_COVERAGE: '1' - py27-nobyte-numpy-xdist: + py27-nobyte-lsof-numpy: python.version: '2.7' - tox.env: 'py27-nobyte-numpy-xdist' + tox.env: 'py27-lsof-nobyte-numpy' + # Coverage for: + # - test_supports_breakpoint_module_global + # - test_terminal_reporter_writer_attr (without xdist) + # - "if write" branch in _pytest.assertion.rewrite + # - numpy + # - pytester's LsofFdLeakChecker (being skipped) PYTEST_COVERAGE: '1' py27-trial: python.version: '2.7' tox.env: 'py27-trial' python.needs_vc: True - PYTEST_COVERAGE: '1' py27-pluggymaster-xdist: python.version: '2.7' tox.env: 'py27-pluggymaster-xdist' + # Coverage for: + # - except-IOError in _attempt_to_close_capture_file for py2. + # Also seen with py27-nobyte (using xdist), and py27-xdist. + # But no exception with py27-pexpect,py27-trial,py27-numpy. + PYTEST_COVERAGE: '1' pypy: python.version: 'pypy' tox.env: 'pypy' @@ -42,33 +51,34 @@ jobs: # python.version: 'pypy3' # tox.env: 'pypy3' # python.exe: 'pypy3' - py34: + py34-xdist: python.version: '3.4' - tox.env: 'py34' + tox.env: 'py34-xdist' + # Coverage for: + # - _pytest.compat._bytes_to_ascii PYTEST_COVERAGE: '1' - py35: + py35-xdist: python.version: '3.5' - tox.env: 'py35' + tox.env: 'py35-xdist' + # Coverage for: + # - test_supports_breakpoint_module_global PYTEST_COVERAGE: '1' - py36: + py36-xdist: python.version: '3.6' - tox.env: 'py36' - PYTEST_COVERAGE: '1' + tox.env: 'py36-xdist' py37: python.version: '3.7' tox.env: 'py37' + # Coverage for: + # - _py36_windowsconsoleio_workaround (with py36+) + # - test_request_garbage (no xdist) PYTEST_COVERAGE: '1' py37-linting/docs/doctesting: python.version: '3.7' tox.env: 'linting,docs,doctesting' - py37-xdist: - python.version: '3.7' - tox.env: 'py37-xdist' - PYTEST_COVERAGE: '1' py37-trial/numpy: python.version: '3.7' tox.env: 'py37-trial,py37-numpy' - PYTEST_COVERAGE: '1' py37-pluggymaster-xdist: python.version: '3.7' tox.env: 'py37-pluggymaster-xdist' diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index bfb6acfab..098e106f9 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -1313,8 +1313,7 @@ class TestEarlyRewriteBailout(object): # always (previously triggered via xdist only). # Ref: https://github.com/pytest-dev/py/pull/207 monkeypatch.setattr(sys, "path", [""] + sys.path) - if "pathlib" in sys.modules: - del sys.modules["pathlib"] + monkeypatch.delitem(sys.modules, "pathlib", raising=False) testdir.makepyfile( **{