Merge pull request #2800 from blueyed/coverage

Travis: report coverage with all builds

.coveragerc

@@ -1,4 +1,3 @@
 [run]
-omit =
-    # standlonetemplate is read dynamically and tested by test_genscript
-    *standalonetemplate.py
+source = _pytest,testing
+parallel = 1

.travis.yml

@@ -12,8 +12,6 @@ install:
   - pip install --upgrade --pre tox
 env:
   matrix:
-    # coveralls is not listed in tox's envlist, but should run in travis
-    - TOXENV=coveralls
     # note: please use "tox --listenvs" to populate the build matrix below
     # please remove the linting env in all cases
     - TOXENV=py27-pexpect
@@ -28,15 +26,16 @@ env:
     - TOXENV=py36-pluggymaster
     - TOXENV=py27-nobyte
     - TOXENV=doctesting
-    - TOXENV=docs
+    - TOXENV=docs PYTEST_NO_COVERAGE=1
 
 jobs:
   include:
-    - env: TOXENV=pypy
+    # Coverage tracking is slow with pypy, skip it.
+    - env: TOXENV=pypy PYTEST_NO_COVERAGE=1
       python: 'pypy-5.4'
     - env: TOXENV=py35
       python: '3.5'
-    - env: TOXENV=py36-freeze
+    - env: TOXENV=py36-freeze PYTEST_NO_COVERAGE=1
       python: '3.6'
     - env: TOXENV=py37
       python: '3.7'
@@ -62,7 +61,7 @@ jobs:
       env: TOXENV=py27
     - env: TOXENV=py34
     - env: TOXENV=py36
-    - env: TOXENV=linting
+    - env: TOXENV=linting PYTEST_NO_COVERAGE=1
 
     - stage: deploy
       python: '3.6'
@@ -80,8 +79,32 @@ jobs:
           tags: true
           repo: pytest-dev/pytest
 
+before_script:
+  - |
+    if [[ "$PYTEST_NO_COVERAGE" != 1 ]]; then
+      export _PYTEST_TOX_COVERAGE_RUN="env COVERAGE_FILE=$PWD/.coverage COVERAGE_PROCESS_START=$PWD/.coveragerc coverage run --source {envsitepackagesdir}/_pytest/,$PWD/testing -m"
+      export _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess
+    fi
+
 script: tox --recreate
 
+after_success:
+  - |
+    if [[ "$PYTEST_NO_COVERAGE" != 1 ]]; then
+      set -e
+      pip install codecov
+      coverage combine
+      coverage xml
+      coverage report -m
+      codecov --required -X gcov pycov search -f coverage.xml --flags ${TOXENV//-/ }
+
+      # Coveralls does not support merged reports.
+      if [[ "$TOXENV" = py37 ]]; then
+        pip install coveralls
+        coveralls
+      fi
+    fi
+
 notifications:
   irc:
     channels:

tox.ini

@@ -17,13 +17,21 @@ envlist =
     docs
 
 [testenv]
-commands = pytest --lsof -ra {posargs:testing}
+commands =
+    {env:_PYTEST_TOX_COVERAGE_RUN:} pytest --lsof -ra {env:_PYTEST_TEST_OPTS:} {posargs:testing}
+    coverage: coverage report -m --skip-covered
+setenv =
+    coverage: _PYTEST_TOX_COVERAGE_RUN=coverage run -m
+    coverage: _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess
+    coverage: COVERAGE_FILE={toxinidir}/.coverage
+    coverage: COVERAGE_PROCESS_START={toxinidir}/.coveragerc
 passenv = USER USERNAME
 deps =
     hypothesis>=3.56
     nose
     mock
     requests
+    {env:_PYTEST_TOX_EXTRA_DEP:}
 
 [testenv:py27-subprocess]
 changedir = .
@@ -47,9 +55,10 @@ deps =
     mock
     nose
     hypothesis>=3.56
+    {env:_PYTEST_TOX_EXTRA_DEP:}
 changedir=testing
 commands =
-    pytest -n8 -ra {posargs:.}
+    {env:_PYTEST_TOX_COVERAGE_RUN:} pytest -n8 -ra {posargs:.}
 
 [testenv:py36-xdist]
 deps = {[testenv:py27-xdist]deps}
@@ -58,9 +67,11 @@ commands = {[testenv:py27-xdist]commands}
 [testenv:py27-pexpect]
 changedir = testing
 platform = linux|darwin
-deps = pexpect
+deps =
+    pexpect
+    {env:_PYTEST_TOX_EXTRA_DEP:}
 commands =
-    pytest -ra test_pdb.py test_terminal.py test_unittest.py
+    {env:_PYTEST_TOX_COVERAGE_RUN:} pytest -ra test_pdb.py test_terminal.py test_unittest.py
 
 [testenv:py36-pexpect]
 changedir = {[testenv:py27-pexpect]changedir}
@@ -73,26 +84,32 @@ deps =
     pytest-xdist>=1.13
     hypothesis>=3.56
     mock
+    {env:_PYTEST_TOX_EXTRA_DEP:}
 distribute = true
 changedir=testing
 setenv =
+    {[testenv]setenv}
     PYTHONDONTWRITEBYTECODE=1
 commands =
-    pytest -n3 -ra {posargs:.}
+    {env:_PYTEST_TOX_COVERAGE_RUN:} pytest -n3 -ra {posargs:.}
 
 [testenv:py27-trial]
-deps = twisted
+deps =
+    twisted
+    {env:_PYTEST_TOX_EXTRA_DEP:}
 commands =
-    pytest -ra {posargs:testing/test_unittest.py}
+    {env:_PYTEST_TOX_COVERAGE_RUN:} pytest -ra {posargs:testing/test_unittest.py}
 
 [testenv:py36-trial]
 deps = {[testenv:py27-trial]deps}
 commands = {[testenv:py27-trial]commands}
 
 [testenv:py27-numpy]
-deps = numpy
+deps =
+    numpy
+    {env:_PYTEST_TOX_EXTRA_DEP:}
 commands=
-  pytest -ra {posargs:testing/python/approx.py}
+  {env:_PYTEST_TOX_COVERAGE_RUN:} pytest -ra {posargs:testing/python/approx.py}
 
 [testenv:py36-numpy]
 deps = {[testenv:py27-numpy]deps}
@@ -100,6 +117,7 @@ commands = {[testenv:py27-numpy]commands}
 
 [testenv:py27-pluggymaster]
 setenv=
+    {[testenv]setenv}
     _PYTEST_SETUP_SKIP_PLUGGY_DEP=1
 deps =
     {[testenv]deps}
@@ -123,15 +141,13 @@ commands =
 
 [testenv:doctesting]
 basepython = python
-usedevelop = True
 skipsdist = True
-# ensure the given pyargs can't mean anything else
-changedir = doc/
 deps =
     PyYAML
+    {env:_PYTEST_TOX_EXTRA_DEP:}
 commands =
-    pytest -ra en
-    pytest --doctest-modules --pyargs _pytest
+    {env:_PYTEST_TOX_COVERAGE_RUN:} pytest -ra doc/en
+    {env:_PYTEST_TOX_COVERAGE_RUN:} pytest --doctest-modules --pyargs _pytest
 
 [testenv:regen]
 changedir = doc/en
@@ -155,7 +171,8 @@ commands =
 
 [testenv:py36-freeze]
 changedir = testing/freeze
-deps = pyinstaller
+deps =
+    pyinstaller
 commands =
     {envpython} create_executable.py
     {envpython} tox_run.py
@@ -170,7 +187,7 @@ deps =
     coveralls
     codecov
 commands =
-    coverage run --source=_pytest -m pytest testing
+    coverage run -m pytest testing
     coverage report -m
     coveralls
     codecov