From a3bc6df95061f20630af33c3b12f6a48a6d10acb Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 7 Jan 2020 20:03:38 -0300 Subject: [PATCH 1/2] Implement code coverage in GitHub actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This overwrites the `codecov.yml` file in the root of the repository with `codecov-upstream.yml` file (which contains the code-cov token)ยด, so PRs and branches on the repository can upload coverage. Suggestion from here: https://github.com/pytest-dev/pytest/pull/6421#issuecomment-571934112 Security concerns: the token might be misused, but only to upload bogus coverage to `codecov.io`, so the team believe this is harmless. If we decide to fallback from this decision , we just need to revoke the token. Related to #6369 --- .github/codecov-upstream.yml | 13 +++++++++++ .github/workflows/main.yml | 45 +++++++++++++++++++++++++++++++----- codecov.yml | 2 ++ tox.ini | 2 +- 4 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 .github/codecov-upstream.yml diff --git a/.github/codecov-upstream.yml b/.github/codecov-upstream.yml new file mode 100644 index 000000000..1256a4389 --- /dev/null +++ b/.github/codecov-upstream.yml @@ -0,0 +1,13 @@ +# this file replaces /codecov.yml when building on the main repository and PRs +coverage: + status: + project: true + patch: true + changes: true + +comment: off + +codecov: + # token from: https://codecov.io/gh/pytest-dev/pytest/settings + # use same URL to regenerate it if needed + token: "1eca3b1f-31a2-4fb8-a8c3-138b441b50a7" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3a2bccfc..5d7493e72 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,8 +1,7 @@ -# evaluating GitHub actions for CI, disconsider failures when evaluating PRs +# evaluating GitHub actions for CI, disregard failures when evaluating PRs # # this is still missing: # - deploy -# - coverage # - upload github notes # name: main @@ -17,7 +16,6 @@ on: jobs: build: - runs-on: ${{ matrix.os }} strategy: @@ -86,6 +84,8 @@ jobs: python: "3.7" os: ubuntu-latest tox_env: "py37-freeze" + # coverage does not apply for freeze test, skip it + skip_coverage: true - name: "ubuntu-py38" python: "3.8" os: ubuntu-latest @@ -94,6 +94,8 @@ jobs: python: "pypy3" os: ubuntu-latest tox_env: "pypy3-xdist" + # coverage too slow with pypy3, skip it + skip_coverage: true - name: "macos-py37" python: "3.7" @@ -118,6 +120,37 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install tox - - name: Test - run: tox -e ${{ matrix.tox_env }} + pip install tox coverage + + - name: Test without coverage + if: "matrix.skip_coverage" + run: "tox -e ${{ matrix.tox_env }}" + + - name: Test with coverage + if: "! matrix.skip_coverage" + env: + _PYTEST_TOX_COVERAGE_RUN: "coverage run -m" + COVERAGE_PROCESS_START: ".coveragerc" + _PYTEST_TOX_EXTRA_DEP: "coverage-enable-subprocess" + run: "tox -e ${{ matrix.tox_env }}" + + - name: Prepare coverage token + if: success() && !matrix.skip_coverage && ( github.repository == 'pytest-dev/pytest' || github.event_name == 'pull_request' ) + run: | + cp .github/codecov-upstream.yml codecov.yml + + - name: Combine coverage + if: success() && !matrix.skip_coverage + run: | + python -m coverage combine + python -m coverage xml + + - name: Codecov upload + if: success() && !matrix.skip_coverage + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.codecov }} + file: ./coverage.xml + flags: ${{ runner.os }} + fail_ci_if_error: false + name: ${{ matrix.name }} diff --git a/codecov.yml b/codecov.yml index a0a308588..44fd14710 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,5 @@ +# note: `.github/codecov-upstream.yml` is basically a copy of this file, please propagate +# changes as needed coverage: status: project: true diff --git a/tox.ini b/tox.ini index d33a62224..8fc69e89d 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ passenv = USER USERNAME COVERAGE_* TRAVIS PYTEST_ADDOPTS TERM setenv = _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} - # Configuration to run with coverage similar to Travis/Appveyor, e.g. + # Configuration to run with coverage similar to CI, e.g. # "tox -e py37-coverage". coverage: _PYTEST_TOX_COVERAGE_RUN=coverage run -m coverage: _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess From d291905825d2ff3e5bbbaa245d8fecd24c980db9 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 14 Jan 2020 07:47:21 -0300 Subject: [PATCH 2/2] Append token to codecov.yml instead of duplicating the file --- .github/codecov-upstream.yml | 13 ------------ .github/workflows/main.yml | 2 +- codecov.yml | 2 -- scripts/append_codecov_token.py | 36 +++++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 16 deletions(-) delete mode 100644 .github/codecov-upstream.yml create mode 100644 scripts/append_codecov_token.py diff --git a/.github/codecov-upstream.yml b/.github/codecov-upstream.yml deleted file mode 100644 index 1256a4389..000000000 --- a/.github/codecov-upstream.yml +++ /dev/null @@ -1,13 +0,0 @@ -# this file replaces /codecov.yml when building on the main repository and PRs -coverage: - status: - project: true - patch: true - changes: true - -comment: off - -codecov: - # token from: https://codecov.io/gh/pytest-dev/pytest/settings - # use same URL to regenerate it if needed - token: "1eca3b1f-31a2-4fb8-a8c3-138b441b50a7" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d7493e72..b1bae1c18 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -137,7 +137,7 @@ jobs: - name: Prepare coverage token if: success() && !matrix.skip_coverage && ( github.repository == 'pytest-dev/pytest' || github.event_name == 'pull_request' ) run: | - cp .github/codecov-upstream.yml codecov.yml + python scripts/append_codecov_token.py - name: Combine coverage if: success() && !matrix.skip_coverage diff --git a/codecov.yml b/codecov.yml index 44fd14710..a0a308588 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,5 +1,3 @@ -# note: `.github/codecov-upstream.yml` is basically a copy of this file, please propagate -# changes as needed coverage: status: project: true diff --git a/scripts/append_codecov_token.py b/scripts/append_codecov_token.py new file mode 100644 index 000000000..8eecb0fa5 --- /dev/null +++ b/scripts/append_codecov_token.py @@ -0,0 +1,36 @@ +""" +Appends the codecov token to the 'codecov.yml' file at the root of the repository. + +This is done by CI during PRs and builds on the pytest-dev repository so we can upload coverage, at least +until codecov grows some native integration like it has with Travis and AppVeyor. + +See discussion in https://github.com/pytest-dev/pytest/pull/6441 for more information. +""" +import os.path +from textwrap import dedent + + +def main(): + this_dir = os.path.dirname(__file__) + cov_file = os.path.join(this_dir, "..", "codecov.yml") + + assert os.path.isfile(cov_file), "{cov_file} does not exist".format( + cov_file=cov_file + ) + + with open(cov_file, "a") as f: + # token from: https://codecov.io/gh/pytest-dev/pytest/settings + # use same URL to regenerate it if needed + text = dedent( + """ + codecov: + token: "1eca3b1f-31a2-4fb8-a8c3-138b441b50a7" + """ + ) + f.write(text) + + print("Token updated:", cov_file) + + +if __name__ == "__main__": + main()