From 5aa289e47865ab9c1ca4d1efbffa1fd1024b81f7 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sun, 31 Dec 2023 10:09:33 -0300 Subject: [PATCH 1/2] Improve GitHub release workflow This changes the existing script to just generate the release notes and delegate the actual publishing to the `softprops/action-gh-release@v1` action. This allows us to delete the custom code, which failed recently in https://github.com/pytest-dev/pytest/actions/runs/7370258570/job/20056477756. --- .github/workflows/deploy.yml | 13 +++-- scripts/.gitignore | 1 + ...-notes.py => generate-gh-release-notes.py} | 47 ++++--------------- tox.ini | 11 ++--- 4 files changed, 23 insertions(+), 49 deletions(-) create mode 100644 scripts/.gitignore rename scripts/{publish-gh-release-notes.py => generate-gh-release-notes.py} (56%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ca65662c1..fed725f0e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -82,9 +82,14 @@ jobs: python -m pip install --upgrade pip pip install --upgrade tox - - name: Publish GitHub release notes - env: - GH_RELEASE_NOTES_TOKEN: ${{ github.token }} + - name: Generate release notes run: | sudo apt-get install pandoc - tox -e publish-gh-release-notes + tox -e generate-gh-release-notes -- ${{ github.event.inputs.version }} scripts/latest-release-notes.md + + - name: Publish GitHub Release + uses: softprops/action-gh-release@v1 + with: + body_path: scripts/latest-release-notes.md + files: dist/* + tag_name: ${{ github.event.inputs.version }} diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 000000000..50a75b629 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +latest-release-notes.md diff --git a/scripts/publish-gh-release-notes.py b/scripts/generate-gh-release-notes.py similarity index 56% rename from scripts/publish-gh-release-notes.py rename to scripts/generate-gh-release-notes.py index 68cbd7adf..d30b8ef30 100644 --- a/scripts/publish-gh-release-notes.py +++ b/scripts/generate-gh-release-notes.py @@ -1,3 +1,4 @@ +# mypy:disallow-untyped-defs """ Script used to publish GitHub release notes extracted from CHANGELOG.rst. @@ -19,27 +20,18 @@ The script also requires ``pandoc`` to be previously installed in the system. Requires Python3.6+. """ -import os import re import sys from pathlib import Path -import github3 import pypandoc -def publish_github_release(slug, token, tag_name, body): - github = github3.login(token=token) - owner, repo = slug.split("/") - repo = github.repository(owner, repo) - return repo.create_release(tag_name=tag_name, body=body) - - def parse_changelog(tag_name): p = Path(__file__).parent.parent / "doc/en/changelog.rst" changelog_lines = p.read_text(encoding="UTF-8").splitlines() - title_regex = re.compile(r"pytest (\d\.\d+\.\d+) \(\d{4}-\d{2}-\d{2}\)") + title_regex = re.compile(r"pytest (\d\.\d+\.\d+\w*) \(\d{4}-\d{2}-\d{2}\)") consuming_version = False version_lines = [] for line in changelog_lines: @@ -64,36 +56,17 @@ def convert_rst_to_md(text): def main(argv): - if len(argv) > 1: - tag_name = argv[1] - else: - tag_name = os.environ.get("GITHUB_REF") - if not tag_name: - print("tag_name not given and $GITHUB_REF not set", file=sys.stderr) - return 1 - if tag_name.startswith("refs/tags/"): - tag_name = tag_name[len("refs/tags/") :] + if len(argv) != 3: + print("Usage: generate-gh-release-notes VERSION FILE") + return 2 - token = os.environ.get("GH_RELEASE_NOTES_TOKEN") - if not token: - print("GH_RELEASE_NOTES_TOKEN not set", file=sys.stderr) - return 1 - - slug = os.environ.get("GITHUB_REPOSITORY") - if not slug: - print("GITHUB_REPOSITORY not set", file=sys.stderr) - return 1 - - rst_body = parse_changelog(tag_name) + version, filename = argv[1:3] + print(f"Generating GitHub release notes for version {version}") + rst_body = parse_changelog(version) md_body = convert_rst_to_md(rst_body) - if not publish_github_release(slug, token, tag_name, md_body): - print("Could not publish release notes:", file=sys.stderr) - print(md_body, file=sys.stderr) - return 5 - + Path(filename).write_text(md_body, encoding="UTF-8") print() - print(f"Release notes for {tag_name} published successfully:") - print(f"https://github.com/{slug}/releases/tag/{tag_name}") + print(f"Done: {filename}") print() return 0 diff --git a/tox.ini b/tox.ini index c52a43fd7..e92f6c98b 100644 --- a/tox.ini +++ b/tox.ini @@ -177,18 +177,13 @@ passenv = {[testenv:release]passenv} deps = {[testenv:release]deps} commands = python scripts/prepare-release-pr.py {posargs} -[testenv:publish-gh-release-notes] -description = create GitHub release after deployment +[testenv:generate-gh-release-notes] +description = generate release notes that can be published as GitHub Release basepython = python3 usedevelop = True -passenv = - GH_RELEASE_NOTES_TOKEN - GITHUB_REF - GITHUB_REPOSITORY deps = - github3.py pypandoc -commands = python scripts/publish-gh-release-notes.py {posargs} +commands = python scripts/generate-gh-release-notes.py {posargs} [flake8] max-line-length = 120 From 6321b74faefef7c5dc267862eccc6a56f9a0ec61 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sun, 31 Dec 2023 10:25:13 -0300 Subject: [PATCH 2/2] Enable type-checking in scripts/ --- .pre-commit-config.yaml | 3 ++- scripts/generate-gh-release-notes.py | 13 ++++++++----- scripts/prepare-release-pr.py | 1 + scripts/release.py | 25 ++++++++++++++----------- scripts/towncrier-draft-to-file.py | 7 ++++--- scripts/update-plugin-list.py | 27 +++++++++++++++++++++------ 6 files changed, 50 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c328ad48..ce1cce6e0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,7 +59,7 @@ repos: rev: v1.8.0 hooks: - id: mypy - files: ^(src/|testing/) + files: ^(src/|testing/|scripts/) args: [] additional_dependencies: - iniconfig>=1.1.0 @@ -67,6 +67,7 @@ repos: - packaging - tomli - types-pkg_resources + - types-tabulate # for mypy running on python>=3.11 since exceptiongroup is only a dependency # on <3.11 - exceptiongroup>=1.0.0rc8 diff --git a/scripts/generate-gh-release-notes.py b/scripts/generate-gh-release-notes.py index d30b8ef30..8eddb8055 100644 --- a/scripts/generate-gh-release-notes.py +++ b/scripts/generate-gh-release-notes.py @@ -1,4 +1,4 @@ -# mypy:disallow-untyped-defs +# mypy: disallow-untyped-defs """ Script used to publish GitHub release notes extracted from CHANGELOG.rst. @@ -23,11 +23,12 @@ Requires Python3.6+. import re import sys from pathlib import Path +from typing import Sequence import pypandoc -def parse_changelog(tag_name): +def parse_changelog(tag_name: str) -> str: p = Path(__file__).parent.parent / "doc/en/changelog.rst" changelog_lines = p.read_text(encoding="UTF-8").splitlines() @@ -49,13 +50,15 @@ def parse_changelog(tag_name): return "\n".join(version_lines) -def convert_rst_to_md(text): - return pypandoc.convert_text( +def convert_rst_to_md(text: str) -> str: + result = pypandoc.convert_text( text, "md", format="rst", extra_args=["--wrap=preserve"] ) + assert isinstance(result, str), repr(result) + return result -def main(argv): +def main(argv: Sequence[str]) -> int: if len(argv) != 3: print("Usage: generate-gh-release-notes VERSION FILE") return 2 diff --git a/scripts/prepare-release-pr.py b/scripts/prepare-release-pr.py index 8ffa66964..ce8242a74 100644 --- a/scripts/prepare-release-pr.py +++ b/scripts/prepare-release-pr.py @@ -1,3 +1,4 @@ +# mypy: disallow-untyped-defs """ This script is part of the pytest release process which is triggered manually in the Actions tab of the repository. diff --git a/scripts/release.py b/scripts/release.py index 19fef4284..66617feb5 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -1,3 +1,4 @@ +# mypy: disallow-untyped-defs """Invoke development tasks.""" import argparse import os @@ -10,15 +11,15 @@ from colorama import Fore from colorama import init -def announce(version, template_name, doc_version): +def announce(version: str, template_name: str, doc_version: str) -> None: """Generates a new release announcement entry in the docs.""" # Get our list of authors - stdout = check_output(["git", "describe", "--abbrev=0", "--tags"]) - stdout = stdout.decode("utf-8") + stdout = check_output(["git", "describe", "--abbrev=0", "--tags"], encoding="UTF-8") last_version = stdout.strip() - stdout = check_output(["git", "log", f"{last_version}..HEAD", "--format=%aN"]) - stdout = stdout.decode("utf-8") + stdout = check_output( + ["git", "log", f"{last_version}..HEAD", "--format=%aN"], encoding="UTF-8" + ) contributors = { name @@ -61,7 +62,7 @@ def announce(version, template_name, doc_version): check_call(["git", "add", str(target)]) -def regen(version): +def regen(version: str) -> None: """Call regendoc tool to update examples and pytest output in the docs.""" print(f"{Fore.CYAN}[generate.regen] {Fore.RESET}Updating docs") check_call( @@ -70,7 +71,7 @@ def regen(version): ) -def fix_formatting(): +def fix_formatting() -> None: """Runs pre-commit in all files to ensure they are formatted correctly""" print( f"{Fore.CYAN}[generate.fix linting] {Fore.RESET}Fixing formatting using pre-commit" @@ -78,13 +79,15 @@ def fix_formatting(): call(["pre-commit", "run", "--all-files"]) -def check_links(): +def check_links() -> None: """Runs sphinx-build to check links""" print(f"{Fore.CYAN}[generate.check_links] {Fore.RESET}Checking links") check_call(["tox", "-e", "docs-checklinks"]) -def pre_release(version, template_name, doc_version, *, skip_check_links): +def pre_release( + version: str, template_name: str, doc_version: str, *, skip_check_links: bool +) -> None: """Generates new docs, release announcements and creates a local tag.""" announce(version, template_name, doc_version) regen(version) @@ -102,12 +105,12 @@ def pre_release(version, template_name, doc_version, *, skip_check_links): print("Please push your branch and open a PR.") -def changelog(version, write_out=False): +def changelog(version: str, write_out: bool = False) -> None: addopts = [] if write_out else ["--draft"] check_call(["towncrier", "--yes", "--version", version] + addopts) -def main(): +def main() -> None: init(autoreset=True) parser = argparse.ArgumentParser() parser.add_argument("version", help="Release version") diff --git a/scripts/towncrier-draft-to-file.py b/scripts/towncrier-draft-to-file.py index 1f1068689..7b2748aa8 100644 --- a/scripts/towncrier-draft-to-file.py +++ b/scripts/towncrier-draft-to-file.py @@ -1,11 +1,12 @@ +# mypy: disallow-untyped-defs import sys from subprocess import call -def main(): +def main() -> int: """ - Platform agnostic wrapper script for towncrier. - Fixes the issue (#7251) where windows users are unable to natively run tox -e docs to build pytest docs. + Platform-agnostic wrapper script for towncrier. + Fixes the issue (#7251) where Windows users are unable to natively run tox -e docs to build pytest docs. """ with open( "doc/en/_changelog_towncrier_draft.rst", "w", encoding="utf-8" diff --git a/scripts/update-plugin-list.py b/scripts/update-plugin-list.py index 46f22ad1e..0f811b778 100644 --- a/scripts/update-plugin-list.py +++ b/scripts/update-plugin-list.py @@ -1,8 +1,13 @@ +# mypy: disallow-untyped-defs import datetime import pathlib import re from textwrap import dedent from textwrap import indent +from typing import Any +from typing import Iterable +from typing import Iterator +from typing import TypedDict import packaging.version import platformdirs @@ -109,7 +114,17 @@ def pytest_plugin_projects_from_pypi(session: CachedSession) -> dict[str, int]: } -def iter_plugins(): +class PluginInfo(TypedDict): + """Relevant information about a plugin to generate the summary.""" + + name: str + summary: str + last_release: str + status: str + requires: str + + +def iter_plugins() -> Iterator[PluginInfo]: session = get_session() name_2_serial = pytest_plugin_projects_from_pypi(session) @@ -136,7 +151,7 @@ def iter_plugins(): requires = requirement break - def version_sort_key(version_string): + def version_sort_key(version_string: str) -> Any: """ Return the sort key for the given version string returned by the API. @@ -162,20 +177,20 @@ def iter_plugins(): yield { "name": name, "summary": summary.strip(), - "last release": last_release, + "last_release": last_release, "status": status, "requires": requires, } -def plugin_definitions(plugins): +def plugin_definitions(plugins: Iterable[PluginInfo]) -> Iterator[str]: """Return RST for the plugin list that fits better on a vertical page.""" for plugin in plugins: yield dedent( f""" {plugin['name']} - *last release*: {plugin["last release"]}, + *last release*: {plugin["last_release"]}, *status*: {plugin["status"]}, *requires*: {plugin["requires"]} @@ -184,7 +199,7 @@ def plugin_definitions(plugins): ) -def main(): +def main() -> None: plugins = [*iter_plugins()] reference_dir = pathlib.Path("doc", "en", "reference")