diff --git a/.github/workflows/verify-release.yml b/.github/workflows/verify-release.yml index e1b8e6b..5d9cebc 100644 --- a/.github/workflows/verify-release.yml +++ b/.github/workflows/verify-release.yml @@ -23,12 +23,14 @@ jobs: verify: name: Verify the published release runs-on: ubuntu-latest - # A failed release run has already said so, and half a release cannot verify. - if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + # No conclusion guard: v0.17.0 published seven assets from a run that + # failed on publish-snap, and the guard skipped it. --if-published covers + # the other direction, a run that published nothing. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Through the environment: a dispatch input is caller supplied text. TAG: ${{ inputs.tag || github.event.workflow_run.head_branch }} - run: python3 scripts/verify_release.py "$TAG" + IF_PUBLISHED: ${{ github.event_name == 'workflow_run' && '--if-published' || '' }} + run: python3 scripts/verify_release.py "$TAG" ${IF_PUBLISHED:+"$IF_PUBLISHED"} diff --git a/scripts/verify_release.py b/scripts/verify_release.py index 9dcecb4..0649dbc 100755 --- a/scripts/verify_release.py +++ b/scripts/verify_release.py @@ -13,7 +13,9 @@ release looks like once its run has finished. This reads the release: Nothing is downloaded: GitHub reports a sha256 for every asset it stores, and an asset it reports none for fails rather than being skipped. -Usage: scripts/verify_release.py [tag] (default: the latest stable release) +Usage: scripts/verify_release.py [tag] [--if-published] + Default tag: the latest stable release. --if-published passes a tag that + published nothing. """ from __future__ import annotations @@ -36,6 +38,14 @@ NOTES_MUST_MENTION = ("sha256sum -c", "gh attestation verify", MANIFEST) SLSA_PROVENANCE = "https%3A%2F%2Fslsa.dev%2Fprovenance%2Fv1" +class ReleaseNotFound(SystemExit): + """The API confirmed no release for the tag, with a 404. + + Only this is forgivable. A rate limit or an auth failure stays a plain + SystemExit, or --if-published would pass a release nothing has read. + """ + + def _gh(*args: str) -> subprocess.CompletedProcess: return subprocess.run(["gh", *args], capture_output=True, text=True) @@ -76,6 +86,7 @@ def fetch_release(tag: str | None) -> dict: slug = repo_slug() candidates = [tag] + ([f"v{tag}"] if tag and tag[0].isdigit() else []) errors: list[str] = [] + absent = True for candidate in candidates: path = ( f"/repos/{slug}/releases/tags/{candidate}" @@ -86,10 +97,11 @@ def fetch_release(tag: str | None) -> dict: if done.returncode == 0: return _release_from_api(json.loads(done.stdout)) detail = (done.stderr or done.stdout or "").strip() + absent = absent and "HTTP 404" in detail errors.append(f"{path}: {detail or f'exit {done.returncode}'}") named = " or ".join(c for c in candidates if c) or "the latest stable release" hint = "\n".join(errors) - raise SystemExit( + raise (ReleaseNotFound if absent else SystemExit)( f"no release found for {named}\n{hint}\nusage: verify_release.py [tag], e.g. v0.16.2" ) @@ -205,9 +217,20 @@ def report(results: list[tuple[str, list[str]]]) -> bool: def main(argv: list[str]) -> int: - tag = argv[1] if len(argv) > 1 and argv[1] else None + args = [arg for arg in argv[1:] if arg] + if_published = "--if-published" in args + positional = [arg for arg in args if arg != "--if-published"] + tag = positional[0] if positional else None slug = repo_slug() - release = fetch_release(tag) + try: + release = fetch_release(tag) + except ReleaseNotFound: + if not if_published: + raise + # Nothing published, nothing to be wrong about, and the Release run + # has already gone red. + print(f"no release published for {tag or 'the latest stable tag'}, nothing to verify") + return 0 if release["isDraft"] or release["isPrerelease"]: # Nightlies never go through publish-checksums and carry no manifest. diff --git a/tests/test_verify_release.py b/tests/test_verify_release.py index d8748c8..a2ee30f 100644 --- a/tests/test_verify_release.py +++ b/tests/test_verify_release.py @@ -11,6 +11,8 @@ import json import re from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] WORKFLOW = REPO_ROOT / ".github" / "workflows" / "verify-release.yml" SCRIPT = REPO_ROOT / "scripts" / "verify_release.py" @@ -242,3 +244,84 @@ def test_pypi_version_uses_removeprefix_not_lstrip(): source = inspect.getsource(verify.main) assert 'removeprefix("v")' in source assert 'lstrip("v")' not in source + + +def test_the_job_is_not_skipped_when_the_release_run_fails() -> None: + """v0.17.0 published seven assets from a run that failed on publish-snap. + + The conclusion guard skipped it, and a skipped job is green. + """ + text = WORKFLOW.read_text(encoding="utf-8") + assert "workflow_run.conclusion" not in text, "the job is gated on the release run's conclusion" + assert not re.search(r"^\s*if:", text, re.M), "the verify job carries a job-level if:" + assert "--if-published" in text, "nothing tells the script a missing release is not a failure" + + +def test_only_the_workflow_run_trigger_forgives_a_missing_release() -> None: + """A person naming a tag by hand should get an error, not a pass.""" + text = WORKFLOW.read_text(encoding="utf-8") + flag = re.search(r"IF_PUBLISHED: \$\{\{ (?P.+?) \}\}", text) + assert flag, "no IF_PUBLISHED expression to check" + assert "github.event_name == 'workflow_run'" in flag.group("expr") + + +def test_a_tag_that_published_nothing_passes_under_if_published( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def fake_fetch(_tag: str | None) -> dict: + raise verify.ReleaseNotFound("no release found for v9.9.9") + + monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux") + monkeypatch.setattr(verify, "fetch_release", fake_fetch) + + assert verify.main(["verify_release.py", "v9.9.9", "--if-published"]) == 0 + assert "nothing to verify" in capsys.readouterr().out + + +def test_the_same_tag_fails_without_the_flag(monkeypatch: pytest.MonkeyPatch) -> None: + """`just verify-release v9.9.9` must still say the release is not there.""" + + def fake_fetch(_tag: str | None) -> dict: + raise verify.ReleaseNotFound("no release found for v9.9.9") + + monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux") + monkeypatch.setattr(verify, "fetch_release", fake_fetch) + + with pytest.raises(verify.ReleaseNotFound): + verify.main(["verify_release.py", "v9.9.9"]) + + +def test_the_flag_is_not_mistaken_for_a_tag(monkeypatch: pytest.MonkeyPatch) -> None: + """`--if-published` alone must still mean the latest stable release.""" + seen: list[str | None] = [] + + def fake_fetch(tag: str | None) -> dict: + seen.append(tag) + raise verify.ReleaseNotFound("no release found") + + monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux") + monkeypatch.setattr(verify, "fetch_release", fake_fetch) + + assert verify.main(["verify_release.py", "", "--if-published"]) == 0 + assert seen == [None], f"the flag or an empty tag was read as a tag: {seen}" + + +def test_an_api_error_that_is_not_a_404_is_not_forgiven(monkeypatch: pytest.MonkeyPatch) -> None: + """A rate limit must not read as a tag that published nothing. + + Otherwise --if-published turns every API outage into a green check over a + release nothing has read. + """ + + class Done: + returncode = 1 + stdout = "" + stderr = "gh: API rate limit exceeded (HTTP 403)" + + monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux") + monkeypatch.setattr(verify, "_gh", lambda *args: Done()) + + with pytest.raises(SystemExit) as raised: + verify.main(["verify_release.py", "v0.17.0", "--if-published"]) + assert not isinstance(raised.value, verify.ReleaseNotFound) + assert "HTTP 403" in str(raised.value)