fix(ci): verify a release whose run failed after publishing (#842)

* fix(ci): verify a release whose run failed after publishing

* fix(ci): only forgive a confirmed 404 when verifying a published release

.github/workflows/verify-release.yml

@@ -23,12 +23,14 @@ jobs:
   verify:
     name: Verify the published release
     runs-on: ubuntu-latest
-    # A failed release run has already said so, and half a release cannot verify.
-    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    # No conclusion guard: v0.17.0 published seven assets from a run that
+    # failed on publish-snap, and the guard skipped it. --if-published covers
+    # the other direction, a run that published nothing.
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # Through the environment: a dispatch input is caller supplied text.
           TAG: ${{ inputs.tag || github.event.workflow_run.head_branch }}
-        run: python3 scripts/verify_release.py "$TAG"
+          IF_PUBLISHED: ${{ github.event_name == 'workflow_run' && '--if-published' || '' }}
+        run: python3 scripts/verify_release.py "$TAG" ${IF_PUBLISHED:+"$IF_PUBLISHED"}

scripts/verify_release.py

@@ -13,7 +13,9 @@ release looks like once its run has finished. This reads the release:
 Nothing is downloaded: GitHub reports a sha256 for every asset it stores, and an
 asset it reports none for fails rather than being skipped.
 
-Usage: scripts/verify_release.py [tag]   (default: the latest stable release)
+Usage: scripts/verify_release.py [tag] [--if-published]
+       Default tag: the latest stable release. --if-published passes a tag that
+       published nothing.
 """
 
 from __future__ import annotations
@@ -36,6 +38,14 @@ NOTES_MUST_MENTION = ("sha256sum -c", "gh attestation verify", MANIFEST)
 SLSA_PROVENANCE = "https%3A%2F%2Fslsa.dev%2Fprovenance%2Fv1"
 
 
+class ReleaseNotFound(SystemExit):
+    """The API confirmed no release for the tag, with a 404.
+
+    Only this is forgivable. A rate limit or an auth failure stays a plain
+    SystemExit, or --if-published would pass a release nothing has read.
+    """
+
+
 def _gh(*args: str) -> subprocess.CompletedProcess:
     return subprocess.run(["gh", *args], capture_output=True, text=True)
 
@@ -76,6 +86,7 @@ def fetch_release(tag: str | None) -> dict:
     slug = repo_slug()
     candidates = [tag] + ([f"v{tag}"] if tag and tag[0].isdigit() else [])
     errors: list[str] = []
+    absent = True
     for candidate in candidates:
         path = (
             f"/repos/{slug}/releases/tags/{candidate}"
@@ -86,10 +97,11 @@ def fetch_release(tag: str | None) -> dict:
         if done.returncode == 0:
             return _release_from_api(json.loads(done.stdout))
         detail = (done.stderr or done.stdout or "").strip()
+        absent = absent and "HTTP 404" in detail
         errors.append(f"{path}: {detail or f'exit {done.returncode}'}")
     named = " or ".join(c for c in candidates if c) or "the latest stable release"
     hint = "\n".join(errors)
-    raise SystemExit(
+    raise (ReleaseNotFound if absent else SystemExit)(
         f"no release found for {named}\n{hint}\nusage: verify_release.py [tag], e.g. v0.16.2"
     )
 
@@ -205,9 +217,20 @@ def report(results: list[tuple[str, list[str]]]) -> bool:
 
 
 def main(argv: list[str]) -> int:
-    tag = argv[1] if len(argv) > 1 and argv[1] else None
+    args = [arg for arg in argv[1:] if arg]
+    if_published = "--if-published" in args
+    positional = [arg for arg in args if arg != "--if-published"]
+    tag = positional[0] if positional else None
     slug = repo_slug()
-    release = fetch_release(tag)
+    try:
+        release = fetch_release(tag)
+    except ReleaseNotFound:
+        if not if_published:
+            raise
+        # Nothing published, nothing to be wrong about, and the Release run
+        # has already gone red.
+        print(f"no release published for {tag or 'the latest stable tag'}, nothing to verify")
+        return 0
 
     if release["isDraft"] or release["isPrerelease"]:
         # Nightlies never go through publish-checksums and carry no manifest.

tests/test_verify_release.py

@@ -11,6 +11,8 @@ import json
 import re
 from pathlib import Path
 
+import pytest
+
 REPO_ROOT = Path(__file__).resolve().parents[1]
 WORKFLOW = REPO_ROOT / ".github" / "workflows" / "verify-release.yml"
 SCRIPT = REPO_ROOT / "scripts" / "verify_release.py"
@@ -242,3 +244,84 @@ def test_pypi_version_uses_removeprefix_not_lstrip():
     source = inspect.getsource(verify.main)
     assert 'removeprefix("v")' in source
     assert 'lstrip("v")' not in source
+
+
+def test_the_job_is_not_skipped_when_the_release_run_fails() -> None:
+    """v0.17.0 published seven assets from a run that failed on publish-snap.
+
+    The conclusion guard skipped it, and a skipped job is green.
+    """
+    text = WORKFLOW.read_text(encoding="utf-8")
+    assert "workflow_run.conclusion" not in text, "the job is gated on the release run's conclusion"
+    assert not re.search(r"^\s*if:", text, re.M), "the verify job carries a job-level if:"
+    assert "--if-published" in text, "nothing tells the script a missing release is not a failure"
+
+
+def test_only_the_workflow_run_trigger_forgives_a_missing_release() -> None:
+    """A person naming a tag by hand should get an error, not a pass."""
+    text = WORKFLOW.read_text(encoding="utf-8")
+    flag = re.search(r"IF_PUBLISHED: \$\{\{ (?P<expr>.+?) \}\}", text)
+    assert flag, "no IF_PUBLISHED expression to check"
+    assert "github.event_name == 'workflow_run'" in flag.group("expr")
+
+
+def test_a_tag_that_published_nothing_passes_under_if_published(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    def fake_fetch(_tag: str | None) -> dict:
+        raise verify.ReleaseNotFound("no release found for v9.9.9")
+
+    monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux")
+    monkeypatch.setattr(verify, "fetch_release", fake_fetch)
+
+    assert verify.main(["verify_release.py", "v9.9.9", "--if-published"]) == 0
+    assert "nothing to verify" in capsys.readouterr().out
+
+
+def test_the_same_tag_fails_without_the_flag(monkeypatch: pytest.MonkeyPatch) -> None:
+    """`just verify-release v9.9.9` must still say the release is not there."""
+
+    def fake_fetch(_tag: str | None) -> dict:
+        raise verify.ReleaseNotFound("no release found for v9.9.9")
+
+    monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux")
+    monkeypatch.setattr(verify, "fetch_release", fake_fetch)
+
+    with pytest.raises(verify.ReleaseNotFound):
+        verify.main(["verify_release.py", "v9.9.9"])
+
+
+def test_the_flag_is_not_mistaken_for_a_tag(monkeypatch: pytest.MonkeyPatch) -> None:
+    """`--if-published` alone must still mean the latest stable release."""
+    seen: list[str | None] = []
+
+    def fake_fetch(tag: str | None) -> dict:
+        seen.append(tag)
+        raise verify.ReleaseNotFound("no release found")
+
+    monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux")
+    monkeypatch.setattr(verify, "fetch_release", fake_fetch)
+
+    assert verify.main(["verify_release.py", "", "--if-published"]) == 0
+    assert seen == [None], f"the flag or an empty tag was read as a tag: {seen}"
+
+
+def test_an_api_error_that_is_not_a_404_is_not_forgiven(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A rate limit must not read as a tag that published nothing.
+
+    Otherwise --if-published turns every API outage into a green check over a
+    release nothing has read.
+    """
+
+    class Done:
+        returncode = 1
+        stdout = ""
+        stderr = "gh: API rate limit exceeded (HTTP 403)"
+
+    monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux")
+    monkeypatch.setattr(verify, "_gh", lambda *args: Done())
+
+    with pytest.raises(SystemExit) as raised:
+        verify.main(["verify_release.py", "v0.17.0", "--if-published"])
+    assert not isinstance(raised.value, verify.ReleaseNotFound)
+    assert "HTTP 403" in str(raised.value)