diff --git a/.github/workflows/verify-release.yml b/.github/workflows/verify-release.yml new file mode 100644 index 0000000..e1b8e6b --- /dev/null +++ b/.github/workflows/verify-release.yml @@ -0,0 +1,34 @@ +name: Verify Published Release + +# The guards in tests/ read workflow files, so none of them can see a release +# once its run has finished. This runs scripts/verify_release.py against the +# real thing. It cannot see a later hand edit either, so re-run it by hand +# after one: `just verify-release`. +on: + workflow_run: + workflows: [Release] + types: [completed] + workflow_dispatch: + inputs: + tag: + description: 'Tag to verify (default: the latest stable release)' + required: false + type: string + +permissions: + contents: read + attestations: read + +jobs: + verify: + name: Verify the published release + runs-on: ubuntu-latest + # A failed release run has already said so, and half a release cannot verify. + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Through the environment: a dispatch input is caller supplied text. + TAG: ${{ inputs.tag || github.event.workflow_run.head_branch }} + run: python3 scripts/verify_release.py "$TAG" diff --git a/AGENTS.md b/AGENTS.md index 5fe88e2..5103a37 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,6 +81,7 @@ just model-checksums # refresh pinned model digests after adding a model just appimage # build the AppImage in its pinned base image (needs docker) just appimage-boot fedora:42 # boot that AppImage in a distro container just aur-gate # build the AUR PKGBUILD on current Arch (needs docker) +just verify-release # check a published release as published (needs gh) just pre-commit # pre-commit run --all-files just run-debug # vocalinux --debug just run-source-debug diff --git a/docs/RELEASE_PROCESS.md b/docs/RELEASE_PROCESS.md index 8a5db95..e804bd5 100644 --- a/docs/RELEASE_PROCESS.md +++ b/docs/RELEASE_PROCESS.md @@ -79,8 +79,16 @@ Install `just` from https://just.systems or the distro package `just`. - [ ] Push branch and create PR - [ ] After PR merge, create tag: `git tag -a vX.Y.Z-PHASE -m "Release X.Y.Z-PHASE"` - [ ] Push tag: `git push origin vX.Y.Z-PHASE` + +### After the release run finishes +- [ ] Run `just verify-release` - the published release verifies as published +- [ ] Re-run it after any hand edit to the release: attaching an asset, or rewriting the notes ``` +`verify-release.yml` runs the same check when the release workflow finishes. +It cannot see a hand edit made afterwards, which is how every v0.16.2 defect +arrived, so run it yourself after touching a published release. + ## Detailed Release Steps ### Step 1: Determine Version Number @@ -158,7 +166,12 @@ Use these rules for every GitHub Release body (and for the draft pasted into the 6. Optional: `## Improvements`, `## Docs`, `## Packaging` 7. `## Thanks` — external PR authors and issue reporters by `@handle` 8. `## Install / Upgrade` — `install.sh`, AUR, PyPI, **AppImage**, Flatpak status (honest) -9. Footer compare link: `https://github.com/VocaHQ/vocalinux/compare/vPREV...vX.Y.Z` +9. `### Verifying what you downloaded` (required, and easy to lose). `release.yml` + generates it, with `sha256sum -c --ignore-missing SHA256SUMS` and + `gh attestation verify`. A hand-written body replaces the generated one, so carry + this block over deliberately. v0.16.2 published a checksum manifest and build + provenance and mentioned neither, so nothing a user reads told them either existed +10. Footer compare link: `https://github.com/VocaHQ/vocalinux/compare/vPREV...vX.Y.Z` #### Include / exclude diff --git a/justfile b/justfile index d579b2a..923d361 100644 --- a/justfile +++ b/justfile @@ -122,6 +122,12 @@ aur-gate: fi docker run "${ARGS[@]}" archlinux:latest bash "$PWD/packaging/aur/build-test.sh" +# Check that a published release verifies as published: manifest, provenance, +# notes and PyPI digests. Needs gh, downloads nothing. +# Usage: `just verify-release` for the latest, or `just verify-release v0.16.2` +verify-release tag="": + python3 scripts/verify_release.py {{tag}} + # Regenerate uv.lock and the hash-pinned requirements/* exports. # Bump the torch/torchaudio +cpu pins in requirements/whisper.in together # when you want newer CPU builds (torchaudio on the CPU index lags torch). diff --git a/scripts/verify_release.py b/scripts/verify_release.py new file mode 100755 index 0000000..9dcecb4 --- /dev/null +++ b/scripts/verify_release.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Check that a published release verifies as published. + +The guards in tests/ read workflow files, so none of them can see what a +release looks like once its run has finished. This reads the release: + + manifest SHA256SUMS is attached, lists every other asset, and its digests + match the bytes GitHub stores + provenance every artifact the manifest lists resolves in the attestations API + notes the body still tells users how to check both + pypi the wheel and the sdist on PyPI are the bytes on the release + +Nothing is downloaded: GitHub reports a sha256 for every asset it stores, and an +asset it reports none for fails rather than being skipped. + +Usage: scripts/verify_release.py [tag] (default: the latest stable release) +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys + +MANIFEST = "SHA256SUMS" +PYPI_PROJECT = "vocalinux" +PYPI_SUFFIXES = (".whl", ".tar.gz") +PYPI_TIMEOUT = 30 + +#: Both commands the notes should hand the user, plus the file they act on. +NOTES_MUST_MENTION = ("sha256sum -c", "gh attestation verify", MANIFEST) + +#: What `gh attestation verify` asks for, so this asks for the same. Pre-encoded: +#: `gh api` reads an unescaped `://` in a query as a protocol and refuses. +SLSA_PROVENANCE = "https%3A%2F%2Fslsa.dev%2Fprovenance%2Fv1" + + +def _gh(*args: str) -> subprocess.CompletedProcess: + return subprocess.run(["gh", *args], capture_output=True, text=True) + + +def _gh_ok(*args: str) -> str: + done = _gh(*args) + if done.returncode != 0: + raise SystemExit(f"gh {' '.join(args)} failed:\n{done.stderr.strip()}") + return done.stdout + + +def repo_slug() -> str: + slug = os.environ.get("GITHUB_REPOSITORY") + return slug or json.loads(_gh_ok("repo", "view", "--json", "nameWithOwner"))["nameWithOwner"] + + +def _release_from_api(payload: dict) -> dict: + """Map a Releases REST payload onto the field names the rest of this script uses.""" + return { + "tagName": payload["tag_name"], + "body": payload.get("body"), + "isDraft": payload["draft"], + "isPrerelease": payload["prerelease"], + "assets": payload.get("assets") or [], + } + + +def fetch_release(tag: str | None) -> dict: + """The latest stable release, or the one named by tag. `0.16.2` is accepted + as well as `v0.16.2`, since that is how the version is usually written. + + Digests come from the Releases REST API (`gh api`), not `gh release view + --json assets`. On gh 2.46 (common in distro packages, and what + `just verify-release` often runs) view's asset objects have no digest + field at all, so every asset would fail as undigested. The REST payload + includes `digest` for assets GitHub stores a hash for. + """ + slug = repo_slug() + candidates = [tag] + ([f"v{tag}"] if tag and tag[0].isdigit() else []) + errors: list[str] = [] + for candidate in candidates: + path = ( + f"/repos/{slug}/releases/tags/{candidate}" + if candidate + else f"/repos/{slug}/releases/latest" + ) + done = _gh("api", path) + if done.returncode == 0: + return _release_from_api(json.loads(done.stdout)) + detail = (done.stderr or done.stdout or "").strip() + errors.append(f"{path}: {detail or f'exit {done.returncode}'}") + named = " or ".join(c for c in candidates if c) or "the latest stable release" + hint = "\n".join(errors) + raise SystemExit( + f"no release found for {named}\n{hint}\nusage: verify_release.py [tag], e.g. v0.16.2" + ) + + +def parse_manifest(text: str) -> dict[str, str]: + """` ` per line, as sha256sum writes and reads it.""" + entries = {} + for number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + parts = line.split(None, 1) + if len(parts) != 2: + raise ValueError(f"{MANIFEST} line {number} is not ` `: {line!r}") + # Binary mode marks the name with a leading star, which is not the name. + entries[parts[1].strip().lstrip("*")] = parts[0].strip().lower() + return entries + + +def asset_digests(assets: list[dict]) -> tuple[dict[str, str], list[str]]: + """(name -> sha256, names GitHub reports no digest for).""" + digests: dict[str, str] = {} + undigested: list[str] = [] + for asset in assets: + raw = (asset.get("digest") or "").strip().lower() + if raw.startswith("sha256:"): + digests[asset["name"]] = raw.split(":", 1)[1] + else: + undigested.append(asset["name"]) + return digests, undigested + + +def check_manifest(stored: dict, manifest: dict, undigested: list) -> list[str]: + """Both directions: a missing line reads as tampering to anyone running + `sha256sum -c`, and an extra one is a promise nothing keeps.""" + covered, listed = set(stored) - {MANIFEST}, set(manifest) + return ( + [f"GitHub reports no digest for {name}" for name in undigested if name != MANIFEST] + + [f"{name} is published but absent from {MANIFEST}" for name in sorted(covered - listed)] + + [f"{MANIFEST} lists {name}, which is not an asset" for name in sorted(listed - covered)] + + [ + f"{name}: {MANIFEST} says {manifest[name]}, GitHub stores {stored[name]}" + for name in sorted(covered & listed) + if manifest[name] != stored[name] + ] + ) + + +def check_provenance(slug: str, manifest: dict[str, str]) -> list[str]: + """The manifest is the subject-checksums input, so it is not a subject itself.""" + problems = [] + for name, digest in sorted(manifest.items()): + query = f"/repos/{slug}/attestations/sha256:{digest}" + done = _gh("api", f"{query}?predicate_type={SLSA_PROVENANCE}") + try: + bundles = json.loads(done.stdout).get("attestations") if done.returncode == 0 else None + except json.JSONDecodeError: + bundles = None + if not bundles: + problems.append(f"{name} has no build provenance") + return problems + + +def check_notes(body: str | None) -> list[str]: + # GitHub returns JSON null for an empty body; treat that as missing notes. + text = body or "" + return [f"the notes never mention `{p}`" for p in NOTES_MUST_MENTION if p not in text] + + +def check_pypi(version: str, stored: dict[str, str]) -> list[str]: + expected = {n: d for n, d in stored.items() if n.endswith(PYPI_SUFFIXES)} + # Per suffix: a release that lost its sdist would otherwise pass on the wheel. + problems = [ + f"the release carries no {suffix} to compare against PyPI" + for suffix in PYPI_SUFFIXES + if not any(name.endswith(suffix) for name in expected) + ] + if not expected: + return problems + + # Past the early return, and not at module scope: urllib.error reaches + # tempfile, which two tests here leave as a MagicMock in sys.modules. + import urllib.error + import urllib.request + + try: + url = f"https://pypi.org/pypi/{PYPI_PROJECT}/{version}/json" + with urllib.request.urlopen(url, timeout=PYPI_TIMEOUT) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + if error.code == 404: + return problems + [f"PyPI has no {PYPI_PROJECT} {version}"] + raise + published = {i["filename"]: i["digests"]["sha256"].lower() for i in payload["urls"]} + + for name, digest in sorted(expected.items()): + if name not in published: + problems.append(f"{name} is on the release but not on PyPI") + elif published[name] != digest: + problems.append(f"{name}: PyPI serves {published[name]}, the release serves {digest}") + for name in sorted(published): + if name.endswith(PYPI_SUFFIXES) and name not in expected: + problems.append(f"{name} is on PyPI but not on the release") + return problems + + +def report(results: list[tuple[str, list[str]]]) -> bool: + width = max(len(label) for label, _ in results) + for label, problems in results: + print(f" {label.ljust(width)} {'FAIL' if problems else 'PASS'}") + for problem in problems: + print(f" {' ' * width} {problem}") + return not any(problems for _, problems in results) + + +def main(argv: list[str]) -> int: + tag = argv[1] if len(argv) > 1 and argv[1] else None + slug = repo_slug() + release = fetch_release(tag) + + if release["isDraft"] or release["isPrerelease"]: + # Nightlies never go through publish-checksums and carry no manifest. + print(f"{release['tagName']} is not a published stable release") + return 1 + + tag = release["tagName"] + stored, undigested = asset_digests(release["assets"]) + print(f"Verifying {slug} {tag}, {len(release['assets'])} assets\n") + + if MANIFEST not in stored and MANIFEST not in undigested: + results = [("manifest", [f"{tag} has no {MANIFEST} attached"])] + else: + text = _gh_ok("release", "download", tag, "--pattern", MANIFEST, "--output", "-") + manifest = parse_manifest(text) + results = [ + ("manifest", check_manifest(stored, manifest, undigested)), + ("provenance", check_provenance(slug, manifest)), + ("notes", check_notes(release["body"])), + ("pypi", check_pypi(tag.removeprefix("v"), stored)), + ] + + ok = report(results) + print(f"\n{tag} {'verifies' if ok else 'does not verify'} as published") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/test_verify_release.py b/tests/test_verify_release.py new file mode 100644 index 0000000..d8748c8 --- /dev/null +++ b/tests/test_verify_release.py @@ -0,0 +1,244 @@ +"""Guards for scripts/verify_release.py. + +Only the comparisons that carry real logic, plus the one property of the +workflow that is worth pinning. Asserting that the yml says what the yml says +is the antipattern verify_release.py exists to work around. +""" + +import importlib +import importlib.util +import json +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "verify-release.yml" +SCRIPT = REPO_ROOT / "scripts" / "verify_release.py" + +_spec = importlib.util.spec_from_file_location("verify_release", SCRIPT) +verify = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(verify) + + +def test_the_dispatch_input_never_reaches_the_shell(): + """Interpolating it into `run:` splices caller text into a script that + holds a token. Through the environment it stays a value.""" + text = WORKFLOW.read_text(encoding="utf-8") + run_lines = [line for line in text.splitlines() if re.match(r"^\s*-?\s*run:", line)] + assert run_lines, "found no run: step, so this guard is scanning nothing" + for line in run_lines: + assert "${{" not in line, f"a run: step interpolates a template expression: {line.strip()}" + assert 'python3 scripts/verify_release.py "$TAG"' in text + + +def test_the_manifest_parses_as_sha256sum_writes_it(): + parsed = verify.parse_manifest( + "0ee0d8e6 Vocalinux-0.16.2-x86_64.AppImage\n22deb51a *vocalinux-0.16.2.whl\n\n" + ) + assert parsed == { + "Vocalinux-0.16.2-x86_64.AppImage": "0ee0d8e6", + "vocalinux-0.16.2.whl": "22deb51a", + } + + +def test_an_asset_missing_from_the_manifest_fails(): + """The case that reads as tampering to anyone running `sha256sum -c`.""" + problems = verify.check_manifest( + {"a.AppImage": "aa", "b.flatpak": "bb", verify.MANIFEST: "cc"}, {"a.AppImage": "aa"}, [] + ) + assert any("b.flatpak is published but absent" in problem for problem in problems) + + +def test_a_manifest_line_with_no_asset_behind_it_fails(): + """How a hand edit adds a promise nothing keeps.""" + problems = verify.check_manifest( + {"a.AppImage": "aa", verify.MANIFEST: "cc"}, + {"a.AppImage": "aa", "ghost.flatpak": "bb"}, + [], + ) + assert any("which is not an asset" in problem for problem in problems) + + +def test_a_digest_that_disagrees_with_the_stored_bytes_fails(): + problems = verify.check_manifest({"a.AppImage": "aa"}, {"a.AppImage": "bb"}, []) + assert any("GitHub stores aa" in problem for problem in problems) + + +def test_an_asset_with_no_digest_fails_rather_than_being_skipped(): + """The manifest's own digest is never checked, so it is not reported.""" + problems = verify.check_manifest({}, {}, ["mystery.snap", verify.MANIFEST]) + assert problems == ["GitHub reports no digest for mystery.snap"] + + +def test_a_manifest_that_matches_the_release_passes(): + assert not verify.check_manifest( + {"a.AppImage": "aa", "b.flatpak": "bb", verify.MANIFEST: "cc"}, + {"a.AppImage": "aa", "b.flatpak": "bb"}, + [], + ) + + +def test_notes_without_the_verification_block_fail(): + """v0.16.2 published a manifest and told nobody it existed.""" + assert verify.check_notes("Download the AppImage and run it.") + assert not verify.check_notes( + "Check it with `sha256sum -c --ignore-missing SHA256SUMS`, then `gh attestation verify`." + ) + + +def test_provenance_asks_only_for_slsa_build_provenance(monkeypatch): + """Unfiltered, an SBOM would read as the provenance users are told to expect.""" + + class Done: + returncode = 0 + stdout = '{"attestations": [{}]}' + + asked = [] + + def fake_gh(*args): + asked.append(args[-1]) + return Done() + + monkeypatch.setattr(verify, "_gh", fake_gh) + assert not verify.check_provenance("owner/repo", {"a.whl": "aa"}) + assert f"predicate_type={verify.SLSA_PROVENANCE}" in asked[0] + + +def test_a_release_missing_a_distribution_names_it(): + """Comparing only what is left would pass while PyPI serves an unchecked file.""" + assert verify.check_pypi("0.16.2", {"Vocalinux-0.16.2-x86_64.AppImage": "aa"}) == [ + "the release carries no .whl to compare against PyPI", + "the release carries no .tar.gz to compare against PyPI", + ] + + +def test_an_extra_pypi_distribution_fails(monkeypatch): + """A differently-named wheel on PyPI would otherwise go unchecked.""" + import io + import sys + + # urllib.request imports tempfile. test_recognition_manager.py and + # test_speech_recognition.py leave a MagicMock in sys.modules, which + # makes that import raise a metaclass conflict. Drop the mock first: + # import_module would otherwise return it. + sys.modules.pop("tempfile", None) + sys.modules["tempfile"] = importlib.import_module("tempfile") + for name in [key for key in sys.modules if key == "urllib" or key.startswith("urllib.")]: + del sys.modules[name] + + import urllib.request + + wheel = "vocalinux-0.16.2-py3-none-any.whl" + sdist = "vocalinux-0.16.2.tar.gz" + extra = "vocalinux-0.16.2-extra.whl" + payload = { + "urls": [ + {"filename": wheel, "digests": {"sha256": "aa"}}, + {"filename": sdist, "digests": {"sha256": "bb"}}, + {"filename": extra, "digests": {"sha256": "cc"}}, + ] + } + + def fake_urlopen(url, timeout=None): + return io.StringIO(json.dumps(payload)) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + problems = verify.check_pypi("0.16.2", {wheel: "aa", sdist: "bb"}) + assert problems == [f"{extra} is on PyPI but not on the release"] + + +def test_github_digests_are_read_without_their_algorithm_prefix(): + stored, undigested = verify.asset_digests( + [{"name": "a.AppImage", "digest": "sha256:AABB"}, {"name": "b.snap"}] + ) + assert stored == {"a.AppImage": "aabb"} + assert undigested == ["b.snap"] + + +def test_a_null_release_body_fails_notes_without_typeerror(): + """GitHub serves JSON null for an empty body; that must be notes FAIL.""" + problems = verify.check_notes(None) + assert problems + assert all("never mention" in problem for problem in problems) + + +def test_fetch_release_reads_digests_from_the_rest_api(monkeypatch): + """`gh release view --json assets` omits digest on gh 2.46; REST does not.""" + + class Done: + returncode = 0 + stdout = "" + stderr = "" + + def __init__(self, stdout=""): + self.stdout = stdout + + calls = [] + + def fake_gh(*args): + calls.append(args) + assert args[:2] == ("api", "/repos/VocaHQ/vocalinux/releases/tags/v0.16.2") + payload = { + "tag_name": "v0.16.2", + "body": "sha256sum -c and gh attestation verify via SHA256SUMS", + "draft": False, + "prerelease": False, + "assets": [ + {"name": "a.AppImage", "digest": "sha256:AABB"}, + {"name": "b.snap", "digest": None}, + ], + } + return Done(stdout=json.dumps(payload)) + + monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux") + monkeypatch.setattr(verify, "_gh", fake_gh) + + release = verify.fetch_release("v0.16.2") + assert release["tagName"] == "v0.16.2" + assert release["body"] + assert release["isDraft"] is False + assert release["isPrerelease"] is False + stored, undigested = verify.asset_digests(release["assets"]) + assert stored == {"a.AppImage": "aabb"} + assert undigested == ["b.snap"] + assert all(args[0] == "api" for args in calls) + assert not any(args[0] == "release" for args in calls) + + +def test_fetch_release_tries_v_prefix_then_surfaces_api_errors(monkeypatch): + class Done: + returncode = 1 + stdout = "" + stderr = "HTTP 404: Not Found" + + paths = [] + + def fake_gh(*args): + paths.append(args[1]) + return Done() + + monkeypatch.setenv("GITHUB_REPOSITORY", "VocaHQ/vocalinux") + monkeypatch.setattr(verify, "_gh", fake_gh) + + try: + verify.fetch_release("0.16.2") + except SystemExit as error: + message = str(error) + else: + raise AssertionError("expected SystemExit") + + assert paths == [ + "/repos/VocaHQ/vocalinux/releases/tags/0.16.2", + "/repos/VocaHQ/vocalinux/releases/tags/v0.16.2", + ] + assert "HTTP 404: Not Found" in message + assert "no release found for 0.16.2 or v0.16.2" in message + + +def test_pypi_version_uses_removeprefix_not_lstrip(): + """lstrip('v') eats every leading v; removeprefix only the tag prefix.""" + import inspect + + source = inspect.getsource(verify.main) + assert 'removeprefix("v")' in source + assert 'lstrip("v")' not in source