From e61133e246d4c92273db9739e014a33bb156b04c Mon Sep 17 00:00:00 2001 From: Sergey Date: Sat, 5 Sep 2026 08:25:09 +0200 Subject: [PATCH] feat(ci): gate AUR PKGBUILD builds on every PR (#772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ci): gate AUR PKGBUILD builds on every PR Nothing ever ran makepkg on the PKGBUILD before a tag pushed it to AUR users, and the channel broke twice that way (#736, #757) — a PKGBUILD change did not even start a CI job, because no paths filter matched packaging/aur/**. The gate (packaging/aur/build-test.sh, run in archlinux:latest — the AUR is a rolling channel, so the image stays deliberately unpinned): - swaps the tag-archive source= for a git archive of the checkout under the same filename and prefix, so build()/package() run unchanged and the gate answers "does this commit build on Arch" - installs depends/makedepends from the repos; python-pywhispercpp (virtual, provided by the AUR backends, #579) and python-pynput (AUR) are satisfied by a stub so makepkg still verifies the full depends set, and pip-installed from the hash-pinned requirements/runtime.txt for the import smoke - runs namcap on the PKGBUILD and the built package, failing on errors only (its warnings are noise around lazy imports and virtual depends) - installs the package and imports the whole UI/recognition stack, plus the vocalinux --version entry point Part of #701 (phase 6). * fix(ci): stop partial-upgrading in the AUR gate and correct the #736 attribution --- .github/workflows/unified-pipeline.yml | 31 ++++ AGENTS.md | 1 + docs/AUR.md | 6 + justfile | 17 +++ packaging/aur/build-test.sh | 192 +++++++++++++++++++++++++ tests/test_aur_packaging.py | 174 ++++++++++++++++++++++ 6 files changed, 421 insertions(+) create mode 100644 packaging/aur/build-test.sh create mode 100644 tests/test_aur_packaging.py diff --git a/.github/workflows/unified-pipeline.yml b/.github/workflows/unified-pipeline.yml index 3d7d45c..deb0e74 100644 --- a/.github/workflows/unified-pipeline.yml +++ b/.github/workflows/unified-pipeline.yml @@ -23,6 +23,7 @@ jobs: outputs: web: ${{ steps.filter.outputs.web }} python: ${{ steps.filter.outputs.python }} + aur: ${{ steps.filter.outputs.aur }} deploy_website: ${{ steps.deploy_check.outputs.deploy }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -52,6 +53,14 @@ jobs: - 'justfile' - '.pre-commit-config.yaml' - '.github/workflows/unified-pipeline.yml' + # The AUR gate builds packaging/aur/vocalinux/PKGBUILD from this + # tree. Its depends= mirror pyproject.toml, so every python change + # reaches the gate through the python filter above; this filter + # exists so a PKGBUILD-only change runs the gate without dragging + # the whole python matrix along. + aur: + - 'packaging/aur/**' + - '.github/workflows/unified-pipeline.yml' - name: Check if website should deploy id: deploy_check env: @@ -307,6 +316,28 @@ jobs: "${{ matrix.distro }}" \ bash /pk/boot-test.sh "/dist/$(basename "$APPIMAGE")" + # Nothing built the PKGBUILD before a tag pushed it to AUR users, which is + # how #757 shipped: Arch's setuptools 84 against our old <82 cap. The gate + # swaps the not-yet-existing tag tarball for a git archive of the checkout, + # so it answers whether *this commit* builds on Arch. Deliberately + # archlinux:latest: the AUR is a rolling channel and #757 was Arch moving + # under us. + aur-build-test: + name: 🏛️ Build AUR package (gate) + runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.python == 'true' || needs.changes.outputs.aur == 'true' + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build the PKGBUILD on Arch + run: | + docker run --rm \ + -v "$PWD:/repo:ro" \ + archlinux:latest \ + bash /repo/packaging/aur/build-test.sh + web-build: name: 🌐 Build Website runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 650215d..5fe88e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,6 +80,7 @@ just lock-check # fail if uv.lock is stale vs pyproject.toml just model-checksums # refresh pinned model digests after adding a model just appimage # build the AppImage in its pinned base image (needs docker) just appimage-boot fedora:42 # boot that AppImage in a distro container +just aur-gate # build the AUR PKGBUILD on current Arch (needs docker) just pre-commit # pre-commit run --all-files just run-debug # vocalinux --debug just run-source-debug diff --git a/docs/AUR.md b/docs/AUR.md index b42003c..89c46cd 100644 --- a/docs/AUR.md +++ b/docs/AUR.md @@ -32,6 +32,12 @@ git clone ssh://aur@aur.archlinux.org/vocalinux.git Sources: `packaging/aur/vocalinux/PKGBUILD`. Community git package: https://aur.archlinux.org/packages/vocalinux-git +## CI gate + +Every PR that changes the Python source or `packaging/aur/**` builds the PKGBUILD on `archlinux:latest` — the `Build AUR package (gate)` job in `unified-pipeline.yml`. Run it locally with `just aur-gate` (needs docker). + +The gate swaps the tag-archive `source=` for a `git archive` of the checkout, so it answers whether *this commit* builds on Arch — #757 reached users after the tag, when nothing could fail a build anymore (#736 reached Arch users too, but through `install.sh`; this package built fine). `python-pywhispercpp` (virtual, provided by the AUR backends) and `python-pynput` (AUR) cannot come from the repos, so the gate satisfies them with a stub for dependency resolution and pip-installs the pinned versions from `requirements/runtime.txt` for the import smoke. What AUR actually ships (python-pywhispercpp-cpu is still 1.4.x — see Troubleshooting) is a monitoring question, not the gate's. + ## Troubleshooting **`ERROR Missing dependencies: setuptools<82`** diff --git a/justfile b/justfile index 53f3afa..d579b2a 100644 --- a/justfile +++ b/justfile @@ -105,6 +105,23 @@ appimage: build appimage-boot distro="debian:12": docker run --rm -v "$PWD/dist:/dist:ro" -v "$PWD/packaging/appimage:/pk:ro" {{distro}} bash /pk/boot-test.sh "/dist/$(basename "$(ls dist/*.AppImage)")" +# Build the AUR package from this checkout on current Arch, as the CI gate +# does. Answers "does this commit build on Arch" — the tag tarball source= is +# swapped for a git archive of HEAD (needs docker). +# +# The checkout is mounted at its own path, and the main .git with it when this +# is a linked worktree: a worktree's .git is a pointer to a gitdir outside the +# tree, so git archive inside the container cannot see the objects without it. +aur-gate: + #!/usr/bin/env bash + set -euo pipefail + COMMON="$(cd "$(git rev-parse --git-common-dir)" && pwd)" + ARGS=(--rm -v "$PWD:$PWD:ro" -e REPO="$PWD") + if [ "$COMMON" != "$PWD/.git" ]; then + ARGS+=(-v "$COMMON:$COMMON:ro") + fi + docker run "${ARGS[@]}" archlinux:latest bash "$PWD/packaging/aur/build-test.sh" + # Regenerate uv.lock and the hash-pinned requirements/* exports. # Bump the torch/torchaudio +cpu pins in requirements/whisper.in together # when you want newer CPU builds (torchaudio on the CPU index lags torch). diff --git a/packaging/aur/build-test.sh b/packaging/aur/build-test.sh new file mode 100644 index 0000000..902cc46 --- /dev/null +++ b/packaging/aur/build-test.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# Build the AUR package from this checkout, on Arch as it is today. +# +# Usage (the read-only repo mount is enough — the tree is taken via git archive): +# docker run --rm -v "$PWD:/repo:ro" archlinux:latest \ +# bash /repo/packaging/aur/build-test.sh +# or: just aur-gate +# +# This is the gate #757 needed: Arch extra moved to setuptools 84 while +# build-system.requires still capped it below 82, so the PKGBUILD's +# --no-isolation build broke for AUR users — and nothing ever ran makepkg on +# the PKGBUILD before a tag published it. +# The published source= points at the tag tarball, which does not exist until +# the tag is pushed, so the gate replaces it with a git archive of HEAD under +# the same filename: build()/package() run unchanged, and the gate answers +# "does this commit build on Arch", not "did the last tag build". +# +# Not this gate's, despite reaching Arch users: #736 came through install.sh, +# not through this package. The AUR build of v0.16.0 was intact, because the +# tree it built carried neither model_checksums.txt nor the code reading it. +# Closing that one takes a build gate for install.sh, not for this package. +# +# Deliberately archlinux:latest, unpinned: the AUR is a rolling channel and +# #757 was Arch moving under us. A pinned image would test a world that no +# longer exists. +# +# Two depends cannot come from the repos: python-pywhispercpp is a virtual +# name that the AUR -cpu/-cuda/-rocm backends provide (#579), and +# python-pynput lives in the AUR itself. A stub package with provides=() +# satisfies them so makepkg verifies the full depends set; for the import +# smoke they are pip-installed from the hash-pinned requirements/runtime.txt, +# so the smoke runs the versions we pin, not whatever AUR ships — that drift +# (AUR -cpu is still 1.4.x) is a monitoring job, not this gate. +# +# What the smoke therefore does NOT prove: that depends=() is complete. This +# container carries base-devel and python-pip with their own closures, and the +# two names above arrive via pip --no-deps, so a missing *direct* dependency +# still passes. Transitive ones are the provider's to declare — the backend +# providing python-pywhispercpp declares python-platformdirs itself. +set -euo pipefail + +REPO="${REPO:-/repo}" +PKGDIR="$REPO/packaging/aur/vocalinux" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +# Retry: a mirror going mid-transaction slow leaves half the cache and a +# clean database, so re-running the same transaction resumes rather than +# restarts. +pacman_retry() { + local attempt + for attempt in 1 2 3; do + pacman "$@" && return 0 + echo " pacman failed (attempt $attempt); retrying" >&2 + sleep 5 + done + return 1 +} + +. /etc/os-release +[ "$ID" = "arch" ] || fail "this gate runs on Arch Linux (got '$ID')" + +[ -f "$PKGDIR/PKGBUILD" ] || fail "$PKGDIR/PKGBUILD not found" + +# Upgrade before installing anything. archlinux:latest lags the mirrors, so +# -Sy alone would put fresh packages on a stale base — the partial upgrade +# Arch warns about, and a breakage this gate would be causing rather than +# detecting. It is also what "Arch as it is today" has to mean here. +pacman_retry -Syu --noconfirm >/dev/null + +# The base image ships no git; the tree for makepkg comes from git archive. +# safe.directory: the mount is owned by another uid than the container user. +pacman_retry -S --needed --noconfirm git >/dev/null +git config --global --add safe.directory '*' +git -C "$REPO" rev-parse --verify HEAD >/dev/null \ + || fail "$REPO is not a git checkout; git archive needs it" + +# Sourcing gives us pkgname, _tag and the depends/makedepends arrays the same +# way makepkg itself reads them. +# shellcheck source=/dev/null +source "$PKGDIR/PKGBUILD" + +echo "== Arch, $(date -u +%F) ==" +echo " building ${pkgname} ${pkgver}-${pkgrel} (tagged _tag=${_tag})" +pacman -Q python python-setuptools 2>/dev/null || true + +echo "== Install depends and makedepends from the repos ==" +# AUR-resolved depends (see header): satisfied by the stub below, not pacman. +AUR_RESOLVED="python-pywhispercpp python-pynput" +repo_deps=() +for dep in "${depends[@]}" "${makedepends[@]}"; do + case " $AUR_RESOLVED " in + *" $dep "*) ;; + *) repo_deps+=("$dep") ;; + esac +done +# --needed keeps this idempotent; an unknown name here is a gate failure in +# its own right: it means the PKGBUILD depends on something Arch does not +# package (how a python-pywhispercpp-cpu rename would go unnoticed again). +# base-devel is the documented prerequisite for building any AUR package +# (fakeroot, debugedit and friends); makepkg aborts without it. +pacman_retry -S --needed --noconfirm --asdeps base-devel namcap python-pip \ + "${repo_deps[@]}" >/dev/null + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +echo "== Satisfy the AUR-resolved depends for dependency resolution ==" +# makepkg verifies every depends entry against the local pacman database. +# Without this stub it would fail on the two names no repo can provide. +STUB="$WORKDIR/stub" +mkdir -p "$STUB" +cat >"$STUB/PKGBUILD" </dev/null + pacman -U --noconfirm ./*.pkg.tar.* >/dev/null +) + +echo "== makepkg ==" +( + cd "$BUILD" + # No -s and no --nodeps: depends were installed above, the stub satisfies + # the AUR-resolved two, so makepkg's own verification is part of the gate. + runuser -u builder -- makepkg -f --noconfirm +) + +echo "== namcap ==" +check_namcap() { + local target="$1" out + out="$(namcap "$target" 2>&1 || true)" + echo "$out" | sed 's/^/ /' + if echo "$out" | grep -q ' E: '; then + fail "namcap reports errors on $target" + fi +} +check_namcap "$BUILD/PKGBUILD" +check_namcap "$BUILD"/*.pkg.tar.* + +echo "== Install the package and smoke it ==" +# The two AUR-resolved deps as importable modules, at the versions we pin. +awk '/^(pywhispercpp|pynput)==/{p=1} p{print; if ($0 !~ /\\$/) p=0}' \ + "$REPO/requirements/runtime.txt" >"$WORKDIR/pinned.txt" +pip install --no-deps --break-system-packages --require-hashes -r "$WORKDIR/pinned.txt" >/dev/null + +pacman -U --noconfirm "$BUILD"/*.pkg.tar.* >/dev/null + +# The console script proves the wheel's entry points landed; the imports walk +# every depends entry the package declares (gi/Gtk/AppIndicator, IBus, cairo, +# pyaudio, numpy, requests, tqdm, psutil, lxml, pydub, evdev, xlib, pynput). +vocalinux --version +python - <<'PY' || fail "the installed package does not import" +import vocalinux.main +import vocalinux.ui.tray_indicator +import vocalinux.speech_recognition.recognition_manager +import vocalinux.speech_recognition.command_processor +import vocalinux.text_injection.text_injector +import vocalinux.text_injection.ibus_engine +import vocalinux.ui.keyboard_backends +import pywhispercpp.model + +print(" main, tray, recognition, injection and keyboard backends import") +PY + +echo "PASS: the PKGBUILD builds on Arch (this commit, not the last tag)" diff --git a/tests/test_aur_packaging.py b/tests/test_aur_packaging.py new file mode 100644 index 0000000..8e16560 --- /dev/null +++ b/tests/test_aur_packaging.py @@ -0,0 +1,174 @@ +"""Regression guards for the AUR packaging and its build gate. + +#757 reached AUR users because no CI ever ran makepkg on the PKGBUILD before +a tag pushed it to them — and a PKGBUILD change triggered zero jobs in the +first place, because no paths filter matched packaging/aur/**. +""" + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +AUR = REPO_ROOT / "packaging" / "aur" +PKGBUILD = AUR / "vocalinux" / "PKGBUILD" +GATE_SH = AUR / "build-test.sh" +WORKFLOWS = REPO_ROOT / ".github" / "workflows" +PIPELINE = WORKFLOWS / "unified-pipeline.yml" +RELEASE = WORKFLOWS / "release.yml" +RUNTIME_EXPORT = REPO_ROOT / "requirements" / "runtime.txt" + +#: depends entries no Arch repo can provide: python-pywhispercpp is a virtual +#: name the AUR -cpu/-cuda/-rocm backends provide (#579), python-pynput lives +#: in the AUR. The gate satisfies them with a stub package; adding another +#: AUR-resolved name to the PKGBUILD means adding it here too. +AUR_RESOLVED = ("python-pywhispercpp", "python-pynput") + + +def _pipeline_text() -> str: + return PIPELINE.read_text(encoding="utf-8") + + +def _job_block(workflow_text: str, job: str) -> str: + """The job's lines, from its key to the next key at the same indent.""" + lines = workflow_text.splitlines(keepends=True) + start = next( + (i for i, line in enumerate(lines) if line.rstrip("\n") == f" {job}:"), + None, + ) + assert start is not None, f"no {job} job in the workflow" + block = [lines[start]] + for line in lines[start + 1 :]: + # The next job starts at two-space indent; deeper lines belong to us. + if re.match(r"^ \S", line): + break + block.append(line) + return "".join(block) + + +def _aur_depends() -> set: + """The depends=() entries of the PKGBUILD, one per line by convention.""" + lines = PKGBUILD.read_text(encoding="utf-8").splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("depends=(")) + names = set() + for line in lines[start + 1 :]: + if line.strip() == ")": + break + match = re.match(r"\s*'([^']+)'", line) + assert match, f"depends=() entry '{line}' is not one-per-line quoted" + names.add(match.group(1)) + return names + + +def test_aur_changes_reach_ci(): + """The bug that let the channel break silently: packaging/aur/** matched + no paths filter, so a PKGBUILD change started zero jobs.""" + text = _pipeline_text() + assert re.search(r"^ aur:\n", text, re.M), ( + "unified-pipeline.yml has no aur filter; PKGBUILD changes again start" " zero jobs" + ) + aur_block = re.search(r"^ aur:\n((?:\s+- '[^']+'\n)+)", text, re.M) + assert "packaging/aur/**" in aur_block.group(1) + # The filter's output has to be consumed, or the job never runs. + assert "aur: ${{ steps.filter.outputs.aur }}" in text + + +def test_the_gate_runs_on_python_changes_and_aur_changes(): + """depends=() mirrors pyproject.toml, so every python change must reach + the AUR gate; the dedicated aur filter exists so a PKGBUILD-only change + runs it without dragging the whole python matrix along.""" + block = _job_block(_pipeline_text(), "aur-build-test") + assert "needs.changes.outputs.python == 'true'" in block + assert "needs.changes.outputs.aur == 'true'" in block + assert "packaging/aur/build-test.sh" in block, "the job must run the gate" + assert "archlinux" in block, "the gate builds on Arch, not the runner" + + +def test_the_gate_builds_this_commit_not_the_last_tag(): + """The published source= points at the tag tarball, which does not exist + until the tag is pushed — the reason nothing could run the PKGBUILD on a + PR. The gate swaps it for a git archive of HEAD under the same filename + and the same directory prefix build()/package() cd into.""" + script = GATE_SH.read_text(encoding="utf-8") + pkgbuild = PKGBUILD.read_text(encoding="utf-8") + + assert "git archive" in script and '--prefix="${pkgname}-${_tag}/"' in script + assert re.search(r'cd "\$\{pkgname\}-\$\{_tag\}"', pkgbuild), ( + "build()/package() no longer cd into ${pkgname}-${_tag}; the gate's" + " archive prefix and the PKGBUILD must agree" + ) + # The substitution itself. + assert re.search(r"s\|\^source=.*\|source=", script) + + # And the published PKGBUILD still fetches the tag archive — if that + # changes, updpkgsums at publish time and this gate both need a relook. + assert "archive/refs/tags/v${_tag}.tar.gz" in pkgbuild + assert "sha256sums=('SKIP')" in pkgbuild + assert "updpkgsums: 'true'" in RELEASE.read_text(encoding="utf-8"), ( + "release.yml no longer computes the digest at publish time; the" + " sha256sums=('SKIP') in the repo would ship to users" + ) + + +def test_the_gate_uses_a_non_root_build_user(): + """makepkg refuses to run as root; the gate must drop privileges.""" + script = GATE_SH.read_text(encoding="utf-8") + assert "useradd" in script + assert re.search(r"runuser -u \w+ -- makepkg", script) + + +def test_namcap_covers_both_the_pkgbuild_and_the_package_and_fails_on_errors(): + """namcap is noisy about lazy imports and virtual depends, so the gate + fails on E tags only — but it must run on both targets to be worth its + output.""" + script = GATE_SH.read_text(encoding="utf-8") + assert 'check_namcap "$BUILD/PKGBUILD"' in script + assert 'check_namcap "$BUILD"/*.pkg.tar.*' in script + assert "grep -q ' E: '" in script + + +def test_the_stub_matches_the_aur_resolved_depends(): + """makepkg verifies every depends entry, so the gate's stub has to + provide exactly the names no repo can. A name the stub still provides + that the PKGBUILD dropped, or a new AUR-resolved dep the stub does not + know, fails the gate at pacman -Sy — loudly, but far from the cause.""" + script = GATE_SH.read_text(encoding="utf-8") + names = _aur_depends() + + for name in AUR_RESOLVED: + assert name in names, ( + f"{name} is satisfied by the gate's stub but no longer depended" + " on; update AUR_RESOLVED in both the script and this test" + ) + assert f"'{name}'" in script, ( + f"{name} is AUR-resolved but the gate's stub does not provide it;" + " makepkg will fail on dependency verification" + ) + + +def test_the_smoke_walks_the_pinned_deps(): + """The import smoke runs the two AUR-resolved deps from the hash-pinned + export (under their PyPI names), not from whatever AUR ships, so the pin + has to exist there.""" + script = GATE_SH.read_text(encoding="utf-8") + export = RUNTIME_EXPORT.read_text(encoding="utf-8") + + assert "requirements/runtime.txt" in script + assert "--require-hashes" in script + assert "pacman -U" in script and "vocalinux --version" in script + # Arch name -> PyPI name for the export the gate installs from. + pypi_names = {"python-pywhispercpp": "pywhispercpp", "python-pynput": "pynput"} + for arch_name, pypi_name in pypi_names.items(): + assert arch_name in AUR_RESOLVED or arch_name in _aur_depends() + assert re.search(rf"^{pypi_name}==\S+ \\", export, re.M), ( + f"{pypi_name} is not pinned in requirements/runtime.txt; the smoke" + " would have nothing versioned to install" + ) + + +def test_the_pkgbuild_builds_without_isolation_on_distro_setuptools(): + """#757: the PKGBUILD builds --no-isolation against Arch's setuptools + (84 at the time), which is why the setuptools upper cap had to come off + pyproject.toml. Switching to an isolated build is a decision that + changes what the gate proves, not a slip to make in passing.""" + pkgbuild = PKGBUILD.read_text(encoding="utf-8") + assert "--no-isolation" in pkgbuild