fix(aur): gate source publication on attested sdists (#1020)
* fix(aur): gate source publication on attested sdists Prepare and verify the canonical release source, build the exact PKGBUILD on Arch, and publish the gated artifact without further rewrites. Part of #701. * fix(aur): isolate local gate and binary publication Keep local gate distributions and diagnostics in a private workspace, preserving other build outputs and failed diagnostics. Publish vocalinux-bin after its own AppImage inputs without depending on the source gate or unrelated release jobs. Add regression coverage for both review findings.
This commit is contained in:
@@ -871,9 +871,51 @@ jobs:
|
||||
with:
|
||||
skip-existing: true
|
||||
|
||||
aur-source-test:
|
||||
name: Verify AUR release source (gate)
|
||||
needs: [build-and-release, publish-checksums]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
attestations: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: '0.13.3'
|
||||
- name: Download the published sdist and checksum manifest
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
gh release download "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" \
|
||||
--pattern 'vocalinux-*.tar.gz' --pattern SHA256SUMS --dir release-source
|
||||
- name: Verify provenance and prepare the exact source PKGBUILD
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
uv run --python "$(command -v python)" --locked --only-group release-build \
|
||||
python scripts/prepare_aur_source.py \
|
||||
--sdist release-source/vocalinux-*.tar.gz --manifest release-source/SHA256SUMS \
|
||||
--tag "$GITHUB_REF_NAME" --source-commit "$GITHUB_SHA" --output build/aur-source
|
||||
- name: Build and install the exact source PKGBUILD on Arch
|
||||
run: |
|
||||
docker run --rm -v "$PWD:/repo:ro" -e AUR_SOURCE_ONLY=1 -e AUR_FETCH_SOURCE=1 \
|
||||
archlinux:latest bash /repo/packaging/aur/build-test.sh
|
||||
- name: Retain only the verified source PKGBUILD for publication
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: aur-source-pkgbuild
|
||||
path: build/aur-source/PKGBUILD
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
publish-aur:
|
||||
name: Publish to AUR
|
||||
needs: build-and-release
|
||||
needs: [build-and-release, aur-source-test]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -891,27 +933,20 @@ jobs:
|
||||
echo "configured=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Checkout code
|
||||
- name: Download the verified source PKGBUILD
|
||||
if: steps.aur.outputs.configured == 'true'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Bump PKGBUILD version
|
||||
if: steps.aur.outputs.configured == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${GITHUB_REF#refs/tags/v}"
|
||||
for PKGBUILD in packaging/aur/vocalinux/PKGBUILD packaging/aur/vocalinux-bin/PKGBUILD; do
|
||||
sed -i "s/^pkgver=.*/pkgver=${TAG//-/}/; s/^_tag=.*/_tag=${TAG}/; s/^pkgrel=.*/pkgrel=1/" "$PKGBUILD"
|
||||
grep -E '^(pkgver|_tag|pkgrel)=' "$PKGBUILD"
|
||||
done
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: aur-source-pkgbuild
|
||||
path: aur-source
|
||||
|
||||
- name: Publish AUR package
|
||||
if: steps.aur.outputs.configured == 'true'
|
||||
uses: KSXGitHub/github-actions-deploy-aur@084b0d9b15415bf9cdb65d44dad1efe37a354050 # v4.2.0
|
||||
with:
|
||||
pkgname: vocalinux
|
||||
pkgbuild: packaging/aur/vocalinux/PKGBUILD
|
||||
updpkgsums: 'true'
|
||||
pkgbuild: aur-source/PKGBUILD
|
||||
updpkgsums: 'false'
|
||||
commit_username: ${{ secrets.AUR_USERNAME }}
|
||||
commit_email: ${{ secrets.AUR_EMAIL }}
|
||||
ssh_private_key: ${{ secrets.AUR_SSH_PRIVATE_KEY }}
|
||||
@@ -919,11 +954,12 @@ jobs:
|
||||
|
||||
# Separate job: the -bin PKGBUILD pins digests of BOTH AppImages, and the
|
||||
# aarch64 build attaches its asset after build-and-release returns. Waiting
|
||||
# for it here keeps the source package publishing early while the binary
|
||||
# package never curls a 404.
|
||||
# for it here keeps the binary package from curling a 404. Do not depend on
|
||||
# source publication: its Arch gate and unrelated checksum producers must
|
||||
# not block ready AppImages.
|
||||
publish-aur-bin:
|
||||
name: Publish -bin to AUR
|
||||
needs: [publish-aur, build-appimage-arm64]
|
||||
needs: [build-and-release, build-appimage-arm64]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -67,6 +67,9 @@ jobs:
|
||||
# the whole python matrix along.
|
||||
aur:
|
||||
- 'packaging/aur/**'
|
||||
- 'MANIFEST.in'
|
||||
- 'README.md'
|
||||
- 'LICENSE'
|
||||
- '.github/workflows/unified-pipeline.yml'
|
||||
# Same carve-out for the .deb/.rpm gate: nfpm.yaml mirrors
|
||||
# pyproject.toml's dependencies against distro package names, so a
|
||||
@@ -116,6 +119,15 @@ jobs:
|
||||
version: '0.13.3'
|
||||
- name: Build and verify reproducible wheel
|
||||
run: python scripts/build_reproducible_wheel.py --python "$(command -v python)"
|
||||
- name: Share the canonical distributions with packaging gates
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: python-dist
|
||||
path: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
- name: Preserve failed wheel builds and diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
@@ -383,21 +395,35 @@ jobs:
|
||||
"${{ matrix.distro }}" \
|
||||
bash /pk/boot-test.sh "/dist/$(basename "$APPIMAGE")"
|
||||
|
||||
# Nothing built the PKGBUILD before a tag pushed it to AUR users, which is
|
||||
# how #757 shipped: Arch's setuptools 84 against our old <82 cap. The gate
|
||||
# swaps the not-yet-existing tag tarball for a git archive of the checkout,
|
||||
# so it answers whether *this commit* builds on Arch. Deliberately
|
||||
# archlinux:latest: the AUR is a rolling channel and #757 was Arch moving
|
||||
# under us.
|
||||
# Build the canonical sdist produced by this commit's wheel gate. PR sources
|
||||
# are explicitly unattested fixtures; release.yml verifies published sources
|
||||
# before passing the same prepared PKGBUILD to this Arch gate. Deliberately
|
||||
# archlinux:latest: the AUR follows Arch's current distro toolchain (#757).
|
||||
aur-build-test:
|
||||
name: 🏛️ Build AUR package (gate)
|
||||
runs-on: ubuntu-latest
|
||||
needs: changes
|
||||
needs: [changes, reproducible-wheel]
|
||||
if: needs.changes.outputs.python == 'true' || needs.changes.outputs.aur == 'true'
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: '0.13.3'
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: python-dist
|
||||
path: dist
|
||||
- name: Prepare the local unattested source fixture
|
||||
run: |
|
||||
uv run --python "$(command -v python)" --locked --only-group release-build \
|
||||
python scripts/prepare_aur_source.py \
|
||||
--local-unattested --sdist dist/vocalinux-*.tar.gz --output build/aur-source
|
||||
|
||||
- name: Build the PKGBUILD on Arch
|
||||
run: |
|
||||
docker run --rm \
|
||||
|
||||
@@ -91,7 +91,7 @@ just appimage # build the AppImage in its pinned base image (needs docker)
|
||||
just appimage-boot fedora:42 # boot that AppImage in a distro container
|
||||
just native-packages # build the .deb and .rpm in the pinned base image (needs docker)
|
||||
just native-smoke fedora:42 # install the built package in a distro container
|
||||
just aur-gate # build the AUR PKGBUILD on current Arch (needs docker)
|
||||
just aur-gate # build canonical sdist + unchanged AUR fixture on current Arch (needs uv/docker)
|
||||
just install-gate debian:12 # run install.sh unattended in a distro container
|
||||
just remote-install-gate debian:12 # remote curl|bash path end to end against HEAD (needs docker)
|
||||
just verify-release # check a published release as published (needs gh)
|
||||
@@ -127,6 +127,7 @@ Use **uv 0.13.3** for `just lock` and `just export-check`, matching CI. Other ve
|
||||
| pywhispercpp | Pinned in `install.sh` as `PYWHISPERCPP_VERSION` (keep in sync with `uv.lock`) |
|
||||
| Installer build tools | `[dependency-groups].installer-build` exports to `requirements/installer-build.txt`. Bootstrap uses hashed wheels on new and reused venvs; source builds use these tools without build isolation. Its setuptools `>=83` floor keeps the pinned installer tools patched; do not add it to `[build-system]` (Arch). Backend rebuilds select pywhispercpp's complete hash block from `runtime.txt` via `scripts/installer_requirements.py`, with `--no-deps` so they cannot replace the runtime |
|
||||
| Release build tools | `[dependency-groups].release-build` exports to `requirements/release-build.txt`. `scripts/build_reproducible_wheel.py` builds in two fresh source trees/venvs with hashed tools and `--no-isolation`, compares wheel bytes, and stages only canonical distributions in `dist/`. These build-only venvs exclude system site-packages: host setuptools plugins must not influence locked builds, and packaging does not import GTK. Repeat runs clean recognized diagnostics and staged archives matching retained canonical digests; staged copies are deleted first so interrupted cleanup remains retryable. Unrelated/modified files are preserved and block cleanup. Never wipe all of `build/`. PR and release CI use Python 3.11 and pinned uv. Failed build outputs/logs stay under `build/reproducible-wheel/`, excluded from release artifacts |
|
||||
| AUR source publication | Use the canonical release sdist, not a GitHub tag archive. `scripts/prepare_aur_source.py` (run with the locked `release-build` group) requires its published checksum and provenance from the exact release workflow/tag/commit before preparing a pinned PKGBUILD. Release CI builds/installs that unchanged recipe on current Arch; publication consumes only the gated artifact with `updpkgsums: false`. PR/local fixtures must explicitly use `--local-unattested` and are never publication inputs. `just aur-gate` keeps distributions, diagnostics and tooling in its own temporary directory, leaving shared packaging outputs and dev environments untouched; failures retain that directory and print its path. `vocalinux-bin` publication depends only on release creation and both AppImages, not the source gate or global checksums. Existing AUR provider stubs/pinned import smoke remain limited as documented in `docs/AUR.md` |
|
||||
| Installer Whisper export | Compiled universally from Python 3.11, constrained by `runtime.txt` and `installer-build.txt` to keep shared pins consistent. Uses a `+cpu` torch pin and best-match index resolution so the CPU index's older copies of shared packages do not downgrade the runtime |
|
||||
| Remote installer | After cloning, hand off to the tagged `install.sh` with the original arguments and remote venv path. The installer and its exports must come from the same revision, including when the tag predates the pinned installer |
|
||||
| `[vad]` extra | `onnxruntime` for Silero VAD |
|
||||
@@ -240,7 +241,7 @@ Site-only work: `web/AGENTS.md` (commands, layout map). Product claims: `web/PRO
|
||||
|
||||
## Runtime (any Linux dev machine)
|
||||
|
||||
- Create runtime/dev venvs with `--system-site-packages`. Build-only venvs in `scripts/build_reproducible_wheel.py` are the exception: exclude host packages to keep the release toolchain isolated
|
||||
- Create runtime/dev venvs with `--system-site-packages`. Build-only venvs for `scripts/build_reproducible_wheel.py` and `just aur-gate` preparation are the exception: exclude host packages to keep the locked release tools isolated; never prune the developer environment to run a packaging gate
|
||||
- Kill the app by PID, never `pkill -f vocalinux` (matches editors, shells, and cwd paths)
|
||||
- Stale instance: `$XDG_DATA_HOME/vocalinux/instance.lock` (default `~/.local/share/vocalinux/instance.lock`)
|
||||
- Default dictation shortcut: hold Right Alt (push-to-talk). Existing `~/.config/vocalinux/config.json` wins
|
||||
|
||||
+67
-8
@@ -3,7 +3,7 @@
|
||||
## Users
|
||||
|
||||
```bash
|
||||
yay -S vocalinux # latest release tag
|
||||
yay -S vocalinux # canonical release sdist, built against Arch dependencies
|
||||
yay -S vocalinux-git # community package; tracks main
|
||||
```
|
||||
|
||||
@@ -11,13 +11,16 @@ yay -S vocalinux-git # community package; tracks main
|
||||
|
||||
`packaging/aur/vocalinux-bin/` builds a `-bin` variant that installs the
|
||||
prebuilt AppImage from the release — no Python, engine, or GPU dependency
|
||||
builds, since the bundle carries them (#817). The release workflow publishes
|
||||
it alongside the source package: it first pins the per-arch AppImage digests
|
||||
in the PKGBUILD, then pushes to AUR in the `Publish AUR -bin package` step.
|
||||
builds, since the bundle carries them (#817). Its publication waits only for
|
||||
release creation and both AppImages, not the source-package gate or other
|
||||
packaging jobs. It first pins the per-arch AppImage digests in the PKGBUILD,
|
||||
then pushes to AUR in the `Publish AUR -bin package` step.
|
||||
|
||||
## Maintainers (one-time setup)
|
||||
|
||||
Release tags auto-push `packaging/aur/vocalinux/PKGBUILD` when these secrets exist:
|
||||
Release tags prepare and test the source PKGBUILD from
|
||||
`packaging/aur/vocalinux/PKGBUILD`, then auto-push the verified file when these
|
||||
secrets exist:
|
||||
|
||||
| Secret | Value |
|
||||
|--------|--------|
|
||||
@@ -38,11 +41,67 @@ git clone ssh://aur@aur.archlinux.org/vocalinux.git
|
||||
|
||||
Sources: `packaging/aur/vocalinux/PKGBUILD`. Community git package: https://aur.archlinux.org/packages/vocalinux-git
|
||||
|
||||
## CI gate
|
||||
## Source archive and trust model
|
||||
|
||||
Every PR that changes the Python source or `packaging/aur/**` builds the PKGBUILD on `archlinux:latest` — the `Build AUR package (gate)` job in `unified-pipeline.yml`. Run it locally with `just aur-gate` (needs docker).
|
||||
The source package consumes the **canonical sdist attached to the GitHub release**,
|
||||
the same bytes published to PyPI, not GitHub's automatically generated tag archive.
|
||||
The sdist is produced by the locked reproducible-wheel gate. Only wheels are
|
||||
compared for reproducibility; sdists are built and validated, not compared.
|
||||
|
||||
The gate swaps the tag-archive `source=` for a `git archive` of the checkout, so it answers whether *this commit* builds on Arch — #757 reached users after the tag, when nothing could fail a build anymore (#736 reached Arch users too, but through `install.sh`; this package built fine). `python-pywhispercpp` (virtual, provided by the AUR backends) and `python-pynput` (AUR) cannot come from the repos, so the gate satisfies them with a stub for dependency resolution and pip-installs the pinned versions from `requirements/runtime.txt` for the import smoke. What AUR actually ships (python-pywhispercpp-cpu is still 1.4.x — see Troubleshooting) is a monitoring question, not the gate's.
|
||||
`scripts/prepare_aur_source.py` verifies the downloaded sdist against the published
|
||||
`SHA256SUMS` and runs `gh attestation verify`, requiring the VocaHQ/vocalinux
|
||||
repository, `.github/workflows/release.yml` signer, exact tag ref, and expected
|
||||
source commit. Missing hashes, mismatches, rejected provenance, or mismatched
|
||||
tag/package versions stop preparation. It preserves the original archive bytes
|
||||
and pins their digest in the prepared PKGBUILD. `_tag` retains the release tag;
|
||||
`_version` is the PEP 440-normalized sdist version, including for prereleases.
|
||||
|
||||
This verification is a **maintainer publication gate**. An AUR user's `makepkg`
|
||||
checks the pinned SHA-256; it does not independently verify GitHub provenance.
|
||||
Arch still owns the dependency/toolchain versions: this does not claim that AUR
|
||||
builds use our Python lock or reproduce the PyPI wheel byte-for-byte.
|
||||
|
||||
## CI gates
|
||||
|
||||
### PR and local source gate
|
||||
|
||||
Python/AUR changes and changes to the source manifest, README or license build the
|
||||
canonical sdist from the current checkout's reproducible-wheel gate, then prepare
|
||||
an explicitly `--local-unattested` fixture. The `Build AUR package (gate)` job in
|
||||
`unified-pipeline.yml` builds and installs its PKGBUILD on `archlinux:latest`.
|
||||
Run `just aur-gate` locally (needs uv and Docker); it includes the reproducible-wheel
|
||||
build and uses a fresh temporary directory for the distributions, diagnostics,
|
||||
tooling and prepared fixture. Existing `dist/`, `build/reproducible-wheel/` and
|
||||
the developer environment are untouched, so other packaging commands can run
|
||||
before or after it. Successful runs clean the temporary directory; failed runs
|
||||
retain it and print its path for diagnosis.
|
||||
|
||||
### Release gate
|
||||
|
||||
After the release checksum/provenance job succeeds, `aur-source-test` downloads
|
||||
the **published** sdist and manifest, verifies them, and prepares the PKGBUILD.
|
||||
The same Arch gate makes `makepkg` download the actual source URL afresh
|
||||
(`AUR_FETCH_SOURCE=1`) and verify its checksum, runs `namcap` on the recipe and
|
||||
package, installs the package, checks the installed distribution version, and
|
||||
runs the import smoke. It does not rewrite the source URL or disable source
|
||||
checksum checks. The tested recipe and downloaded archive must match the
|
||||
prepared PKGBUILD and provenance-verified sdist byte-for-byte.
|
||||
|
||||
Only after this succeeds is `aur-source-pkgbuild` uploaded for `publish-aur`.
|
||||
Publication downloads that exact file; there is no version rewrite or
|
||||
`updpkgsums` afterwards. The gate runs even without AUR credentials; only the
|
||||
final push is secret-gated. This intentionally makes source publication wait
|
||||
for the release's checksums and attestations. `vocalinux-bin` uses a separate
|
||||
AppImage publication path independent of this source gate and the global
|
||||
checksum job; its PR smoke still uses a stub AppImage.
|
||||
|
||||
Both gates deliberately use rolling `archlinux:latest`: #757 was Arch's
|
||||
setuptools moving underneath us. `python-pywhispercpp` (virtual, provided by the
|
||||
AUR backends) and `python-pynput` (AUR) cannot come from the repos, so the gate
|
||||
satisfies them with a stub for dependency resolution and pip-installs their
|
||||
pinned versions from `requirements/runtime.txt` for the import smoke. It does
|
||||
not test those AUR providers or prove dependency completeness in a minimal host.
|
||||
What those providers actually ship remains a separate monitoring question.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
||||
+19
-1
@@ -129,6 +129,24 @@ active run fails without changing the first run's outputs. Archives are copied t
|
||||
an ignored temporary sibling of `dist/` and then atomically renamed into place,
|
||||
so interrupted staging leaves only complete archives in `dist/` and can be retried.
|
||||
|
||||
### AUR source publication gate
|
||||
|
||||
After `publish-checksums`, `aur-source-test` downloads the published canonical
|
||||
sdist and `SHA256SUMS`. `scripts/prepare_aur_source.py` checks the digest and
|
||||
cryptographically verifies provenance against this repository's release workflow,
|
||||
tag ref and source commit before preparing the checksum-pinned PKGBUILD.
|
||||
The exact prepared recipe is built, checked with namcap, installed and smoked on
|
||||
current Arch. `publish-aur` downloads only that verified PKGBUILD artifact and
|
||||
pushes it without further version/checksum rewrites (`updpkgsums: false`).
|
||||
|
||||
This gate runs even without AUR publication credentials. Failure prevents the
|
||||
AUR push, but does not unpublish GitHub/PyPI artifacts already released. It is
|
||||
separate from PR/local `--local-unattested` fixtures, which exercise the same
|
||||
preparation/build path but are never used for publication. See [AUR.md](AUR.md)
|
||||
for the trust model and the existing AUR-provider smoke-test limitations.
|
||||
`vocalinux-bin` publication waits only for release creation and both AppImages;
|
||||
source-gate failures or unrelated checksum producers do not block it.
|
||||
|
||||
## Detailed Release Steps
|
||||
|
||||
### Step 1: Determine Version Number
|
||||
@@ -183,7 +201,7 @@ When shipping bug fixes only (PATCH bump, same MINOR):
|
||||
- **Website changelog** (`web/src/app/changelog/page.tsx`) may list only the delta for the new entry; older entries stay historical.
|
||||
- **AppStream** (`packaging/flatpak/com.vocalinux.Vocalinux.metainfo.xml`): add a short `<release>` note for the patch delta only; leave the long product feature `<ul>` as series/product-level.
|
||||
- **SECURITY.md** supported versions table usually stays on the minor line (`0.14.x`) without a row per patch.
|
||||
- **AUR PKGBUILD**: bump `pkgver` / `_tag`; leave `sha256sums=('SKIP')` until the tag tarball exists. The release workflow runs `updpkgsums` when publishing to the AUR.
|
||||
- **AUR source PKGBUILD**: the release gate prepares `pkgver`, `_tag`, the normalized `_version`, and the canonical sdist digest, then tests that exact file. Do not introduce `SKIP` or rehash it during publication. The separate `vocalinux-bin` path still pins both AppImage digests before its own publication.
|
||||
- **Dates**: patch release dates must not predate the previous release in changelog / AppStream / version history.
|
||||
|
||||
### GitHub Release Notes Writing Rules
|
||||
|
||||
@@ -127,29 +127,38 @@ native-packages: build
|
||||
native-smoke distro="debian:12":
|
||||
docker run --rm -v "$PWD/dist:/out:ro" -v "$PWD/packaging/native:/pk:ro" {{distro}} bash /pk/smoke-test.sh
|
||||
|
||||
# Build the AUR package from this checkout on current Arch, as the CI gate
|
||||
# does. Answers "does this commit build on Arch" — the tag tarball source= is
|
||||
# swapped for a git archive of HEAD (needs docker).
|
||||
#
|
||||
# The checkout is mounted at its own path, and the main .git with it when this
|
||||
# is a linked worktree: a worktree's .git is a pointer to a gitdir outside the
|
||||
# tree, so git archive inside the container cannot see the objects without it.
|
||||
# Build the canonical sdist, prepare an explicitly unattested local PKGBUILD,
|
||||
# then build/install it unchanged on current Arch (needs uv and docker).
|
||||
# Keep distributions and diagnostics away from other packaging commands' outputs.
|
||||
aur-gate:
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
COMMON="$(cd "$(git rev-parse --git-common-dir)" && pwd)"
|
||||
ARGS=(--rm -v "$PWD:$PWD:ro" -e REPO="$PWD")
|
||||
if [ "$COMMON" != "$PWD/.git" ]; then
|
||||
ARGS+=(-v "$COMMON:$COMMON:ro")
|
||||
fi
|
||||
docker run "${ARGS[@]}" archlinux:latest bash "$PWD/packaging/aur/build-test.sh"
|
||||
WORK="$(mktemp -d)"
|
||||
cleanup() {
|
||||
local status=$?
|
||||
if [ "$status" -eq 0 ]; then
|
||||
rm -rf "$WORK"
|
||||
else
|
||||
echo "AUR gate failed; diagnostics retained in $WORK" >&2
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
python3 scripts/build_reproducible_wheel.py --python "${PYTHON:-python3}" \
|
||||
--output "$WORK/dist" --diagnostics "$WORK/diagnostics"
|
||||
uv venv --no-config --python "${PYTHON:-python3}" "$WORK/venv"
|
||||
uv pip install --no-config --python "$WORK/venv/bin/python" \
|
||||
--require-hashes --no-deps --only-binary=:all: -r requirements/release-build.txt
|
||||
"$WORK/venv/bin/python" scripts/prepare_aur_source.py \
|
||||
--local-unattested --sdist "$WORK"/dist/vocalinux-*.tar.gz --output "$WORK/source"
|
||||
docker run --rm -v "$PWD:/repo:ro" -v "$WORK/source:/aur-source:ro" \
|
||||
-e AUR_SOURCE=/aur-source archlinux:latest bash /repo/packaging/aur/build-test.sh
|
||||
|
||||
# Run install.sh unattended in a distro container, as the CI gate does. Answers
|
||||
# "does this commit install" (needs docker).
|
||||
#
|
||||
# The tree is taken via git archive of HEAD and extracted inside the container,
|
||||
# so the venv local mode creates lands in the container's copy rather than in
|
||||
# your working tree. Same worktree handling as aur-gate above.
|
||||
# your working tree. Mount the common gitdir as well for linked worktrees.
|
||||
#
|
||||
# Usage: `just install-gate` for debian:12, or `just install-gate fedora:42`
|
||||
install-gate distro="debian:12":
|
||||
|
||||
+31
-24
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the AUR package from this checkout, on Arch as it is today.
|
||||
# Build a prepared, checksum-pinned AUR source package on current Arch.
|
||||
#
|
||||
# Usage (the read-only repo mount is enough — the tree is taken via git archive):
|
||||
# Usage (first prepare build/aur-source with scripts/prepare_aur_source.py):
|
||||
# docker run --rm -v "$PWD:/repo:ro" archlinux:latest \
|
||||
# bash /repo/packaging/aur/build-test.sh
|
||||
# or: just aur-gate
|
||||
@@ -10,10 +10,11 @@
|
||||
# build-system.requires still capped it below 82, so the PKGBUILD's
|
||||
# --no-isolation build broke for AUR users — and nothing ever ran makepkg on
|
||||
# the PKGBUILD before a tag published it.
|
||||
# The published source= points at the tag tarball, which does not exist until
|
||||
# the tag is pushed, so the gate replaces it with a git archive of HEAD under
|
||||
# the same filename: build()/package() run unchanged, and the gate answers
|
||||
# "does this commit build on Arch", not "did the last tag build".
|
||||
# Both PR and release gates consume the canonical sdist and fully prepared
|
||||
# PKGBUILD without rewriting source=, versions or checksums. PR fixtures are
|
||||
# explicitly unattested; the release job verifies published bytes and provenance
|
||||
# before running this gate. AUR_SOURCE_ONLY=1 leaves the existing -bin smoke out;
|
||||
# AUR_FETCH_SOURCE=1 makes makepkg download the real published source URL afresh.
|
||||
#
|
||||
# Not this gate's, despite reaching Arch users: #736 came through install.sh,
|
||||
# not through this package. The AUR build of v0.16.0 was intact, because the
|
||||
@@ -40,7 +41,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
REPO="${REPO:-/repo}"
|
||||
PKGDIR="$REPO/packaging/aur/vocalinux"
|
||||
PKGDIR="${AUR_SOURCE:-$REPO/build/aur-source}"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
@@ -71,13 +72,6 @@ pacman_retry() {
|
||||
# detecting. It is also what "Arch as it is today" has to mean here.
|
||||
pacman_retry -Syu --noconfirm >/dev/null
|
||||
|
||||
# The base image ships no git; the tree for makepkg comes from git archive.
|
||||
# safe.directory: the mount is owned by another uid than the container user.
|
||||
pacman_retry -S --needed --noconfirm git >/dev/null
|
||||
git config --global --add safe.directory '*'
|
||||
git -C "$REPO" rev-parse --verify HEAD >/dev/null \
|
||||
|| fail "$REPO is not a git checkout; git archive needs it"
|
||||
|
||||
# Sourcing gives us pkgname, _tag and the depends/makedepends arrays the same
|
||||
# way makepkg itself reads them.
|
||||
# shellcheck source=/dev/null
|
||||
@@ -102,7 +96,7 @@ done
|
||||
# package (how a python-pywhispercpp-cpu rename would go unnoticed again).
|
||||
# base-devel is the documented prerequisite for building any AUR package
|
||||
# (fakeroot, debugedit and friends); makepkg aborts without it.
|
||||
pacman_retry -S --needed --noconfirm --asdeps base-devel namcap python-pip \
|
||||
pacman_retry -S --needed --noconfirm --asdeps base-devel namcap python-pip curl \
|
||||
"${repo_deps[@]}" >/dev/null
|
||||
|
||||
WORKDIR="$(mktemp -d)"
|
||||
@@ -122,17 +116,17 @@ arch=('any')
|
||||
provides=('python-pywhispercpp' 'python-pynput')
|
||||
EOF
|
||||
|
||||
echo "== Replace source= with a git archive of this checkout =="
|
||||
echo "== Copy the prepared PKGBUILD and original sdist without rewriting =="
|
||||
BUILD="$WORKDIR/pkg"
|
||||
mkdir -p "$BUILD"
|
||||
# Same filename the published source= uses, and the same directory prefix
|
||||
# build()/package() cd into — only where the tarball comes from changes.
|
||||
git -C "$REPO" archive --prefix="${pkgname}-${_tag}/" -o "$BUILD/${pkgname}-${_tag}.tar.gz" HEAD
|
||||
# makepkg consumes the cached archive under the same filename as source= and
|
||||
# still verifies its real checksum. No SKIP or --skipchecksums for source builds.
|
||||
cp "$PKGDIR/PKGBUILD" "$BUILD/PKGBUILD"
|
||||
sed -i "s|^source=.*|source=(\"${pkgname}-${_tag}.tar.gz\")|" "$BUILD/PKGBUILD"
|
||||
# sha256sums=('SKIP') stays: the real digest is computed at publish time by
|
||||
# release.yml (updpkgsums), and the archive differs from the tag tarball by
|
||||
# design.
|
||||
if [ "${AUR_FETCH_SOURCE:-0}" = 1 ]; then
|
||||
echo " release gate: makepkg must download the published source URL afresh"
|
||||
else
|
||||
cp "$PKGDIR/${pkgname}-${_version}.tar.gz" "$BUILD/"
|
||||
fi
|
||||
|
||||
# makepkg refuses to run as root; a build user is the container-standard fix.
|
||||
useradd -m builder
|
||||
@@ -176,7 +170,12 @@ pacman -U --noconfirm "$BUILD"/*.pkg.tar.* >/dev/null
|
||||
# every depends entry the package declares (gi/Gtk/AppIndicator, IBus, cairo,
|
||||
# pyaudio, numpy, requests, tqdm, psutil, evdev, xlib, pynput).
|
||||
vocalinux --version
|
||||
python - <<'PY' || fail "the installed package does not import"
|
||||
python - "$_version" <<'PY' || fail "the installed package does not import or has the wrong version"
|
||||
import sys
|
||||
from importlib.metadata import version
|
||||
|
||||
assert version("vocalinux") == sys.argv[1], "installed distribution version differs from the sdist"
|
||||
|
||||
import vocalinux.main
|
||||
import vocalinux.ui.tray_indicator
|
||||
import vocalinux.speech_recognition.recognition_manager
|
||||
@@ -189,6 +188,14 @@ import pywhispercpp.model
|
||||
print(" main, tray, recognition, injection and keyboard backends import")
|
||||
PY
|
||||
|
||||
cmp -s "$PKGDIR/PKGBUILD" "$BUILD/PKGBUILD" || fail "the tested PKGBUILD was modified"
|
||||
cmp -s "$PKGDIR/${pkgname}-${_version}.tar.gz" "$BUILD/${pkgname}-${_version}.tar.gz" \
|
||||
|| fail "the tested source archive was modified"
|
||||
echo "PASS: the exact prepared source PKGBUILD builds and installs on current Arch"
|
||||
if [ "${AUR_SOURCE_ONLY:-0}" = 1 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "== Build vocalinux-bin against a stub AppImage =="
|
||||
# The -bin package's arch sources are the published release assets, which do
|
||||
# not exist for an untagged HEAD. A placeholder exercises the same package()
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
|
||||
pkgname=vocalinux
|
||||
# AUR pkgver cannot contain hyphens (v0.14.0-beta -> 0.14.0beta).
|
||||
pkgver=0.18.1
|
||||
_tag=0.18.1
|
||||
pkgver=0.18.4
|
||||
_tag=0.18.4
|
||||
# PEP 440-normalized sdist version may differ from a prerelease tag.
|
||||
_version=0.18.4
|
||||
pkgrel=1
|
||||
pkgdesc="Free, offline voice dictation for Linux"
|
||||
arch=('any')
|
||||
@@ -56,17 +58,18 @@ optdepends=(
|
||||
'wtype: Wayland keystroke injection fallback'
|
||||
)
|
||||
conflicts=('vocalinux-git')
|
||||
source=("${pkgname}-${_tag}.tar.gz::https://github.com/VocaHQ/vocalinux/archive/refs/tags/v${_tag}.tar.gz")
|
||||
# SKIP until the v${_tag} tarball exists. release.yml sets updpkgsums=true on publish.
|
||||
sha256sums=('SKIP')
|
||||
source=("${pkgname}-${_version}.tar.gz::https://github.com/VocaHQ/vocalinux/releases/download/v${_tag}/${pkgname}-${_version}.tar.gz")
|
||||
# Canonical release sdist, also published on PyPI. prepare_aur_source.py verifies
|
||||
# its checksum/provenance before pinning the next version; publication never rehashes.
|
||||
sha256sums=('f14ae90e37b13b86c6aa2153ba5aa8a2aa62d65b93082344695f54807924e243')
|
||||
|
||||
build() {
|
||||
cd "${pkgname}-${_tag}"
|
||||
cd "${pkgname}-${_version}"
|
||||
python -m build --wheel --no-isolation
|
||||
}
|
||||
|
||||
package() {
|
||||
cd "${pkgname}-${_tag}"
|
||||
cd "${pkgname}-${_version}"
|
||||
python -m installer --destdir="${pkgdir}" dist/*.whl
|
||||
|
||||
install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare the exact AUR source inputs, verifying published release provenance first.
|
||||
|
||||
Run with the locked release-build dependency group (provides packaging). Release
|
||||
mode requires SHA256SUMS and the expected source commit. --local-unattested is
|
||||
only for PR/local fixtures built by the reproducible-wheel gate, never publishing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from email.parser import BytesParser
|
||||
from pathlib import Path
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TEMPLATE = ROOT / "packaging/aur/vocalinux/PKGBUILD"
|
||||
REPOSITORY = "VocaHQ/vocalinux"
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
"""Hash the original archive bytes, without extracting or repacking them."""
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def sdist_version(sdist: Path) -> str:
|
||||
"""Require one canonical Vocalinux sdist with matching top-level metadata."""
|
||||
match = re.fullmatch(r"vocalinux-([0-9][A-Za-z0-9.+!]*)\.tar\.gz", sdist.name)
|
||||
if not match:
|
||||
raise ValueError("expected a canonical vocalinux-VERSION.tar.gz sdist")
|
||||
version = match[1]
|
||||
if str(Version(version)) != version:
|
||||
raise ValueError("sdist filename version is not canonical")
|
||||
root = f"vocalinux-{version}"
|
||||
with tarfile.open(sdist, "r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
if any(
|
||||
member.name.split("/", 1)[0] != root or ".." in member.name.split("/")
|
||||
for member in members
|
||||
):
|
||||
raise ValueError("sdist has an unexpected archive root or unsafe member path")
|
||||
metadata = [member for member in members if member.name == f"{root}/PKG-INFO"]
|
||||
if len(metadata) != 1 or not metadata[0].isfile():
|
||||
raise ValueError("sdist must contain exactly one regular top-level PKG-INFO")
|
||||
stream = archive.extractfile(metadata[0])
|
||||
assert stream is not None
|
||||
with stream:
|
||||
info = BytesParser().parse(stream)
|
||||
if info.get_all("Name") != ["vocalinux"] or info.get_all("Version") != [version]:
|
||||
raise ValueError("sdist metadata does not match its filename")
|
||||
return version
|
||||
|
||||
|
||||
def verify_release(sdist: Path, manifest: Path, tag: str, commit: str) -> str:
|
||||
"""Fail closed on missing/mismatched hashes or untrusted release provenance."""
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", commit):
|
||||
raise ValueError("release verification requires a full source commit SHA")
|
||||
entries = []
|
||||
for line in manifest.read_text(encoding="utf-8").splitlines():
|
||||
match = re.fullmatch(r"([0-9a-f]{64}) (\S+)", line)
|
||||
if not match:
|
||||
raise ValueError("malformed SHA256SUMS entry")
|
||||
if match[2] == sdist.name:
|
||||
entries.append(match[1])
|
||||
if len(entries) != 1:
|
||||
raise ValueError("SHA256SUMS must pin the sdist exactly once")
|
||||
if digest(sdist) != entries[0]:
|
||||
raise ValueError("sdist checksum does not match SHA256SUMS")
|
||||
subprocess.run(
|
||||
[
|
||||
"gh",
|
||||
"attestation",
|
||||
"verify",
|
||||
str(sdist),
|
||||
"--repo",
|
||||
REPOSITORY,
|
||||
"--signer-workflow",
|
||||
f"{REPOSITORY}/.github/workflows/release.yml",
|
||||
"--source-ref",
|
||||
f"refs/tags/{tag}",
|
||||
"--source-digest",
|
||||
commit,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
return entries[0]
|
||||
|
||||
|
||||
def prepare(
|
||||
sdist: Path,
|
||||
output: Path,
|
||||
tag: str | None = None,
|
||||
manifest: Path | None = None,
|
||||
commit: str | None = None,
|
||||
local_unattested: bool = False,
|
||||
) -> None:
|
||||
"""Stage an unchanged sdist and its fully pinned PKGBUILD only after validation."""
|
||||
version = sdist_version(sdist)
|
||||
tag = tag or (f"v{version}" if local_unattested else "")
|
||||
if not re.fullmatch(r"v[0-9][A-Za-z0-9.+!-]*", tag) or Version(tag[1:]) != Version(version):
|
||||
raise ValueError("release tag and sdist version do not match")
|
||||
if local_unattested:
|
||||
if manifest is not None or commit is not None:
|
||||
raise ValueError("local-unattested mode cannot take release verification inputs")
|
||||
print("UNATTESTED local/PR fixture: not authorized for AUR publication")
|
||||
checksum = digest(sdist)
|
||||
else:
|
||||
if manifest is None or commit is None:
|
||||
raise ValueError("release mode requires --manifest and --source-commit")
|
||||
checksum = verify_release(sdist, manifest, tag, commit)
|
||||
|
||||
text = TEMPLATE.read_text(encoding="utf-8")
|
||||
for key, value in {
|
||||
"pkgver": tag[1:].replace("-", ""),
|
||||
"_tag": tag[1:],
|
||||
"_version": version,
|
||||
"pkgrel": "1",
|
||||
"sha256sums": f"('{checksum}')",
|
||||
}.items():
|
||||
text, count = re.subn(rf"^{key}=.*$", f"{key}={value}", text, flags=re.MULTILINE)
|
||||
if count != 1:
|
||||
raise ValueError(f"PKGBUILD template must assign {key} exactly once")
|
||||
if output.exists() or output.is_symlink():
|
||||
raise ValueError(f"output already exists; choose a fresh directory: {output}")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix=".aur-source-", dir=output.parent) as temporary:
|
||||
staging = Path(temporary) / "source"
|
||||
staging.mkdir()
|
||||
shutil.copyfile(sdist, staging / sdist.name)
|
||||
if digest(staging / sdist.name) != checksum:
|
||||
raise ValueError("sdist changed while staging")
|
||||
(staging / "PKGBUILD").write_text(text, encoding="utf-8")
|
||||
staging.rename(output)
|
||||
print(f"Prepared {tag}: {checksum} {sdist.name} in {output}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Prepare verified release inputs or an explicitly unattested local fixture."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--sdist", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--tag", help="exact v* tag; defaults to the sdist version in local mode")
|
||||
parser.add_argument("--manifest", type=Path, help="published SHA256SUMS")
|
||||
parser.add_argument("--source-commit", help="expected full commit SHA of the release tag")
|
||||
parser.add_argument("--local-unattested", action="store_true", help="PR/local fixture only")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
prepare(
|
||||
args.sdist,
|
||||
args.output,
|
||||
args.tag,
|
||||
args.manifest,
|
||||
args.source_commit,
|
||||
args.local_unattested,
|
||||
)
|
||||
except (OSError, ValueError, tarfile.TarError, subprocess.CalledProcessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Exercise local AUR gate orchestration without network access or Docker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def gate_recipe() -> str:
|
||||
"""Read the real shell recipe, rejecting prerequisites that use shared outputs."""
|
||||
match = re.search(
|
||||
r"^aur-gate:([^\n]*)\n((?:[ \t].*\n|\n)+)",
|
||||
(ROOT / "justfile").read_text(),
|
||||
re.MULTILINE,
|
||||
)
|
||||
assert match, "aur-gate recipe missing"
|
||||
assert not match[1].strip(), "aur-gate must not invoke a shared-output prerequisite"
|
||||
return textwrap.dedent(match[2])
|
||||
|
||||
|
||||
def executable(path: Path, content: str) -> None:
|
||||
"""Write a Python stand-in for an external command."""
|
||||
path.write_text(f"#!{sys.executable}\n" + textwrap.dedent(content))
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["", "build", "arch"])
|
||||
def test_local_gate_isolates_outputs_and_retains_failed_diagnostics(
|
||||
tmp_path: Path, failure: str
|
||||
) -> None:
|
||||
"""Use real output validation and source preparation, stubbing only tool work."""
|
||||
recipe = gate_recipe()
|
||||
checkout = tmp_path / "checkout"
|
||||
scripts = checkout / "scripts"
|
||||
scripts.mkdir(parents=True)
|
||||
template = checkout / "packaging/aur/vocalinux"
|
||||
template.mkdir(parents=True)
|
||||
shutil.copy2(ROOT / "packaging/aur/vocalinux/PKGBUILD", template / "PKGBUILD")
|
||||
shutil.copy2(ROOT / "scripts/prepare_aur_source.py", scripts / "prepare_aur_source.py")
|
||||
|
||||
# Keep the real cleanup policy: shared dist/ would reject these outputs.
|
||||
(scripts / "build_reproducible_wheel.py").write_text(textwrap.dedent(f"""\
|
||||
import argparse
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--python', required=True)
|
||||
parser.add_argument('--output', type=Path, default=Path('dist'))
|
||||
parser.add_argument('--diagnostics', type=Path, default=Path('build/reproducible-wheel'))
|
||||
args = parser.parse_args()
|
||||
assert args.python == os.environ['PYTHON']
|
||||
with open(os.environ['AUR_GATE_TEST_LOG'], 'a') as log:
|
||||
log.write(json.dumps(['build', str(args.output), str(args.diagnostics)]) + '\\n')
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
'wheel_gate', {str(ROOT / 'scripts/build_reproducible_wheel.py')!r}
|
||||
)
|
||||
gate = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(gate)
|
||||
gate.prepare_outputs(args.output, args.diagnostics)
|
||||
(args.diagnostics / 'canonical.log').write_text('retained build diagnostics')
|
||||
if os.environ['AUR_GATE_TEST_FAILURE'] == 'build':
|
||||
sys.exit(31)
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(args.output / 'vocalinux-0.18.4.tar.gz', 'w:gz') as archive:
|
||||
metadata = b'Name: vocalinux\\nVersion: 0.18.4\\n'
|
||||
member = tarfile.TarInfo('vocalinux-0.18.4/PKG-INFO')
|
||||
member.size = len(metadata)
|
||||
archive.addfile(member, io.BytesIO(metadata))
|
||||
(args.output / 'vocalinux-0.18.4-py3-none-any.whl').write_text('new wheel')
|
||||
"""))
|
||||
|
||||
# Existing packaging outputs, wheel diagnostics and dev tooling are not ours.
|
||||
protected = {}
|
||||
for name in (
|
||||
"dist/vocalinux-0.18.4-py3-none-any.whl",
|
||||
"dist/vocalinux-0.18.4.tar.gz",
|
||||
"dist/Vocalinux-0.18.4-x86_64.AppImage",
|
||||
"dist/vocalinux_0.18.4_amd64.deb",
|
||||
"dist/vocalinux-0.18.4.x86_64.rpm",
|
||||
"build/reproducible-wheel/canonical.log",
|
||||
".venv/keep.txt",
|
||||
):
|
||||
path = checkout / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
protected[path] = f"existing {name}".encode()
|
||||
path.write_bytes(protected[path])
|
||||
|
||||
tools = tmp_path / "tools"
|
||||
tools.mkdir()
|
||||
interpreter = "#!/bin/sh\nexec " + shlex.quote(sys.executable) + ' "$@"\n'
|
||||
executable(
|
||||
tools / "uv",
|
||||
f"""\
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if sys.argv[1] == 'venv':
|
||||
target = Path(sys.argv[-1]) / 'bin/python'
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text({interpreter!r})
|
||||
target.chmod(0o755)
|
||||
else:
|
||||
assert sys.argv[1:3] == ['pip', 'install']
|
||||
assert {{'--require-hashes', '--no-deps', '--only-binary=:all:'}} <= set(sys.argv)
|
||||
assert sys.argv[-1] == 'requirements/release-build.txt'
|
||||
""",
|
||||
)
|
||||
executable(
|
||||
tools / "docker",
|
||||
"""\
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
source = next(arg.removesuffix(':/aur-source:ro') for arg in sys.argv
|
||||
if arg.endswith(':/aur-source:ro'))
|
||||
assert (Path(source) / 'PKGBUILD').is_file()
|
||||
assert (Path(source) / 'vocalinux-0.18.4.tar.gz').is_file()
|
||||
assert 'AUR_SOURCE=/aur-source' in sys.argv
|
||||
with open(os.environ['AUR_GATE_TEST_LOG'], 'a') as log:
|
||||
log.write(json.dumps(['arch', source]) + '\\n')
|
||||
sys.exit(42 if os.environ['AUR_GATE_TEST_FAILURE'] == 'arch' else 0)
|
||||
""",
|
||||
)
|
||||
temporary = tmp_path / "temporary"
|
||||
temporary.mkdir()
|
||||
log = tmp_path / "calls.jsonl"
|
||||
result = subprocess.run(
|
||||
["bash", "-c", recipe],
|
||||
cwd=checkout,
|
||||
env={
|
||||
**os.environ,
|
||||
"PATH": f"{tools}{os.pathsep}{os.environ['PATH']}",
|
||||
"PYTHON": sys.executable,
|
||||
"TMPDIR": str(temporary),
|
||||
"AUR_GATE_TEST_LOG": str(log),
|
||||
"AUR_GATE_TEST_FAILURE": failure,
|
||||
},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
assert result.returncode == {"": 0, "build": 31, "arch": 42}[failure], result.stderr
|
||||
calls = [json.loads(line) for line in log.read_text().splitlines()]
|
||||
assert calls[0][0] == "build"
|
||||
output, diagnostics = (Path(path) for path in calls[0][1:])
|
||||
work = output.parent
|
||||
assert work.parent == temporary
|
||||
assert diagnostics.parent == work and diagnostics != output
|
||||
if failure != "build":
|
||||
assert calls[1] == ["arch", str(work / "source")]
|
||||
if failure:
|
||||
assert str(work) in result.stderr
|
||||
assert (diagnostics / "canonical.log").read_text() == "retained build diagnostics"
|
||||
else:
|
||||
assert not work.exists()
|
||||
assert {path: path.read_bytes() for path in protected} == protected
|
||||
+53
-20
@@ -86,30 +86,63 @@ def test_the_gate_runs_on_python_changes_and_aur_changes():
|
||||
assert "archlinux" in block, "the gate builds on Arch, not the runner"
|
||||
|
||||
|
||||
def test_the_gate_builds_this_commit_not_the_last_tag():
|
||||
"""The published source= points at the tag tarball, which does not exist
|
||||
until the tag is pushed — the reason nothing could run the PKGBUILD on a
|
||||
PR. The gate swaps it for a git archive of HEAD under the same filename
|
||||
and the same directory prefix build()/package() cd into."""
|
||||
def test_the_gate_builds_the_prepared_sdist_without_rewriting() -> None:
|
||||
"""PR and release gates build the canonical sdist with real checksums.
|
||||
|
||||
Release publication must consume the exact PKGBUILD tested on Arch, not
|
||||
rewrite its version, source or checksum after a different fixture passed.
|
||||
"""
|
||||
script = GATE_SH.read_text(encoding="utf-8")
|
||||
pkgbuild = PKGBUILD.read_text(encoding="utf-8")
|
||||
|
||||
assert "git archive" in script and '--prefix="${pkgname}-${_tag}/"' in script
|
||||
assert re.search(r'cd "\$\{pkgname\}-\$\{_tag\}"', pkgbuild), (
|
||||
"build()/package() no longer cd into ${pkgname}-${_tag}; the gate's"
|
||||
" archive prefix and the PKGBUILD must agree"
|
||||
)
|
||||
# The substitution itself.
|
||||
assert re.search(r"s\|\^source=.*\|source=", script)
|
||||
assert "git archive" not in script
|
||||
assert not re.search(r"sed\s+-i", script)
|
||||
assert 'cp "$PKGDIR/PKGBUILD" "$BUILD/PKGBUILD"' in script
|
||||
assert 'cmp -s "$PKGDIR/PKGBUILD" "$BUILD/PKGBUILD"' in script
|
||||
assert 'cd "${pkgname}-${_version}"' in pkgbuild
|
||||
assert "releases/download/v${_tag}/${pkgname}-${_version}.tar.gz" in pkgbuild
|
||||
assert "archive/refs/tags" not in pkgbuild
|
||||
assert re.search(r"^sha256sums=\('\w{64}'\)$", pkgbuild, re.M)
|
||||
assert "'SKIP'" not in pkgbuild
|
||||
|
||||
# And the published PKGBUILD still fetches the tag archive — if that
|
||||
# changes, updpkgsums at publish time and this gate both need a relook.
|
||||
assert "archive/refs/tags/v${_tag}.tar.gz" in pkgbuild
|
||||
assert "sha256sums=('SKIP')" in pkgbuild
|
||||
assert "updpkgsums: 'true'" in RELEASE.read_text(encoding="utf-8"), (
|
||||
"release.yml no longer computes the digest at publish time; the"
|
||||
" sha256sums=('SKIP') in the repo would ship to users"
|
||||
)
|
||||
|
||||
def test_pr_gate_consumes_this_commits_canonical_distributions() -> None:
|
||||
ci = _pipeline_text()
|
||||
wheel = _job_block(ci, "reproducible-wheel")
|
||||
aur = _job_block(ci, "aur-build-test")
|
||||
assert "name: python-dist" in wheel and "dist/*.tar.gz" in wheel
|
||||
assert "needs: [changes, reproducible-wheel]" in aur
|
||||
assert "name: python-dist" in aur
|
||||
assert "scripts/prepare_aur_source.py" in aur and "--local-unattested" in aur
|
||||
for path in ("MANIFEST.in", "README.md", "LICENSE"):
|
||||
assert f"- '{path}'" in ci
|
||||
|
||||
|
||||
def test_release_gate_verifies_published_sources_before_building() -> None:
|
||||
block = _job_block(RELEASE.read_text(), "aur-source-test")
|
||||
assert "needs: [build-and-release, publish-checksums]" in block
|
||||
assert "gh release download" in block and "--pattern SHA256SUMS" in block
|
||||
assert '--tag "$GITHUB_REF_NAME" --source-commit "$GITHUB_SHA"' in block
|
||||
assert "--manifest release-source/SHA256SUMS" in block
|
||||
assert "--local-unattested" not in block
|
||||
assert "continue-on-error" not in block
|
||||
assert "secrets.AUR_" not in block # Run the gate even without publication credentials.
|
||||
assert "AUR_SOURCE_ONLY=1" in block
|
||||
assert "AUR_FETCH_SOURCE=1" in block
|
||||
assert block.index("scripts/prepare_aur_source.py") < block.index("docker run")
|
||||
assert block.index("docker run") < block.index("name: aur-source-pkgbuild")
|
||||
assert "path: build/aur-source/PKGBUILD" in block
|
||||
|
||||
|
||||
def test_aur_publication_uses_only_the_gated_pkgbuild() -> None:
|
||||
block = _job_block(RELEASE.read_text(), "publish-aur")
|
||||
assert "needs: [build-and-release, aur-source-test]" in block
|
||||
assert "name: aur-source-pkgbuild" in block
|
||||
assert "pkgbuild: aur-source/PKGBUILD" in block
|
||||
assert "updpkgsums: 'false'" in block
|
||||
assert "sed -i" not in block
|
||||
assert "scripts/prepare_aur_source.py" not in block
|
||||
assert "actions/checkout@" not in block
|
||||
|
||||
|
||||
def test_the_gate_uses_a_non_root_build_user():
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Behavioral checks for the AUR source/provenance publication boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "scripts/prepare_aur_source.py"
|
||||
COMMIT = "a" * 40
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prepare_tool() -> ModuleType:
|
||||
"""Load the standalone release helper without importing the application."""
|
||||
spec = importlib.util.spec_from_file_location("prepare_aur_source", SCRIPT)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def sdist(
|
||||
directory: Path,
|
||||
version: str = "0.18.4",
|
||||
metadata: str | None = None,
|
||||
extra: tuple[str, ...] = (),
|
||||
) -> Path:
|
||||
"""Write a small real sdist with independently controllable metadata/members."""
|
||||
path = directory / f"vocalinux-{version}.tar.gz"
|
||||
info = metadata if metadata is not None else f"Name: vocalinux\nVersion: {version}\n"
|
||||
with tarfile.open(path, "w:gz") as archive:
|
||||
for name, text in [(f"vocalinux-{version}/PKG-INFO", info)] + [
|
||||
(name, "payload") for name in extra
|
||||
]:
|
||||
member = tarfile.TarInfo(name)
|
||||
payload = text.encode()
|
||||
member.size = len(payload)
|
||||
archive.addfile(member, io.BytesIO(payload))
|
||||
return path
|
||||
|
||||
|
||||
def manifest(directory: Path, tool: ModuleType, archive: Path) -> Path:
|
||||
"""Pin the exact source bytes, like a published release manifest."""
|
||||
path = directory / "SHA256SUMS"
|
||||
path.write_text(f"{tool.digest(archive)} {archive.name}\n")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag,version,pkgver",
|
||||
[
|
||||
("v0.18.4", "0.18.4", "0.18.4"),
|
||||
("v0.19.0-beta.1", "0.19.0b1", "0.19.0beta.1"),
|
||||
("v0.19.0-rc1", "0.19.0rc1", "0.19.0rc1"),
|
||||
],
|
||||
)
|
||||
def test_local_fixture_pins_original_bytes_and_preserves_prerelease_tag(
|
||||
prepare_tool: ModuleType, tmp_path: Path, tag: str, version: str, pkgver: str
|
||||
) -> None:
|
||||
archive = sdist(tmp_path, version)
|
||||
output = tmp_path / "prepared"
|
||||
prepare_tool.prepare(archive, output, tag=tag, local_unattested=True)
|
||||
assert (output / archive.name).read_bytes() == archive.read_bytes()
|
||||
# Evaluate the real PKGBUILD with bash, including source variable expansion.
|
||||
values = subprocess.check_output(
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
'source "$1"; printf "%s\\n" "$pkgver" "$_tag" "$_version" "${source[0]}" "${sha256sums[0]}"',
|
||||
"bash",
|
||||
str(output / "PKGBUILD"),
|
||||
],
|
||||
text=True,
|
||||
).splitlines()
|
||||
assert values == [
|
||||
pkgver,
|
||||
tag[1:],
|
||||
version,
|
||||
f"{archive.name}::https://github.com/VocaHQ/vocalinux/releases/download/{tag}/{archive.name}",
|
||||
prepare_tool.digest(archive),
|
||||
]
|
||||
|
||||
|
||||
def test_release_verifies_identity_ref_commit_and_bytes_before_staging(
|
||||
prepare_tool: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
archive = sdist(tmp_path)
|
||||
sums = manifest(tmp_path, prepare_tool, archive)
|
||||
output = tmp_path / "prepared"
|
||||
|
||||
def verified(argv: list[str], **kwargs: object) -> None:
|
||||
assert not output.exists()
|
||||
assert kwargs == {"check": True}
|
||||
assert argv == [
|
||||
"gh",
|
||||
"attestation",
|
||||
"verify",
|
||||
str(archive),
|
||||
"--repo",
|
||||
"VocaHQ/vocalinux",
|
||||
"--signer-workflow",
|
||||
"VocaHQ/vocalinux/.github/workflows/release.yml",
|
||||
"--source-ref",
|
||||
"refs/tags/v0.18.4",
|
||||
"--source-digest",
|
||||
COMMIT,
|
||||
]
|
||||
|
||||
check = Mock(side_effect=verified)
|
||||
monkeypatch.setattr(prepare_tool.subprocess, "run", check)
|
||||
prepare_tool.prepare(archive, output, "v0.18.4", sums, COMMIT)
|
||||
check.assert_called_once()
|
||||
assert sorted(path.name for path in output.iterdir()) == ["PKGBUILD", archive.name]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("problem", ["missing", "wrong", "duplicate", "skip", "mixed"])
|
||||
def test_bad_or_unpinned_checksum_fails_before_attestation_or_staging(
|
||||
prepare_tool: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, problem: str
|
||||
) -> None:
|
||||
archive = sdist(tmp_path)
|
||||
sums = manifest(tmp_path, prepare_tool, archive)
|
||||
good = sums.read_text()
|
||||
sums.write_text(
|
||||
{
|
||||
"missing": "",
|
||||
"wrong": f"{'0' * 64} {archive.name}\n",
|
||||
"duplicate": good + good,
|
||||
"skip": f"SKIP {archive.name}\n",
|
||||
"mixed": good + f"SKIP {archive.name}\n",
|
||||
}[problem]
|
||||
)
|
||||
check = Mock()
|
||||
monkeypatch.setattr(prepare_tool.subprocess, "run", check)
|
||||
output = tmp_path / "prepared"
|
||||
with pytest.raises(ValueError, match="SHA256SUMS"):
|
||||
prepare_tool.prepare(archive, output, "v0.18.4", sums, COMMIT)
|
||||
check.assert_not_called()
|
||||
assert not output.exists()
|
||||
|
||||
|
||||
def test_rejected_provenance_does_not_expose_publishable_inputs(
|
||||
prepare_tool: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
archive = sdist(tmp_path)
|
||||
sums = manifest(tmp_path, prepare_tool, archive)
|
||||
check = Mock(side_effect=subprocess.CalledProcessError(1, ["gh", "attestation", "verify"]))
|
||||
monkeypatch.setattr(prepare_tool.subprocess, "run", check)
|
||||
output = tmp_path / "prepared"
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
prepare_tool.prepare(archive, output, "v0.18.4", sums, COMMIT)
|
||||
assert not output.exists()
|
||||
|
||||
|
||||
def test_source_changes_after_verification_are_rejected(
|
||||
prepare_tool: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
archive = sdist(tmp_path)
|
||||
sums = manifest(tmp_path, prepare_tool, archive)
|
||||
check = Mock(
|
||||
side_effect=lambda *args, **kwargs: archive.write_bytes(b"changed after verification")
|
||||
)
|
||||
monkeypatch.setattr(prepare_tool.subprocess, "run", check)
|
||||
output = tmp_path / "prepared"
|
||||
with pytest.raises(ValueError, match="changed while staging"):
|
||||
prepare_tool.prepare(archive, output, "v0.18.4", sums, COMMIT)
|
||||
assert not output.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs,with_manifest,error",
|
||||
[
|
||||
({}, False, "requires --manifest"),
|
||||
({"commit": COMMIT}, False, "requires --manifest"),
|
||||
({}, True, "requires --manifest"),
|
||||
({"commit": "abc"}, True, "full source commit"),
|
||||
({"tag": "v0.18.3"}, False, "tag and sdist"),
|
||||
({"tag": "v0.18.4;echo unsafe"}, False, "tag and sdist"),
|
||||
({"tag": "v0.18.4-"}, False, "Invalid version"),
|
||||
],
|
||||
)
|
||||
def test_release_verification_cannot_be_silently_skipped(
|
||||
prepare_tool: ModuleType,
|
||||
tmp_path: Path,
|
||||
kwargs: dict[str, str],
|
||||
with_manifest: bool,
|
||||
error: str,
|
||||
) -> None:
|
||||
archive = sdist(tmp_path)
|
||||
output = tmp_path / "prepared"
|
||||
arguments = {"tag": "v0.18.4", **kwargs}
|
||||
if with_manifest:
|
||||
arguments["manifest"] = manifest(tmp_path, prepare_tool, archive)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
prepare_tool.prepare(archive, output, **arguments)
|
||||
assert not output.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metadata,extra",
|
||||
[
|
||||
("Name: other\nVersion: 0.18.4\n", ()),
|
||||
("Name: vocalinux\nVersion: 0.18.3\n", ()),
|
||||
("Name: vocalinux\nVersion: 0.18.4\nVersion: 0.18.3\n", ()),
|
||||
(None, ("other-root/file",)),
|
||||
(None, ("vocalinux-0.18.4/../unsafe",)),
|
||||
(None, ("vocalinux-0.18.4/PKG-INFO",)),
|
||||
],
|
||||
)
|
||||
def test_wrong_or_ambiguous_archive_metadata_is_rejected(
|
||||
prepare_tool: ModuleType, tmp_path: Path, metadata: str | None, extra: tuple[str, ...]
|
||||
) -> None:
|
||||
archive = sdist(tmp_path, metadata=metadata, extra=extra)
|
||||
with pytest.raises(ValueError):
|
||||
prepare_tool.prepare(archive, tmp_path / "prepared", local_unattested=True)
|
||||
|
||||
|
||||
def test_existing_outputs_are_not_overwritten(prepare_tool: ModuleType, tmp_path: Path) -> None:
|
||||
archive = sdist(tmp_path)
|
||||
output = tmp_path / "prepared"
|
||||
output.mkdir()
|
||||
user_file = output / "keep.txt"
|
||||
user_file.write_text("user work")
|
||||
with pytest.raises(ValueError, match="output already exists"):
|
||||
prepare_tool.prepare(archive, output, local_unattested=True)
|
||||
assert user_file.read_text() == "user work"
|
||||
|
||||
|
||||
def test_local_mode_rejects_release_verification_arguments(
|
||||
prepare_tool: ModuleType, tmp_path: Path
|
||||
) -> None:
|
||||
archive = sdist(tmp_path)
|
||||
with pytest.raises(ValueError, match="cannot take release verification"):
|
||||
prepare_tool.prepare(archive, tmp_path / "prepared", commit=COMMIT, local_unattested=True)
|
||||
|
||||
|
||||
def test_cli_failure_is_nonzero_and_leaves_no_publishable_file(tmp_path: Path) -> None:
|
||||
"""An incomplete release invocation must not fall back to local fixture mode."""
|
||||
archive = sdist(tmp_path)
|
||||
output = tmp_path / "prepared"
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--sdist",
|
||||
str(archive),
|
||||
"--tag",
|
||||
"v0.18.4",
|
||||
"--output",
|
||||
str(output),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 1
|
||||
assert "requires --manifest" in result.stderr
|
||||
assert not output.exists()
|
||||
@@ -112,13 +112,32 @@ def test_pr_ci_runs_the_same_reproducibility_gate():
|
||||
def test_release_publication_cannot_bypass_the_wheel_gate():
|
||||
"""Even independent Snap builds and always-run checksum jobs must wait."""
|
||||
jobs = _jobs()
|
||||
for name in ("publish-pypi", "publish-snap", "publish-aur", "publish-flatpak-remote"):
|
||||
for name in (
|
||||
"publish-pypi",
|
||||
"publish-snap",
|
||||
"publish-aur",
|
||||
"publish-aur-bin",
|
||||
"publish-flatpak-remote",
|
||||
):
|
||||
assert "build-and-release" in _needs(jobs[name]), name
|
||||
checksums = jobs["publish-checksums"]
|
||||
assert "needs.build-and-release.result == 'success'" in checksums
|
||||
assert "pattern: '@(python-dist|appimage-*|native-*|flatpak-*|snap-amd64)'" in checksums
|
||||
|
||||
|
||||
def test_aur_bin_publication_depends_only_on_its_appimage_inputs() -> None:
|
||||
"""Source-gate or unrelated package failures must not block ready AppImages."""
|
||||
jobs = _jobs()
|
||||
|
||||
def prerequisites(name: str) -> set[str]:
|
||||
direct = _needs(jobs[name])
|
||||
return direct.union(*(prerequisites(dependency) for dependency in direct))
|
||||
|
||||
expected = {"build-and-release", "build-appimage-arm64"}
|
||||
assert _needs(jobs["publish-aur-bin"]) == expected
|
||||
assert prerequisites("publish-aur-bin") == expected
|
||||
|
||||
|
||||
def test_every_consumer_downloads_the_build_instead_of_repeating_it():
|
||||
jobs = _jobs()
|
||||
for name in ("build-appimage-arm64", "publish-pypi"):
|
||||
|
||||
Reference in New Issue
Block a user