test(appimage): boot the built AppImage on six distros (#744)

* test(appimage): boot the built AppImage on six distros

* docs(agents): document the AppImage boot matrix

.github/workflows/unified-pipeline.yml

@@ -269,6 +269,44 @@ jobs:
           path: dist/*.AppImage
           retention-days: 7
 
+  # The build host cannot see what breaks here: its libraries match what it
+  # bundled. Two regressions reached a user that way (#743) — one loading a GI
+  # library, one spawning a host binary — and both die on the new distros below.
+  # Old ones prove the glibc floor the build image sets.
+  appimage-boot-test:
+    name: 🐧 Boot AppImage (${{ matrix.distro }})
+    runs-on: ubuntu-latest
+    needs: [changes, appimage-smoke-test]
+    if: needs.changes.outputs.python == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        distro:
+          - debian:12
+          - ubuntu:22.04
+          - ubuntu:24.04
+          - fedora:42
+          - archlinux:latest
+          - opensuse/tumbleweed
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download the AppImage built above
+        uses: actions/download-artifact@v4
+        with:
+          name: vocalinux-appimage-x86_64
+          path: dist
+
+      - name: Boot it on ${{ matrix.distro }}
+        run: |
+          APPIMAGE=$(ls dist/*.AppImage)
+          chmod +x "$APPIMAGE"
+          docker run --rm \
+            -v "$PWD/dist:/dist:ro" \
+            -v "$PWD/packaging/appimage:/pk:ro" \
+            "${{ matrix.distro }}" \
+            bash /pk/boot-test.sh "/dist/$(basename "$APPIMAGE")"
+
   web-build:
     name: 🌐 Build Website
     runs-on: ubuntu-latest

AGENTS.md

@@ -79,6 +79,7 @@ just lock          # regenerate uv.lock + requirements/*.txt
 just lock-check    # fail if uv.lock is stale vs pyproject.toml
 just model-checksums  # refresh pinned model digests after adding a model
 just appimage      # build the AppImage in its pinned base image (needs docker)
+just appimage-boot fedora:42   # boot that AppImage in a distro container
 just pre-commit    # pre-commit run --all-files
 just run-debug     # vocalinux --debug
 just run-source-debug
@@ -108,6 +109,7 @@ Website: `web/AGENTS.md`, `web/PRODUCT.md`, `web/DESIGN.md`. Do not duplicate si
 | `[vad]` extra | `onnxruntime` for Silero VAD |
 | AppImage PyGObject | Pinned separately in `requirements/appimage.in`, and below 3.52: the AppImage bundles its own interpreter and builds PyGObject against the base image's girepository-1.0, while uv.lock's 3.56 needs girepository-2.0 (glib 2.80+) |
 | AppImage build inputs | Base image, tooling, interpreter, shaderc and the Vulkan headers are pinned in `packaging/appimage/tool_checksums.txt`. Build with `just appimage` (docker) — building on the host ships the host's glibc, which is what kept the AppImage off Debian 12 |
+| AppImage boot matrix | `packaging/appimage/boot-test.sh` runs the finished AppImage in distro containers, and the matrix in `unified-pipeline.yml` must keep distros both older and newer than the build image: the old ones prove the glibc floor, the new ones catch a bundle that breaks the host binaries it spawns. A distro added there needs its package recipe in the script — `tests/test_appimage_packaging.py` checks both |
 | Speech models | Verified against digests pinned in `src/vocalinux/utils/model_checksums.txt` before install, by both `install.sh` and the runtime downloaders. **Fails closed** — an unpinned model is refused. Regenerate with `just model-checksums` (never by hand); `tests/test_model_checksums.py` fails if it falls behind. whisper.cpp URLs use a pinned Hugging Face commit, never `main` |
 
 Optional extras: `vosk`, `whisper`, `vad`, `dev`.

justfile

@@ -97,6 +97,14 @@ build:
 appimage: build
     bash packaging/appimage/docker-build.sh dist/*.whl "$(grep -oP '__version__\s*=\s*"\K[^"]+' src/vocalinux/version.py)" dist
 
+# Start the built AppImage in a distro container, as the CI matrix does. Pick a
+# distro older than the build image to test the glibc floor, or newer to test
+# that the bundle does not break the host binaries it spawns.
+#
+# Boot the built AppImage on a distro, e.g. `just appimage-boot fedora:42`
+appimage-boot distro="debian:12":
+    docker run --rm -v "$PWD/dist:/dist:ro" -v "$PWD/packaging/appimage:/pk:ro" {{distro}} bash /pk/boot-test.sh "/dist/$(basename "$(ls dist/*.AppImage)")"
+
 # Regenerate uv.lock and the hash-pinned requirements/* exports.
 # Bump the torch/torchaudio +cpu pins in requirements/whisper.in together
 # when you want newer CPU builds (torchaudio on the CPU index lags torch).

packaging/appimage/boot-test.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+# Start the finished AppImage in a distro container and check it comes up.
+#
+# Usage: boot-test.sh <path-to-appimage>
+#
+# The build host cannot catch what this catches: its libraries match what it
+# bundled. Both #743 regressions reached a user for that reason — one when the
+# bundle loaded a GI library the host also had (libibus), one when the app
+# handed its own library path to a host binary (ibus). Old distros prove the
+# glibc floor; new ones prove the bundle does not poison what it shells out to.
+set -euo pipefail
+
+APPIMAGE="$(readlink -f "${1:?usage: boot-test.sh <path-to-appimage>}")"
+export APPIMAGE_EXTRACT_AND_RUN=1
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+export XDG_CONFIG_HOME="$WORKDIR/config" XDG_DATA_HOME="$WORKDIR/data" \
+       XDG_CACHE_HOME="$WORKDIR/cache" HOME="$WORKDIR"
+mkdir -p "$XDG_CONFIG_HOME" "$XDG_DATA_HOME" "$XDG_CACHE_HOME"
+
+. /etc/os-release
+DISTRO="$PRETTY_NAME"
+echo "== $DISTRO (glibc $(ldd --version | head -1 | grep -oE '[0-9]+\.[0-9]+$')) =="
+
+# Xvfb and a session bus for the tray, a GLib CLI to prove host binaries still
+# run, the distro's GTK 3 runtime, and xdotool. The last two are not the AppImage
+# failing
+# to be self-contained: the AppImage excludelist deliberately leaves libX11,
+# libharfbuzz and friends to the host, because bundling them breaks more than it
+# fixes, and xdotool is a documented host prerequisite for every install path
+# except the Flatpak. Any desktop has them; a bare container does not.
+echo "== Installing test prerequisites =="
+case "${ID}${ID_LIKE:+ $ID_LIKE}" in
+  *debian*|ubuntu*)
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update -qq
+    apt-get install -y -qq --no-install-recommends \
+      xvfb xauth dbus-x11 libglib2.0-bin ca-certificates libgtk-3-0 xdotool >/dev/null
+    ;;
+  *fedora*|*rhel*)
+    dnf install -y -q xorg-x11-server-Xvfb xorg-x11-xauth dbus-x11 glib2 gtk3 xdotool >/dev/null
+    ;;
+  *arch*)
+    pacman -Sy --noconfirm --quiet xorg-server-xvfb xorg-xauth dbus glib2 gtk3 xdotool >/dev/null
+    ;;
+  *suse*)
+    # openSUSE splits these differently: xvfb-run ships on its own, the session
+    # bus is in dbus-1-daemon, and the GTK 3 runtime is libgtk-3-0.
+    zypper -n -q in xorg-x11-server-Xvfb xvfb-run xauth dbus-1-daemon \
+      glib2-tools libgtk-3-0 xdotool >/dev/null
+    ;;
+  *)
+    echo "No package recipe for '$ID'; add one to boot-test.sh" >&2
+    exit 1
+    ;;
+esac
+
+fail() {
+  echo "FAIL: $*" >&2
+  exit 1
+}
+
+echo "== It runs at all =="
+"$APPIMAGE" --version || fail "the AppImage does not start on $DISTRO"
+
+# Unpack once so the checks below can use the bundle's own interpreter.
+cd "$WORKDIR"
+"$APPIMAGE" --appimage-extract >/dev/null
+BUNDLE="$WORKDIR/squashfs-root"
+bundle_python() {
+  APPDIR="$BUNDLE" \
+  PYTHONHOME="$BUNDLE/usr" \
+  PYTHONPATH="$BUNDLE/usr/lib/python3:$BUNDLE/usr/lib/python3/site-packages" \
+  GI_TYPELIB_PATH="$BUNDLE/usr/lib/girepository-1.0" \
+  LD_LIBRARY_PATH="$BUNDLE/usr/lib" \
+  "$BUNDLE/usr/bin/python3" "$@"
+}
+
+echo "== Its GI stack is the bundled one =="
+# A typelib whose library is missing loads the host's copy against the bundle's
+# older GLib. That is how libibus turned IBus.Engine into a 'void' GType.
+bundle_python - <<'PY' || fail "the bundled GI stack does not load"
+import gi
+
+gi.require_version("Gtk", "3.0")
+gi.require_version("IBus", "1.0")
+from gi.repository import Gtk, IBus  # noqa: F401
+
+if IBus.Engine.__gtype__.name in ("void", "invalid"):
+    raise SystemExit("IBus typelib loaded but its library did not")
+
+from vocalinux.text_injection import text_injector  # noqa: F401
+
+print("  Gtk 3.0, IBus and text_injection all import")
+PY
+
+echo "== Host binaries still run when the app spawns them =="
+# The bundle's LD_LIBRARY_PATH reaches every child process, so a host tool built
+# against a newer GLib dies before it does anything: `ibus` exited with
+# `undefined symbol: g_free_sized` and dictated text reached no window.
+bundle_python - <<'PY' || fail "a host binary cannot run from inside the bundle"
+import subprocess
+
+from vocalinux.utils.host_process import host_env
+
+PROBE = ["gsettings", "--version"]
+inherited = subprocess.run(PROBE, capture_output=True, text=True)
+sanitized = subprocess.run(PROBE, capture_output=True, text=True, env=host_env())
+if sanitized.returncode != 0:
+    raise SystemExit(f"gsettings failed with a sanitized environment: {sanitized.stderr.strip()}")
+if inherited.returncode != 0:
+    print("  host GLib is newer than the bundle's; host_env() is what keeps it working")
+print(f"  gsettings {sanitized.stdout.strip()} runs")
+PY
+
+echo "== It reaches the tray =="
+LOG="$WORKDIR/boot.log"
+set +e
+timeout -s TERM 40 xvfb-run -a dbus-run-session -- \
+  "$APPIMAGE" --debug --start-minimized >"$LOG" 2>&1
+status=$?
+set -e
+# 124 is the timeout we asked for: the app is a tray application, so staying up
+# until killed is the pass condition, not an early exit.
+if [ "$status" -ne 124 ] && [ "$status" -ne 0 ] && [ "$status" -ne 143 ]; then
+  echo "--- last 30 lines ---" >&2
+  tail -n 30 "$LOG" >&2
+  fail "the app exited with $status instead of running"
+fi
+
+grep -q "Initializing Vocalinux" "$LOG" || {
+  tail -n 30 "$LOG" >&2
+  fail "startup never reached component initialisation"
+}
+grep -q "Initializing system tray indicator" "$LOG" || {
+  tail -n 30 "$LOG" >&2
+  fail "startup never reached the tray indicator"
+}
+if grep -q "Failed to create AppIndicator" "$LOG"; then
+  tail -n 30 "$LOG" >&2
+  fail "the tray indicator could not be created"
+fi
+# A traceback before the tray is a failed startup. One after it is the noise
+# pynput's X listener makes when we TERM the app under Xvfb, so it is reported
+# rather than fatal — the process staying up to the timeout is the real signal.
+startup="$WORKDIR/startup.log"
+sed -n '1,/Initializing system tray indicator/p' "$LOG" > "$startup"
+if grep -q "^Traceback" "$startup"; then
+  grep -A 15 "^Traceback" "$startup" | head -n 20 >&2
+  fail "startup raised an exception"
+fi
+if grep -q "^Traceback" "$LOG"; then
+  echo "  note: traceback on the shutdown path, after the tray came up" >&2
+fi
+
+echo "PASS: $DISTRO"

tests/test_appimage_packaging.py

@@ -6,6 +6,15 @@ from pathlib import Path
 REPO_ROOT = Path(__file__).resolve().parents[1]
 APPIMAGE = REPO_ROOT / "packaging" / "appimage"
 BUILD_SH = APPIMAGE / "build.sh"
+BOOT_TEST_SH = APPIMAGE / "boot-test.sh"
+#: Image name prefix -> the token boot-test.sh matches on ($ID / $ID_LIKE).
+BOOT_FAMILIES = {
+    "debian": "debian",
+    "ubuntu": "ubuntu",
+    "fedora": "fedora",
+    "archlinux": "arch",
+    "opensuse": "suse",
+}
 PINS = APPIMAGE / "tool_checksums.txt"
 REQUIREMENTS = REPO_ROOT / "requirements"
 WORKFLOWS = REPO_ROOT / ".github" / "workflows"
@@ -250,3 +259,44 @@ def test_vulkan_shaders_get_the_compiler_the_base_image_lacks():
         assert (
             "vocalinux-appimage" in content
         ), f"{workflow} does not cache the built glslc; every run rebuilds it"
+
+
+def _boot_matrix() -> list:
+    """The distros the boot job runs the finished AppImage on."""
+    workflow = (WORKFLOWS / "unified-pipeline.yml").read_text(encoding="utf-8")
+    block = re.search(r"\n        distro:\n((?:\s+- \S+\n)+)", workflow)
+    assert block, "unified-pipeline.yml has no boot-test distro matrix"
+    return re.findall(r"- (\S+)", block.group(1))
+
+
+def test_the_boot_matrix_covers_both_ends_of_the_range():
+    """Built on 22.04, the AppImage can fail in either direction.
+
+    Too new a glibc and the old distros cannot start it; too old a GLib and the
+    new distros' own binaries break when the app spawns them. Both #743
+    regressions were the second kind, and only a new distro shows them.
+    """
+    distros = _boot_matrix()
+    assert any(
+        distro.startswith(("debian:12", "ubuntu:22.04")) for distro in distros
+    ), f"nothing older than the build image in {distros}"
+    assert any(
+        distro.startswith(("archlinux", "opensuse", "fedora")) for distro in distros
+    ), f"nothing newer than the build image in {distros}"
+
+
+def test_boot_test_has_a_package_recipe_for_every_distro_it_runs_on():
+    """A matrix entry the script cannot install prerequisites for fails late."""
+    script = BOOT_TEST_SH.read_text(encoding="utf-8")
+    case_block = re.search(r"case \"\$\{ID\}.*?esac", script, re.S)
+    assert case_block, "boot-test.sh no longer dispatches on the distro"
+
+    missing = []
+    for distro in _boot_matrix():
+        family = next(
+            (token for prefix, token in BOOT_FAMILIES.items() if distro.startswith(prefix)),
+            None,
+        )
+        if family is None or family not in case_block.group(0):
+            missing.append(distro)
+    assert not missing, f"boot-test.sh installs nothing for: {missing}"