diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fc48977..076892b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -64,13 +64,21 @@ jobs: python -m build ls -la dist/ - # glslc is built from the pinned shaderc commit inside the container - - # Ubuntu 22.04 packages none - and depends on nothing but that commit. + # glslc and the pywhispercpp Vulkan rebuild are native artifacts. Neither + # depends on this commit: the id script keys them to the base image, + # tool_checksums.txt, and the pywhispercpp pin. + - name: AppImage native cache id + id: native-cache + run: | + set -euo pipefail + id="$(bash packaging/appimage/native-cache-id.sh)" + echo "id=${id}" >> "$GITHUB_OUTPUT" + - name: Cache AppImage build tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/vocalinux-appimage - key: appimage-tools-${{ runner.arch }}-${{ hashFiles('packaging/appimage/tool_checksums.txt') }} + key: appimage-native-${{ runner.arch }}-${{ steps.native-cache.outputs.id }} - name: Build AppImage (x86_64) env: @@ -275,13 +283,21 @@ jobs: python -m build ls -la dist/ - # glslc is built from the pinned shaderc commit inside the container - - # Ubuntu 22.04 packages none - and depends on nothing but that commit. + # glslc and the pywhispercpp Vulkan rebuild are native artifacts. Neither + # depends on this commit: the id script keys them to the base image, + # tool_checksums.txt, and the pywhispercpp pin. + - name: AppImage native cache id + id: native-cache + run: | + set -euo pipefail + id="$(bash packaging/appimage/native-cache-id.sh)" + echo "id=${id}" >> "$GITHUB_OUTPUT" + - name: Cache AppImage build tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/vocalinux-appimage - key: appimage-tools-${{ runner.arch }}-${{ hashFiles('packaging/appimage/tool_checksums.txt') }} + key: appimage-native-${{ runner.arch }}-${{ steps.native-cache.outputs.id }} - name: Build AppImage (aarch64) env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9bd127e..0439449 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,13 +90,21 @@ jobs: if-no-files-found: error retention-days: 7 - # glslc is built from the pinned shaderc commit inside the container - - # Ubuntu 22.04 packages none - and depends on nothing but that commit. + # glslc and the pywhispercpp Vulkan rebuild are native artifacts. Neither + # depends on this commit: the id script keys them to the base image, + # tool_checksums.txt, and the pywhispercpp pin. + - name: AppImage native cache id + id: native-cache + run: | + set -euo pipefail + id="$(bash packaging/appimage/native-cache-id.sh)" + echo "id=${id}" >> "$GITHUB_OUTPUT" + - name: Cache AppImage build tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/vocalinux-appimage - key: appimage-tools-${{ runner.arch }}-${{ hashFiles('packaging/appimage/tool_checksums.txt') }} + key: appimage-native-${{ runner.arch }}-${{ steps.native-cache.outputs.id }} - name: Build AppImage (x86_64) env: @@ -305,13 +313,21 @@ jobs: name: python-dist path: dist - # glslc is built from the pinned shaderc commit inside the container - - # Ubuntu 22.04 packages none - and depends on nothing but that commit. + # glslc and the pywhispercpp Vulkan rebuild are native artifacts. Neither + # depends on this commit: the id script keys them to the base image, + # tool_checksums.txt, and the pywhispercpp pin. + - name: AppImage native cache id + id: native-cache + run: | + set -euo pipefail + id="$(bash packaging/appimage/native-cache-id.sh)" + echo "id=${id}" >> "$GITHUB_OUTPUT" + - name: Cache AppImage build tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/vocalinux-appimage - key: appimage-tools-${{ runner.arch }}-${{ hashFiles('packaging/appimage/tool_checksums.txt') }} + key: appimage-native-${{ runner.arch }}-${{ steps.native-cache.outputs.id }} - name: Build AppImage (aarch64) env: diff --git a/.github/workflows/unified-pipeline.yml b/.github/workflows/unified-pipeline.yml index 7d33ec3..61d6207 100644 --- a/.github/workflows/unified-pipeline.yml +++ b/.github/workflows/unified-pipeline.yml @@ -268,13 +268,21 @@ jobs: python -m build ls -la dist/ - # glslc is built from the pinned shaderc commit inside the container - - # Ubuntu 22.04 packages none - and depends on nothing but that commit. + # glslc and the pywhispercpp Vulkan rebuild are native artifacts. Neither + # depends on this commit: the id script keys them to the base image, + # tool_checksums.txt, and the pywhispercpp pin. + - name: AppImage native cache id + id: native-cache + run: | + set -euo pipefail + id="$(bash packaging/appimage/native-cache-id.sh)" + echo "id=${id}" >> "$GITHUB_OUTPUT" + - name: Cache AppImage build tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/vocalinux-appimage - key: appimage-tools-${{ matrix.arch }}-${{ hashFiles('packaging/appimage/tool_checksums.txt') }} + key: appimage-native-${{ runner.arch }}-${{ steps.native-cache.outputs.id }} - name: Build AppImage env: diff --git a/packaging/appimage/build.sh b/packaging/appimage/build.sh index 59a4386..c0b815f 100755 --- a/packaging/appimage/build.sh +++ b/packaging/appimage/build.sh @@ -29,12 +29,17 @@ # Set VOCALINUX_APPIMAGE_REQUIRE_VULKAN=1 in CI so a failed rebuild fails # the job instead of shipping a CPU-only image. Set # VOCALINUX_APPIMAGE_SKIP_VULKAN=1 to force the CPU wheel. +# A build that produced libggml-vulkan.so is kept in VOCALINUX_APPIMAGE_CACHE +# and reused while the base image, tool_checksums.txt, and the pywhispercpp +# pin are unchanged. A CPU-only build is not cached. set -euo pipefail WHEEL="$1" VERSION="$2" OUTDIR="${3:-dist}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=native-cache.sh +source "$(dirname "${BASH_SOURCE[0]}")/native-cache.sh" # Same pywhispercpp release install.sh pins. Read from there rather than repeated # here: unpinned, this build picks up whatever PyPI published today, which is how @@ -422,13 +427,10 @@ copy_whisper_native_libs_to_usr_lib() { done < <(find "$APPDIR/usr" \( -name 'libggml*.so*' -o -name 'libwhisper.so*' \) ! -path '*/usr/lib/*' 2>/dev/null || true) } -rebuild_pywhispercpp_vulkan() { - local require_vulkan="${VOCALINUX_APPIMAGE_REQUIRE_VULKAN:-0}" - if [ "${VOCALINUX_APPIMAGE_SKIP_VULKAN:-0}" = "1" ]; then - echo "== Skipping pywhispercpp Vulkan rebuild (VOCALINUX_APPIMAGE_SKIP_VULKAN=1) ==" - return 0 - fi - +# The miss path of rebuild_pywhispercpp_vulkan. require_vulkan, cache_dir, and +# site are locals of that caller; bash shows them to this function. +_compile_pywhispercpp_vulkan() { + local vk_lib if ! has_vulkan_build_deps; then echo "Vulkan build deps missing (libvulkan-dev plus a C++ compiler)." >&2 if [ "$require_vulkan" = "1" ]; then @@ -481,7 +483,6 @@ rebuild_pywhispercpp_vulkan() { return 0 fi - local vk_lib vk_lib="$(find "$APPDIR/usr" -name 'libggml-vulkan.so*' 2>/dev/null | head -1 || true)" if [ -z "$vk_lib" ]; then echo "Vulkan rebuild did not produce libggml-vulkan.so." >&2 @@ -494,6 +495,26 @@ rebuild_pywhispercpp_vulkan() { fi echo " found $vk_lib" copy_whisper_native_libs_to_usr_lib + if ! pywhispercpp_cache_publish "$site" "$cache_dir"; then + echo "Warning: Vulkan build succeeded but was not cached for the next run." >&2 + fi +} + +rebuild_pywhispercpp_vulkan() { + local require_vulkan="${VOCALINUX_APPIMAGE_REQUIRE_VULKAN:-0}" + if [ "${VOCALINUX_APPIMAGE_SKIP_VULKAN:-0}" = "1" ]; then + echo "== Skipping pywhispercpp Vulkan rebuild (VOCALINUX_APPIMAGE_SKIP_VULKAN=1) ==" + return 0 + fi + + # The Vocalinux wheel was installed above and is not part of this cache. + # A hit still replaces the CPU pywhispercpp wheel the lock just installed, + # and it does not run _compile_pywhispercpp_vulkan. + local cache_dir site + cache_dir="$(pywhispercpp_native_cache_dir "$ARCH")" + site="$APPDIR/usr/lib/python${PY_VER}/site-packages" + pywhispercpp_restore_or_run "$cache_dir" "$site" copy_whisper_native_libs_to_usr_lib \ + _compile_pywhispercpp_vulkan } echo "== Copying GObject-Introspection typelibs (not handled by linuxdeploy-plugin-gtk) ==" diff --git a/packaging/appimage/docker-build.sh b/packaging/appimage/docker-build.sh index f8602d7..517dc9c 100755 --- a/packaging/appimage/docker-build.sh +++ b/packaging/appimage/docker-build.sh @@ -9,8 +9,8 @@ # whatever that host happens to run. # # Honours VOCALINUX_APPIMAGE_REQUIRE_VULKAN / _SKIP_VULKAN like build.sh, and -# keeps the glslc it has to build in VOCALINUX_APPIMAGE_CACHE -# (~/.cache/vocalinux-appimage by default) so the next build reuses it. +# keeps glslc and the pywhispercpp Vulkan build in VOCALINUX_APPIMAGE_CACHE +# (~/.cache/vocalinux-appimage by default) so the next build reuses them. set -euo pipefail REPO_ROOT="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/../.." && pwd)" diff --git a/packaging/appimage/native-cache-id.sh b/packaging/appimage/native-cache-id.sh new file mode 100755 index 0000000..d24084a --- /dev/null +++ b/packaging/appimage/native-cache-id.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Id for the reusable pywhispercpp Vulkan build. +# +# That compile depends on the base image (glibc, and the toolchain apt installs +# from it), every pin in tool_checksums.txt, the cmake/ninja pins in +# requirements/appimage-tools.txt, the Vulkan flags in build.sh, and the +# pywhispercpp sdist install.sh names. It does not depend on the rest of the +# Vocalinux sources, so this id ignores the git commit and the Actions run id +# on purpose: putting either in the cache key would rebuild on every push. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo="$(cd "$here/../.." && pwd)" +pins="${VOCALINUX_PINS:-$repo/packaging/appimage/tool_checksums.txt}" +install_sh="${VOCALINUX_INSTALL_SH:-$repo/install.sh}" +tools="${VOCALINUX_APPIMAGE_TOOLS:-$repo/requirements/appimage-tools.txt}" +build_sh="${VOCALINUX_BUILD_SH:-$repo/packaging/appimage/build.sh}" + +base="$(awk '$1=="base-image" {print $3; exit}' "$pins")" +if [ -z "$base" ]; then + echo "No base-image pin in $pins" >&2 + exit 1 +fi +pywhispercpp="$(sed -n 's/^PYWHISPERCPP_VERSION="\([^"]*\)"/\1/p' "$install_sh" | head -n1)" +if [ -z "$pywhispercpp" ]; then + echo "Could not read PYWHISPERCPP_VERSION from $install_sh" >&2 + exit 1 +fi +if [ ! -f "$tools" ]; then + echo "No AppImage build-tool pins at $tools" >&2 + exit 1 +fi +# Cache revision for the native flags. Hashing those lines, not all of +# build.sh, misses the cache when the compile changes and hits when a comment +# elsewhere in the script changes. +flags="$(awk '/GGML_VULKAN=1/ || /CMAKE_ARGS=/ { print }' "$build_sh")" +if ! printf '%s\n' "$flags" | grep -q 'GGML_VULKAN=1'; then + echo "Could not read the Vulkan compile flags from $build_sh" >&2 + exit 1 +fi +if ! printf '%s\n' "$flags" | grep -q 'CMAKE_ARGS='; then + echo "Could not read CMAKE_ARGS from $build_sh" >&2 + exit 1 +fi +pins_hash="$(sha256sum "$pins" | awk '{print $1}')" +tools_hash="$(sha256sum "$tools" | awk '{print $1}')" +printf '%s\n%s\n%s\n%s\n%s\n' \ + "$base" "$pins_hash" "$pywhispercpp" "$tools_hash" "$flags" \ + | sha256sum | awk '{print $1}' diff --git a/packaging/appimage/native-cache.sh b/packaging/appimage/native-cache.sh new file mode 100755 index 0000000..ed29685 --- /dev/null +++ b/packaging/appimage/native-cache.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Reuse a pywhispercpp build that already contains libggml-vulkan.so. +# +# Sourced by build.sh. A tree without that library is a CPU wheel, and caching +# one would ship it to the next run that asked for Vulkan. +# +# The directory name is the id from native-cache-id.sh plus the architecture. +# The .so is not portable across either. + +_NATIVE_CACHE_HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +pywhispercpp_native_cache_dir() { + local arch="$1" + local root id + root="${VOCALINUX_APPIMAGE_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/vocalinux-appimage}" + id="$(bash "$_NATIVE_CACHE_HERE/native-cache-id.sh")" + printf '%s\n' "$root/pywhispercpp-vulkan-${arch}-${id}" +} + +pywhispercpp_cache_has_vulkan() { + local dir="$1" + local found + [ -d "$dir" ] || return 1 + found="$(find "$dir" -name 'libggml-vulkan.so*' -print -quit)" + [ -n "$found" ] +} + +# Copy the pywhispercpp install out of a site-packages directory. +# Leaves the rest of the prefix (Vocalinux, numpy, …) where it is. +pywhispercpp_cache_publish() { + local site="$1" dest="$2" + local parent staging pattern backup + # Called from `if !`, which disables set -e for this whole function. + # A failing cp has to return on its own, or a short write becomes the cache. + parent="$(dirname "$dest")" + mkdir -p "$parent" || return 1 + staging="$(mktemp -d "$parent/.pywhispercpp-staging-XXXXXX")" || return 1 + for pattern in \ + "$site/pywhispercpp" \ + "$site"/pywhispercpp-*.dist-info \ + "$site"/pywhispercpp.libs \ + "$site"/_pywhispercpp* \ + "$site"/libggml*.so* \ + "$site"/libwhisper.so* + do + [ -e "$pattern" ] || [ -L "$pattern" ] || continue + if ! cp -a "$pattern" "$staging/"; then + rm -rf "$staging" + echo "Failed to copy $(basename "$pattern") into the pywhispercpp cache." >&2 + return 1 + fi + done + if ! pywhispercpp_cache_has_vulkan "$staging"; then + rm -rf "$staging" + echo "Refusing to cache a pywhispercpp build with no libggml-vulkan.so" >&2 + return 1 + fi + backup="${dest}.replacing" + rm -rf "$backup" + if [ -e "$dest" ]; then + if ! mv "$dest" "$backup"; then + rm -rf "$staging" + echo "Failed to move the previous pywhispercpp cache aside." >&2 + return 1 + fi + fi + if ! mv "$staging" "$dest"; then + echo "Failed to publish the pywhispercpp cache." >&2 + if [ -e "$backup" ] && ! mv "$backup" "$dest"; then + echo "Also failed to restore the previous pywhispercpp cache." >&2 + fi + rm -rf "$staging" + return 1 + fi + rm -rf "$backup" +} + +# Replace whatever pywhispercpp the prefix just installed (the CPU wheel) with +# the cached Vulkan build. Other packages in site-packages stay. +pywhispercpp_cache_restore() { + local src="$1" site="$2" + if ! pywhispercpp_cache_has_vulkan "$src"; then + echo "Cached pywhispercpp tree has no libggml-vulkan.so" >&2 + return 1 + fi + mkdir -p "$site" || return 1 + find "$site" -depth \( \ + -name 'pywhispercpp' -o \ + -name 'pywhispercpp.libs' -o \ + -name 'pywhispercpp-*.dist-info' -o \ + -name '_pywhispercpp*' -o \ + -name 'libggml*.so*' -o \ + -name 'libwhisper.so*' \ + \) -exec rm -rf {} + 2>/dev/null || true + if ! cp -a "$src"/. "$site/"; then + echo "Restoring the cached pywhispercpp tree failed." >&2 + return 1 + fi +} + +# 0 when the cached tree is in place. 1 when the caller has to compile. +pywhispercpp_try_restore() { + local cache_dir="$1" + local site="$2" + local vk_lib + if ! pywhispercpp_cache_has_vulkan "$cache_dir"; then + return 1 + fi + echo "== Reusing pywhispercpp Vulkan build (${cache_dir##*/}) ==" + if pywhispercpp_cache_restore "$cache_dir" "$site"; then + vk_lib="$(find "$site" -name 'libggml-vulkan.so*' -print -quit 2>/dev/null || true)" + if [ -n "$vk_lib" ]; then + echo " found $vk_lib" + return 0 + fi + fi + echo "Cached Vulkan build could not be restored; compiling again." >&2 + return 1 +} + +# $1 cache dir, $2 site-packages, $3 command to run on a hit, then the compile +# command. A hit does not run the compile command. +pywhispercpp_restore_or_run() { + local cache_dir="$1" + local site="$2" + local on_reuse="$3" + shift 3 + if pywhispercpp_try_restore "$cache_dir" "$site"; then + "$on_reuse" + return $? + fi + if [ "$#" -eq 0 ]; then + echo "No compile command given for the pywhispercpp Vulkan build." >&2 + return 1 + fi + "$@" +} diff --git a/tests/test_appimage_native_cache.py b/tests/test_appimage_native_cache.py new file mode 100644 index 0000000..9c0f797 --- /dev/null +++ b/tests/test_appimage_native_cache.py @@ -0,0 +1,373 @@ +"""The AppImage reuses its pywhispercpp Vulkan build only when the pins match. + +The compile does not depend on the Vocalinux commit. The cache id is the base +image, tool_checksums.txt, the pinned cmake and ninja, the Vulkan flags, and +the pywhispercpp pin. It is not the git SHA and not the Actions run id. A tree +without libggml-vulkan.so is a CPU wheel and is not cached. +""" + +from __future__ import annotations + +import os +import re +import shlex +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +APPIMAGE = REPO_ROOT / "packaging" / "appimage" +ID_SCRIPT = APPIMAGE / "native-cache-id.sh" +CACHE_SH = APPIMAGE / "native-cache.sh" +BUILD_SH = APPIMAGE / "build.sh" +PINS = APPIMAGE / "tool_checksums.txt" +INSTALL_SH = REPO_ROOT / "install.sh" +TOOLS = REPO_ROOT / "requirements" / "appimage-tools.txt" +WORKFLOWS = REPO_ROOT / ".github" / "workflows" +APPIMAGE_WORKFLOWS = ("unified-pipeline.yml", "release.yml", "nightly.yml") +NATIVE_KEY = "appimage-native-${{ runner.arch }}-${{ steps.native-cache.outputs.id }}" + + +def _run(args: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + check=False, + capture_output=True, + text=True, + env=os.environ.copy() if env is None else env, + ) + + +def _cache_id(pins: Path, install: Path, extra: dict[str, str] | None = None) -> str: + env = os.environ.copy() + env["VOCALINUX_PINS"] = str(pins) + env["VOCALINUX_INSTALL_SH"] = str(install) + if extra: + env.update(extra) + result = _run(["bash", str(ID_SCRIPT)], env=env) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + +def _source(body: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + script = "set -euo pipefail\nsource " + shlex.quote(str(CACHE_SH)) + "\n" + body + "\n" + return _run(["bash", "-c", script], env=env) + + +def _flip_pin_value(text: str, name: str) -> str: + """Change the value column of one tool_checksums.txt row.""" + pattern = re.compile(rf"^({re.escape(name)}\s+\S+\s+)(\S+)", re.M) + match = pattern.search(text) + assert match, name + value = match.group(2) + replacement = "0" if value[-1] != "0" else "1" + return text[: match.start(2)] + value[:-1] + replacement + text[match.end(2) :] + + +def test_native_cache_id_tracks_base_image_pins_and_pywhispercpp(tmp_path: Path) -> None: + pins_text = PINS.read_text(encoding="utf-8") + install_text = INSTALL_SH.read_text(encoding="utf-8") + pins = tmp_path / "tool_checksums.txt" + install = tmp_path / "install.sh" + pins.write_text(pins_text, encoding="utf-8") + install.write_text(install_text, encoding="utf-8") + + baseline = _cache_id(pins, install) + assert baseline == _cache_id(PINS, INSTALL_SH) + assert re.fullmatch(r"[0-9a-f]{64}", baseline) + + pins.write_text(_flip_pin_value(pins_text, "base-image"), encoding="utf-8") + assert _cache_id(pins, install) != baseline + + pins.write_text(_flip_pin_value(pins_text, "shaderc"), encoding="utf-8") + assert _cache_id(pins, install) != baseline + + pins.write_text(pins_text, encoding="utf-8") + declared = re.search(r'^PYWHISPERCPP_VERSION="([^"]+)"', install_text, re.M) + assert declared, "install.sh no longer declares the version the id reads" + bumped = install_text.replace( + f'PYWHISPERCPP_VERSION="{declared.group(1)}"', + f'PYWHISPERCPP_VERSION="{declared.group(1)}.1"', + 1, + ) + install.write_text(bumped, encoding="utf-8") + bumped_id = _cache_id(pins, install) + assert bumped_id != baseline + + with_sha = _cache_id(pins, install, extra={"GITHUB_SHA": "abc123", "GITHUB_RUN_ID": "11"}) + with_other = _cache_id(pins, install, extra={"GITHUB_SHA": "def456", "GITHUB_RUN_ID": "99"}) + assert with_sha == with_other == bumped_id + + +def test_native_cache_id_tracks_build_tools_and_compile_flags(tmp_path: Path) -> None: + baseline = _cache_id(PINS, INSTALL_SH) + tools = tmp_path / "appimage-tools.txt" + tools_text = TOOLS.read_text(encoding="utf-8") + tools.write_text(tools_text.replace("cmake", "cmake-other", 1), encoding="utf-8") + changed_tools = _cache_id(PINS, INSTALL_SH, extra={"VOCALINUX_APPIMAGE_TOOLS": str(tools)}) + assert changed_tools != baseline + + commented = tmp_path / "build-comment.sh" + build_text = BUILD_SH.read_text(encoding="utf-8") + commented.write_text( + build_text.replace("# GPU:", "# GPU: unchanged flags\n# GPU:", 1), encoding="utf-8" + ) + assert _cache_id(PINS, INSTALL_SH, extra={"VOCALINUX_BUILD_SH": str(commented)}) == baseline + + flagged = tmp_path / "build-flags.sh" + flagged.write_text( + build_text.replace( + "CMAKE_BUILD_WITH_INSTALL_RPATH=ON", + "CMAKE_BUILD_WITH_INSTALL_RPATH=OFF", + 1, + ), + encoding="utf-8", + ) + assert _cache_id(PINS, INSTALL_SH, extra={"VOCALINUX_BUILD_SH": str(flagged)}) != baseline + + +def test_native_cache_id_does_not_read_the_commit_or_the_run() -> None: + text = ID_SCRIPT.read_text(encoding="utf-8") + assert "base-image" in text + assert "PYWHISPERCPP_VERSION" in text + assert 'sha256sum "$pins"' in text + assert 'sha256sum "$tools"' in text + assert "appimage-tools.txt" in text + assert "CMAKE_ARGS=" in text + for banned in ( + "GITHUB_SHA", + "github.sha", + "GITHUB_RUN_ID", + "github.run_id", + "github.run_number", + "rev-parse", + ): + assert banned not in text, banned + + +def test_publish_refuses_a_cpu_only_tree_and_leaves_no_cache(tmp_path: Path) -> None: + site = tmp_path / "site" + site.mkdir() + (site / "pywhispercpp").mkdir() + (site / "_pywhispercpp.cpython-312-x86_64-linux-gnu.so").write_bytes(b"cpu") + dest = tmp_path / "cache" / "pywhispercpp-vulkan" + result = _source( + "pywhispercpp_cache_publish " + shlex.quote(str(site)) + " " + shlex.quote(str(dest)) + ) + assert result.returncode != 0 + assert "libggml-vulkan" in result.stderr + assert not dest.exists() + assert list((tmp_path / "cache").glob(".pywhispercpp-staging-*")) == [] + + +def test_publish_keeps_the_previous_cache_when_a_copy_fails(tmp_path: Path) -> None: + site = tmp_path / "built" + (site / "pywhispercpp").mkdir(parents=True) + (site / "pywhispercpp" / "secret").write_text("full", encoding="utf-8") + (site / "libggml-vulkan.so").write_bytes(b"new") + os.chmod(site / "pywhispercpp", 0) + dest = tmp_path / "cache" / "tree" + dest.mkdir(parents=True) + (dest / "libggml-vulkan.so").write_bytes(b"old-good") + try: + result = _source( + "pywhispercpp_cache_publish " + shlex.quote(str(site)) + " " + shlex.quote(str(dest)) + ) + finally: + os.chmod(site / "pywhispercpp", 0o755) + assert result.returncode != 0 + assert "Failed to copy" in result.stderr + assert (dest / "libggml-vulkan.so").read_bytes() == b"old-good" + assert list((tmp_path / "cache").glob(".pywhispercpp-staging-*")) == [] + + +def test_restore_refuses_a_cpu_tree_without_touching_the_prefix(tmp_path: Path) -> None: + src = tmp_path / "cache" + src.mkdir() + (src / "_pywhispercpp.cpython-312-x86_64-linux-gnu.so").write_bytes(b"cpu") + site = tmp_path / "site" + (site / "numpy").mkdir(parents=True) + (site / "numpy" / "keep.txt").write_text("stay", encoding="utf-8") + result = _source( + "pywhispercpp_cache_restore " + shlex.quote(str(src)) + " " + shlex.quote(str(site)) + ) + assert result.returncode != 0 + assert (site / "numpy" / "keep.txt").read_text(encoding="utf-8") == "stay" + assert (site / "_pywhispercpp.cpython-312-x86_64-linux-gnu.so").exists() is False + + +def test_publish_and_restore_round_trip_the_vulkan_tree_only(tmp_path: Path) -> None: + site = tmp_path / "built" + (site / "pywhispercpp").mkdir(parents=True) + (site / "pywhispercpp" / "marker.txt").write_text("vulkan-package", encoding="utf-8") + (site / "pywhispercpp-1.5.0.dist-info").mkdir() + (site / "pywhispercpp-1.5.0.dist-info" / "METADATA").write_text( + "Name: pywhispercpp\n", encoding="utf-8" + ) + (site / "_pywhispercpp.cpython-312-x86_64-linux-gnu.so").write_bytes(b"ext") + (site / "libggml-vulkan.so.0.9.8").write_bytes(b"vulkan-bytes") + (site / "libggml-vulkan.so").symlink_to("libggml-vulkan.so.0.9.8") + (site / "libwhisper.so.1").write_bytes(b"whisper") + (site / "numpy").mkdir() + (site / "numpy" / "nope.txt").write_text("not-native", encoding="utf-8") + + dest = tmp_path / "cache" / "tree" + published = _source( + "pywhispercpp_cache_publish " + shlex.quote(str(site)) + " " + shlex.quote(str(dest)) + ) + assert published.returncode == 0, published.stderr + assert not (dest / "numpy").exists() + assert (dest / "libggml-vulkan.so").is_symlink() + assert (dest / "libggml-vulkan.so").read_bytes() == b"vulkan-bytes" + + restored = tmp_path / "prefix" + (restored / "numpy").mkdir(parents=True) + (restored / "numpy" / "keep.txt").write_text("stay", encoding="utf-8") + (restored / "pywhispercpp").mkdir() + (restored / "pywhispercpp" / "marker.txt").write_text("cpu-package", encoding="utf-8") + (restored / "pywhispercpp.libs").mkdir() + (restored / "pywhispercpp.libs" / "cpu.so").write_bytes(b"cpu") + (restored / "_pywhispercpp.cpython-312-x86_64-linux-gnu.so").write_bytes(b"cpu-ext") + + result = _source( + "pywhispercpp_cache_restore " + shlex.quote(str(dest)) + " " + shlex.quote(str(restored)) + ) + assert result.returncode == 0, result.stderr + marker = (restored / "pywhispercpp" / "marker.txt").read_text(encoding="utf-8") + assert marker == "vulkan-package" + assert (restored / "libggml-vulkan.so").read_bytes() == b"vulkan-bytes" + assert (restored / "_pywhispercpp.cpython-312-x86_64-linux-gnu.so").read_bytes() == b"ext" + assert (restored / "libwhisper.so.1").read_bytes() == b"whisper" + assert not (restored / "pywhispercpp.libs").exists() + assert (restored / "numpy" / "keep.txt").read_text(encoding="utf-8") == "stay" + assert not (restored / "numpy" / "nope.txt").exists() + + +def _quote(path: Path) -> str: + return shlex.quote(str(path)) + + +def test_a_cache_hit_does_not_run_the_compile_command(tmp_path: Path) -> None: + site = tmp_path / "built" + (site / "pywhispercpp").mkdir(parents=True) + (site / "libggml-vulkan.so").write_bytes(b"vulkan-bytes") + cache = tmp_path / "cache" / "tree" + published = _source("pywhispercpp_cache_publish " + _quote(site) + " " + _quote(cache)) + assert published.returncode == 0, published.stderr + fresh = tmp_path / "prefix" + fresh.mkdir() + marker = tmp_path / "compiled" + reused = tmp_path / "reused" + result = _source( + "on_reuse() { printf reused > " + + _quote(reused) + + "; }\n" + + "compile() { printf compiled > " + + _quote(marker) + + "; }\n" + + "pywhispercpp_restore_or_run " + + _quote(cache) + + " " + + _quote(fresh) + + " on_reuse compile" + ) + assert result.returncode == 0, result.stderr + assert reused.read_text(encoding="utf-8") == "reused" + assert not marker.exists() + assert (fresh / "libggml-vulkan.so").read_bytes() == b"vulkan-bytes" + assert "Rebuilding pywhispercpp with Vulkan" not in result.stdout + + +def test_a_failed_restore_runs_the_compile_command(tmp_path: Path) -> None: + cache = tmp_path / "cache" + cache.mkdir() + (cache / "libggml-vulkan.so").write_bytes(b"vulkan-bytes") + blocker = tmp_path / "not-a-directory" + blocker.write_text("file", encoding="utf-8") + site = blocker / "site" + marker = tmp_path / "compiled" + reused = tmp_path / "reused" + result = _source( + "on_reuse() { printf reused > " + + _quote(reused) + + "; }\n" + + "compile() { printf compiled > " + + _quote(marker) + + "; }\n" + + "pywhispercpp_restore_or_run " + + _quote(cache) + + " " + + _quote(site) + + " on_reuse compile" + ) + assert result.returncode == 0, result.stderr + assert marker.read_text(encoding="utf-8") == "compiled" + assert not reused.exists() + assert "compiling again" in result.stderr + + +def test_native_cache_dir_binds_the_arch_to_that_id(tmp_path: Path) -> None: + env = os.environ.copy() + env["VOCALINUX_APPIMAGE_CACHE"] = str(tmp_path / "cache") + env.pop("VOCALINUX_PINS", None) + env.pop("VOCALINUX_INSTALL_SH", None) + env.pop("VOCALINUX_APPIMAGE_TOOLS", None) + env.pop("VOCALINUX_BUILD_SH", None) + result = _source("pywhispercpp_native_cache_dir x86_64", env=env) + assert result.returncode == 0, result.stderr + expected = _cache_id(PINS, INSTALL_SH) + assert Path(result.stdout.strip()) == ( + tmp_path / "cache" / f"pywhispercpp-vulkan-x86_64-{expected}" + ) + + arm = _source("pywhispercpp_native_cache_dir aarch64", env=env) + assert arm.returncode == 0, arm.stderr + assert Path(arm.stdout.strip()) == ( + tmp_path / "cache" / f"pywhispercpp-vulkan-aarch64-{expected}" + ) + + +def test_build_reuses_the_cached_vulkan_tree_before_compiling() -> None: + text = BUILD_SH.read_text(encoding="utf-8") + match = re.search(r"\nrebuild_pywhispercpp_vulkan\(\) \{(.+?)\n\}\n", text, re.S) + assert match, "rebuild_pywhispercpp_vulkan disappeared" + body = match.group(1) + assert body.index("VOCALINUX_APPIMAGE_SKIP_VULKAN") < body.index("pywhispercpp_restore_or_run") + assert "GGML_VULKAN=1" not in body + assert "_compile_pywhispercpp_vulkan" in body + compile_fn = re.search(r"\n_compile_pywhispercpp_vulkan\(\) \{(.+?)\n\}\n", text, re.S) + assert compile_fn, "the compile path disappeared" + compile_body = compile_fn.group(1) + assert "GGML_VULKAN=1" in compile_body + missing_lib = compile_body.index("did not produce libggml-vulkan.so") + assert missing_lib < compile_body.index("pywhispercpp_cache_publish") + assert "native-cache.sh" in text + + +def test_appimage_workflows_share_one_native_cache_key() -> None: + """CI, Nightly, and Release have to miss together when a pin changes, and + hit together otherwise. A key that includes the commit never hits.""" + keys = [] + for name in APPIMAGE_WORKFLOWS: + text = (WORKFLOWS / name).read_text(encoding="utf-8") + found = re.findall(r"path: ~/.cache/vocalinux-appimage\n\s+key: (.+)\n", text) + assert found, name + assert all(key == NATIVE_KEY for key in found), found + for token in ("github.sha", "github.run_id", "github.run_number", "github.run_attempt"): + assert token not in NATIVE_KEY + id_steps = re.findall( + r"- name: AppImage native cache id\n.*?(?=- name: Cache AppImage build tools)", + text, + re.S, + ) + assert len(id_steps) == len(found) + for step in id_steps: + assert 'id="$(bash packaging/appimage/native-cache-id.sh)"' in step + assert 'echo "id=${id}"' in step + for token in ("github.sha", "github.run_id", "github.run_number", "github.run_attempt"): + assert token not in step, step + assert "packaging/appimage/docker-build.sh" in text + assert 'VOCALINUX_APPIMAGE_REQUIRE_VULKAN: "1"' in text + assert "aarch64" in text + keys.extend(found) + assert len(keys) == 5 + assert len(set(keys)) == 1