voculinux/packaging/void/container-release.sh

#!/bin/sh
# Container half of packaging/void/build-release.sh (tickets #28 and #29).
# Runs inside the pinned Void glibc image, once per phase. Each phase gets only
# the privileges it needs:
#
#   build    root, network on, NO signing key. Clones void-packages at the
#            pinned commit, renders the voculinux build tree from the tag, and
#            builds every template in smoke-gate order as the unprivileged
#            builder user (xbps-src refuses root). Build scripts run here, so
#            the key is never mounted in this phase.
#   sign     root, NO network, the RSA4096 key mounted read-only. Runs no
#            build code: it indexes and signs the packages the build produced.
#   install  root, network on (the official repository supplies dependencies),
#            NO key. In a clean image, installs voculinux from the staged signed
#            repository and checks what a user sees: the version, the payload,
#            the imports, and that the repository is refused until its key is
#            trusted.
#
# Not executed directly: build-release.sh mounts this file and runs it with one
# phase argument. Inputs arrive as environment variables set there.
set -eu

export PATH=/usr/bin:/bin:/usr/sbin:/sbin
KEY=/run/xbps-sign-key.pem
PKG_STAGE=/mnt/packages
XBPS_STAGE=/mnt/xbps
OFFICIAL=https://repo-default.voidlinux.org/current

fail() {
	echo "container-release.sh: $*" >&2
	exit 1
}

phase_build() {
	: "${VOID_COMMIT:?}" "${PKGS:?}" "${XBPS_VERSION:?}" "${SDIST_NAME:?}"
	export VOID_COMMIT PKGS XBPS_VERSION SDIST_NAME

	xbps-install -Suy xbps >/dev/null
	xbps-install -y git base-devel xtools binutils zstd openssl >/dev/null
	git clone --quiet --depth 1 https://github.com/void-linux/void-packages /void-packages
	git -C /void-packages fetch --quiet --depth 1 origin "$VOID_COMMIT"
	git -C /void-packages checkout --quiet "$VOID_COMMIT"
	cp -r /overlay/. /void-packages/srcpkgs/
	id builder >/dev/null 2>&1 || useradd -m builder

	# Build tree only: the committed template in the repository is never changed.
	for PKG in $PKGS; do
		if [ "$PKG" = voculinux ]; then
			VOID_PACKAGES=/void-packages SDIST_DIR=/mnt/inputs sh /mnt/render-voculinux.sh
		fi
	done

	chown -R builder:builder /void-packages
	su -s /bin/sh builder -c 'cd /void-packages && ./xbps-src binary-bootstrap'
	for PKG in $PKGS; do
		echo "== xlint $PKG"
		su -s /bin/sh builder -c "cd /void-packages && xlint 'srcpkgs/$PKG/template'"
		echo "== xbps-src pkg $PKG"
		su -s /bin/sh builder -c "cd /void-packages && ./xbps-src pkg '$PKG'"
	done

	# The distfile the template names must be the sdist staged for it.
	su -s /bin/sh builder -c 'cd /void-packages && ./xbps-src show voculinux' >/tmp/show.txt 2>&1 ||
		fail "xbps-src show voculinux failed"
	grep -qF "/$SDIST_NAME" /tmp/show.txt || fail "xbps-src does not resolve distfile $SDIST_NAME"

	mkdir -p "$PKG_STAGE"
	for PKG in $PKGS; do
		PKGFILE=$(ls /void-packages/hostdir/binpkgs/"$PKG"-[0-9]*.x86_64.xbps 2>/dev/null | head -n1)
		[ -n "$PKGFILE" ] || fail "built package missing from hostdir/binpkgs: $PKG"
		echo "collected: $(basename "$PKGFILE")"
		cp "$PKGFILE" "$PKG_STAGE/"
	done
}

phase_sign() {
	: "${SIGNEDBY:?}"
	[ -r "$KEY" ] && [ -s "$KEY" ] || fail "signing key is not mounted at $KEY"
	mkdir -p "$XBPS_STAGE/repo"
	cp "$PKG_STAGE"/*.x86_64.xbps "$XBPS_STAGE/repo/"

	echo "== index the full repository"
	xbps-rindex -a "$XBPS_STAGE"/repo/*.x86_64.xbps
	echo "== sign every binary package"
	xbps-rindex -S --privkey "$KEY" --signedby "$SIGNEDBY" "$XBPS_STAGE"/repo/*.x86_64.xbps
	echo "== sign the repository metadata"
	xbps-rindex -s --privkey "$KEY" --signedby "$SIGNEDBY" "$XBPS_STAGE/repo"
	echo "signed repository staged: $(ls "$XBPS_STAGE/repo" | wc -l) files"
}

# The payload checks the package must pass before anyone installs it.
payload_voculinux() {
	PKGFILE=$1
	tar -tf "$PKGFILE" | grep -q "usr/share/applications/voculinux.desktop" ||
		fail "renamed desktop entry missing from the package"
	tar -tf "$PKGFILE" | grep -q "usr/share/icons/hicolor/scalable/apps/voculinux.svg" ||
		fail "renamed icon missing from the package"
	tar -tf "$PKGFILE" | grep -q "usr/share/voculinux/packaged" ||
		fail "packaged-install marker missing from the package"
	if tar -tf "$PKGFILE" | grep -E "models/|\.ggml$|\.bin$|\.pt$|\.whl$"; then
		fail "package payload carries models or wheels"
	fi
	if tar -tf "$PKGFILE" | grep -E "usr/share/applications/vocalinux|usr/share/icons/.*/vocalinux"; then
		fail "pre-rename file names in the OS-visible payload"
	fi
	echo "payload ok: desktop entry, icon and marker present; no models, wheels or pre-rename names"
}

# whisper.cpp ships its Vulkan backend and its CPU backend as runtime-loaded
# variants beside the extension (GGML_BACKEND_DL, #44). Every CPU variant
# carries its own instruction set, but the x86-64 baseline variant must hold no
# AVX code (ymm/zmm), the AVX2 variant must hold some (so the variants are real,
# not one baseline build), and every other shipped native library (the ggml
# base, the registry, whisper, the extension, Vulkan) must stay AVX-free. No
# library may carry host-native tuning.
payload_pywhispercpp() {
	PKGFILE=$1
	tar -tf "$PKGFILE" | grep -q "libggml-vulkan" || fail "the Vulkan backend is missing from $PKGFILE"
	for VARIANT in x64 haswell; do
		tar -tf "$PKGFILE" | grep -q "libggml-cpu-$VARIANT\.so" ||
			fail "CPU variant libggml-cpu-$VARIANT.so is missing from $PKGFILE"
	done
	rm -rf /flagscan && mkdir -p /flagscan
	tar -xf "$PKGFILE" -C /flagscan --wildcards '*.so*'
	while IFS= read -r LIBSCAN; do
		NAME=${LIBSCAN##*/}
		if readelf -p .GCC.command.line "$LIBSCAN" 2>/dev/null | grep -E "march=native|mcpu=native|mtune=native"; then
			fail "host-native CPU tuning in $NAME"
		fi
		case "$NAME" in
		libggml-cpu-*) ;; # each variant is checked below
		*)
			if objdump -d "$LIBSCAN" | grep -Eq '%[yz]mm[0-9]'; then
				fail "$NAME contains AVX code; only the CPU variants may"
			fi
			;;
		esac
	done <<LIBS
$(find /flagscan -type f \( -name '*.so' -o -name '*.so.*' \))
LIBS
	BASELINE=$(find /flagscan -type f -name 'libggml-cpu-x64.so')
	AVX2=$(find /flagscan -type f -name 'libggml-cpu-haswell.so')
	[ -n "$BASELINE" ] && [ -n "$AVX2" ] || fail "the CPU variants were not extracted from $PKGFILE"
	if objdump -d "$BASELINE" | grep -Eq '%[yz]mm[0-9]'; then
		fail "the x86-64 baseline CPU variant contains AVX code; it would not run on every x86-64 CPU"
	fi
	objdump -d "$AVX2" | grep -Eq '%ymm[0-9]' || fail "the AVX2 CPU variant has no AVX2 code"
	echo "payload ok: Vulkan backend; CPU variants present; baseline and other libraries free of AVX; no host-native tuning"
}

phase_install() {
	: "${PKGS:?}" "${XBPS_VERSION:?}" "${VERSION:?}"
	[ -d "$XBPS_STAGE/repo" ] && [ -d "$XBPS_STAGE/keys" ] || fail "staged repository or keys are missing"
	xbps-install -Sy xbps binutils zstd >/dev/null

	echo "== the signed repository is refused while its key is not trusted"
	# No -y: the documented first sync must not accept an unknown key.
	set +e
	REFUSAL=$(xbps-install -S --repository="$XBPS_STAGE/repo" voculinux </dev/null 2>&1)
	REFUSAL_RC=$?
	set -e
	printf '%s\n' "$REFUSAL" | grep -i "fingerprint" || true
	[ "$REFUSAL_RC" -ne 0 ] || fail "the repository installed without its key being trusted"
	xbps-query voculinux >/dev/null 2>&1 && fail "voculinux was installed by the untrusted attempt"
	echo "refused without the key, as required (rc=$REFUSAL_RC)"

	echo "== trust the staged key"
	cp "$XBPS_STAGE"/keys/*.plist /var/db/xbps/keys/

	echo "== stock Void packages below the project floors, as a user who has them already"
	xbps-install -y --repository="$OFFICIAL" -S python3-PyAudio python3-pysocks >/dev/null

	echo "== install voculinux from the signed repository; the floors must upgrade the stock packages"
	xbps-install -y --repository="$XBPS_STAGE/repo" --repository="$OFFICIAL" -S voculinux >/dev/null
	xbps-query voculinux | grep -qx "pkgver: voculinux-${XBPS_VERSION}_1" ||
		fail "installed package is not voculinux-${XBPS_VERSION}_1"
	echo "installed: voculinux-${XBPS_VERSION}_1"
	for PKG in python3-PyAudio python3-pysocks; do
		PKGFILE=$(ls "$PKG_STAGE"/"$PKG"-[0-9]*.x86_64.xbps | head -n1)
		PKGVER=$(basename "$PKGFILE" .x86_64.xbps)
		xbps-query "$PKG" | grep -qx "pkgver: $PKGVER" ||
			fail "$PKG is not $PKGVER after voculinux: the stock package was not upgraded"
		echo "upgraded to the carried template: $PKGVER"
	done

	echo "== the renamed binary reports its version"
	VERSION_OUT=$(voculinux --version 2>&1) || fail "voculinux --version exited non-zero"
	printf '%s\n' "$VERSION_OUT"
	printf '%s' "$VERSION_OUT" | grep -qF "$VERSION" || fail "voculinux --version does not show $VERSION"

	echo "== payload of the built packages; every package needs a smoke recipe"
	for PKG in $PKGS; do
		PKGFILE=$(ls "$PKG_STAGE"/"$PKG"-[0-9]*.x86_64.xbps | head -n1)
		case "$PKG" in
		voculinux) payload_voculinux "$PKGFILE" ;;
		python3-pywhispercpp) payload_pywhispercpp "$PKGFILE" ;;
		python3-pynput | python3-PyAudio | python3-pysocks) ;; # checked by import below
		*) fail "no smoke recipe for package $PKG; add one alongside its template" ;;
		esac
	done

	echo "== imports, without a display or hardware"
	# pynput: the dummy backend proves the package and its loader. The X backend
	# needs an X server and is covered by the manual hardware checklist.
	PYNPUT_BACKEND=dummy python3 -c "import pynput.keyboard, pynput.mouse; print('pynput dummy backend ok')"
	python3 -c "import evdev; print('evdev ok')"
	python3 -c "import gi; gi.require_version('Gtk', '3.0'); from gi.repository import Gtk; print('gi ok: Gtk', Gtk.get_major_version(), Gtk.get_minor_version())"
	python3 -c "from pywhispercpp.model import Model; print('pywhispercpp.model ok')"
	echo "== pywhispercpp reports the version it is pinned to (#43)"
	PWC_VERSION=$(python3 -c "import importlib.metadata as m; print(m.version('pywhispercpp'))") ||
		fail "pywhispercpp has no installed version"
	[ "$PWC_VERSION" = "1.5.0" ] || fail "pywhispercpp reports $PWC_VERSION, not 1.5.0 (#43)"
	echo "pywhispercpp version: $PWC_VERSION"
	echo "== the extension loads its CPU and Vulkan backends from the package (#44)"
	python3 - <<'PY' || fail "the dynamic ggml backends did not load from the package"
import ctypes
import os

import _pywhispercpp
import pywhispercpp.model  # noqa: F401  (importing the extension loads the backends)

libdir = os.path.dirname(_pywhispercpp.__file__)
base = ctypes.CDLL(os.path.join(libdir, "libggml.so.0"))
base.ggml_backend_reg_count.restype = ctypes.c_size_t
base.ggml_backend_reg_get.argtypes = [ctypes.c_size_t]
base.ggml_backend_reg_get.restype = ctypes.c_void_p
base.ggml_backend_reg_name.argtypes = [ctypes.c_void_p]
base.ggml_backend_reg_name.restype = ctypes.c_char_p
names = [
    base.ggml_backend_reg_name(base.ggml_backend_reg_get(i)).decode()
    for i in range(base.ggml_backend_reg_count())
]
print("ggml backends registered:", ", ".join(names))
assert "CPU" in names, names
loaded = set()
for line in open("/proc/self/maps"):
    parts = line.split()
    if len(parts) >= 6 and "libggml-" in parts[-1]:
        loaded.add(os.path.basename(parts[-1]))
cpu = sorted(name for name in loaded if name.startswith("libggml-cpu-"))
assert cpu, "no CPU variant was loaded"
print("CPU variant loaded:", ", ".join(cpu))
# The Vulkan module registers only when a GPU driver answers, and this image
# has none, so prove what it needs instead: its NEEDED libraries (libvulkan,
# the ggml base) resolve from the installed layout.
ctypes.CDLL(os.path.join(libdir, "libggml-vulkan.so"))
print("Vulkan backend library loads from the package")
PY
	python3 -c "import pyaudio; print('pyaudio ok')"
	python3 -c "import socks; print('pysocks ok')"
	python3 -c "import vocalinux.main, vocalinux.speech_recognition.recognition_manager; print('vocalinux imports ok')"
	echo "INSTALL SMOKE OK: voculinux $VERSION installed from the signed repository"
}

case "${1:-}" in
build) phase_build ;;
sign) phase_sign ;;
install) phase_install ;;
*) fail "usage: container-release.sh build|sign|install" ;;
esac