261 lines
12 KiB
Bash
Executable File
261 lines
12 KiB
Bash
Executable File
#!/bin/sh
|
|
# Container half of packaging/void/build-release.sh (tickets #28 and #29).
|
|
# Runs inside the pinned Void glibc image, once per phase. Each phase gets only
|
|
# the privileges it needs:
|
|
#
|
|
# build root, network on, NO signing key. Clones void-packages at the
|
|
# pinned commit, renders the voculinux build tree from the tag, and
|
|
# builds every template in smoke-gate order as the unprivileged
|
|
# builder user (xbps-src refuses root). Build scripts run here, so
|
|
# the key is never mounted in this phase.
|
|
# sign root, NO network, the RSA4096 key mounted read-only. Runs no
|
|
# build code: it indexes and signs the packages the build produced.
|
|
# install root, network on (the official repository supplies dependencies),
|
|
# NO key. In a clean image, installs voculinux from the staged signed
|
|
# repository and checks what a user sees: the version, the payload,
|
|
# the imports, and that the repository is refused until its key is
|
|
# trusted.
|
|
#
|
|
# Not executed directly: build-release.sh mounts this file and runs it with one
|
|
# phase argument. Inputs arrive as environment variables set there.
|
|
set -eu
|
|
|
|
export PATH=/usr/bin:/bin:/usr/sbin:/sbin
|
|
KEY=/run/xbps-sign-key.pem
|
|
PKG_STAGE=/mnt/packages
|
|
XBPS_STAGE=/mnt/xbps
|
|
OFFICIAL=https://repo-default.voidlinux.org/current
|
|
|
|
fail() {
|
|
echo "container-release.sh: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
phase_build() {
|
|
: "${VOID_COMMIT:?}" "${PKGS:?}" "${XBPS_VERSION:?}" "${SDIST_NAME:?}"
|
|
export VOID_COMMIT PKGS XBPS_VERSION SDIST_NAME
|
|
|
|
xbps-install -Suy xbps >/dev/null
|
|
xbps-install -y git base-devel xtools binutils zstd openssl >/dev/null
|
|
git clone --quiet --depth 1 https://github.com/void-linux/void-packages /void-packages
|
|
git -C /void-packages fetch --quiet --depth 1 origin "$VOID_COMMIT"
|
|
git -C /void-packages checkout --quiet "$VOID_COMMIT"
|
|
cp -r /overlay/. /void-packages/srcpkgs/
|
|
id builder >/dev/null 2>&1 || useradd -m builder
|
|
|
|
# Build tree only: the committed template in the repository is never changed.
|
|
for PKG in $PKGS; do
|
|
if [ "$PKG" = voculinux ]; then
|
|
VOID_PACKAGES=/void-packages SDIST_DIR=/mnt/inputs sh /mnt/render-voculinux.sh
|
|
fi
|
|
done
|
|
|
|
chown -R builder:builder /void-packages
|
|
su -s /bin/sh builder -c 'cd /void-packages && ./xbps-src binary-bootstrap'
|
|
for PKG in $PKGS; do
|
|
echo "== xlint $PKG"
|
|
su -s /bin/sh builder -c "cd /void-packages && xlint 'srcpkgs/$PKG/template'"
|
|
echo "== xbps-src pkg $PKG"
|
|
su -s /bin/sh builder -c "cd /void-packages && ./xbps-src pkg '$PKG'"
|
|
done
|
|
|
|
# The distfile the template names must be the sdist staged for it.
|
|
su -s /bin/sh builder -c 'cd /void-packages && ./xbps-src show voculinux' >/tmp/show.txt 2>&1 ||
|
|
fail "xbps-src show voculinux failed"
|
|
grep -qF "/$SDIST_NAME" /tmp/show.txt || fail "xbps-src does not resolve distfile $SDIST_NAME"
|
|
|
|
mkdir -p "$PKG_STAGE"
|
|
for PKG in $PKGS; do
|
|
PKGFILE=$(ls /void-packages/hostdir/binpkgs/"$PKG"-[0-9]*.x86_64.xbps 2>/dev/null | head -n1)
|
|
[ -n "$PKGFILE" ] || fail "built package missing from hostdir/binpkgs: $PKG"
|
|
echo "collected: $(basename "$PKGFILE")"
|
|
cp "$PKGFILE" "$PKG_STAGE/"
|
|
done
|
|
}
|
|
|
|
phase_sign() {
|
|
: "${SIGNEDBY:?}"
|
|
[ -r "$KEY" ] && [ -s "$KEY" ] || fail "signing key is not mounted at $KEY"
|
|
mkdir -p "$XBPS_STAGE/repo"
|
|
cp "$PKG_STAGE"/*.x86_64.xbps "$XBPS_STAGE/repo/"
|
|
|
|
echo "== index the full repository"
|
|
xbps-rindex -a "$XBPS_STAGE"/repo/*.x86_64.xbps
|
|
echo "== sign every binary package"
|
|
xbps-rindex -S --privkey "$KEY" --signedby "$SIGNEDBY" "$XBPS_STAGE"/repo/*.x86_64.xbps
|
|
echo "== sign the repository metadata"
|
|
xbps-rindex -s --privkey "$KEY" --signedby "$SIGNEDBY" "$XBPS_STAGE/repo"
|
|
echo "signed repository staged: $(ls "$XBPS_STAGE/repo" | wc -l) files"
|
|
}
|
|
|
|
# The payload checks the package must pass before anyone installs it.
|
|
payload_voculinux() {
|
|
PKGFILE=$1
|
|
tar -tf "$PKGFILE" | grep -q "usr/share/applications/voculinux.desktop" ||
|
|
fail "renamed desktop entry missing from the package"
|
|
tar -tf "$PKGFILE" | grep -q "usr/share/icons/hicolor/scalable/apps/voculinux.svg" ||
|
|
fail "renamed icon missing from the package"
|
|
tar -tf "$PKGFILE" | grep -q "usr/share/voculinux/packaged" ||
|
|
fail "packaged-install marker missing from the package"
|
|
if tar -tf "$PKGFILE" | grep -E "models/|\.ggml$|\.bin$|\.pt$|\.whl$"; then
|
|
fail "package payload carries models or wheels"
|
|
fi
|
|
if tar -tf "$PKGFILE" | grep -E "usr/share/applications/vocalinux|usr/share/icons/.*/vocalinux"; then
|
|
fail "pre-rename file names in the OS-visible payload"
|
|
fi
|
|
echo "payload ok: desktop entry, icon and marker present; no models, wheels or pre-rename names"
|
|
}
|
|
|
|
# whisper.cpp ships its Vulkan backend and its CPU backend as runtime-loaded
|
|
# variants beside the extension (GGML_BACKEND_DL, #44). Every CPU variant
|
|
# carries its own instruction set, but the x86-64 baseline variant must hold no
|
|
# AVX code (ymm/zmm), the AVX2 variant must hold some (so the variants are real,
|
|
# not one baseline build), and every other shipped native library (the ggml
|
|
# base, the registry, whisper, the extension, Vulkan) must stay AVX-free. No
|
|
# library may carry host-native tuning.
|
|
payload_pywhispercpp() {
|
|
PKGFILE=$1
|
|
tar -tf "$PKGFILE" | grep -q "libggml-vulkan" || fail "the Vulkan backend is missing from $PKGFILE"
|
|
for VARIANT in x64 haswell; do
|
|
tar -tf "$PKGFILE" | grep -q "libggml-cpu-$VARIANT\.so" ||
|
|
fail "CPU variant libggml-cpu-$VARIANT.so is missing from $PKGFILE"
|
|
done
|
|
rm -rf /flagscan && mkdir -p /flagscan
|
|
tar -xf "$PKGFILE" -C /flagscan --wildcards '*.so*'
|
|
while IFS= read -r LIBSCAN; do
|
|
NAME=${LIBSCAN##*/}
|
|
if readelf -p .GCC.command.line "$LIBSCAN" 2>/dev/null | grep -E "march=native|mcpu=native|mtune=native"; then
|
|
fail "host-native CPU tuning in $NAME"
|
|
fi
|
|
case "$NAME" in
|
|
libggml-cpu-*) ;; # each variant is checked below
|
|
*)
|
|
if objdump -d "$LIBSCAN" | grep -Eq '%[yz]mm[0-9]'; then
|
|
fail "$NAME contains AVX code; only the CPU variants may"
|
|
fi
|
|
;;
|
|
esac
|
|
done <<LIBS
|
|
$(find /flagscan -type f \( -name '*.so' -o -name '*.so.*' \))
|
|
LIBS
|
|
BASELINE=$(find /flagscan -type f -name 'libggml-cpu-x64.so')
|
|
AVX2=$(find /flagscan -type f -name 'libggml-cpu-haswell.so')
|
|
[ -n "$BASELINE" ] && [ -n "$AVX2" ] || fail "the CPU variants were not extracted from $PKGFILE"
|
|
if objdump -d "$BASELINE" | grep -Eq '%[yz]mm[0-9]'; then
|
|
fail "the x86-64 baseline CPU variant contains AVX code; it would not run on every x86-64 CPU"
|
|
fi
|
|
objdump -d "$AVX2" | grep -Eq '%ymm[0-9]' || fail "the AVX2 CPU variant has no AVX2 code"
|
|
echo "payload ok: Vulkan backend; CPU variants present; baseline and other libraries free of AVX; no host-native tuning"
|
|
}
|
|
|
|
phase_install() {
|
|
: "${PKGS:?}" "${XBPS_VERSION:?}" "${VERSION:?}"
|
|
[ -d "$XBPS_STAGE/repo" ] && [ -d "$XBPS_STAGE/keys" ] || fail "staged repository or keys are missing"
|
|
xbps-install -Sy xbps binutils zstd >/dev/null
|
|
|
|
echo "== the signed repository is refused while its key is not trusted"
|
|
# No -y: the documented first sync must not accept an unknown key.
|
|
set +e
|
|
REFUSAL=$(xbps-install -S --repository="$XBPS_STAGE/repo" voculinux </dev/null 2>&1)
|
|
REFUSAL_RC=$?
|
|
set -e
|
|
printf '%s\n' "$REFUSAL" | grep -i "fingerprint" || true
|
|
[ "$REFUSAL_RC" -ne 0 ] || fail "the repository installed without its key being trusted"
|
|
xbps-query voculinux >/dev/null 2>&1 && fail "voculinux was installed by the untrusted attempt"
|
|
echo "refused without the key, as required (rc=$REFUSAL_RC)"
|
|
|
|
echo "== trust the staged key"
|
|
cp "$XBPS_STAGE"/keys/*.plist /var/db/xbps/keys/
|
|
|
|
echo "== stock Void packages below the project floors, as a user who has them already"
|
|
xbps-install -y --repository="$OFFICIAL" -S python3-PyAudio python3-pysocks >/dev/null
|
|
|
|
echo "== install voculinux from the signed repository; the floors must upgrade the stock packages"
|
|
xbps-install -y --repository="$XBPS_STAGE/repo" --repository="$OFFICIAL" -S voculinux >/dev/null
|
|
xbps-query voculinux | grep -qx "pkgver: voculinux-${XBPS_VERSION}_1" ||
|
|
fail "installed package is not voculinux-${XBPS_VERSION}_1"
|
|
echo "installed: voculinux-${XBPS_VERSION}_1"
|
|
for PKG in python3-PyAudio python3-pysocks; do
|
|
PKGFILE=$(ls "$PKG_STAGE"/"$PKG"-[0-9]*.x86_64.xbps | head -n1)
|
|
PKGVER=$(basename "$PKGFILE" .x86_64.xbps)
|
|
xbps-query "$PKG" | grep -qx "pkgver: $PKGVER" ||
|
|
fail "$PKG is not $PKGVER after voculinux: the stock package was not upgraded"
|
|
echo "upgraded to the carried template: $PKGVER"
|
|
done
|
|
|
|
echo "== the renamed binary reports its version"
|
|
VERSION_OUT=$(voculinux --version 2>&1) || fail "voculinux --version exited non-zero"
|
|
printf '%s\n' "$VERSION_OUT"
|
|
printf '%s' "$VERSION_OUT" | grep -qF "$VERSION" || fail "voculinux --version does not show $VERSION"
|
|
|
|
echo "== payload of the built packages; every package needs a smoke recipe"
|
|
for PKG in $PKGS; do
|
|
PKGFILE=$(ls "$PKG_STAGE"/"$PKG"-[0-9]*.x86_64.xbps | head -n1)
|
|
case "$PKG" in
|
|
voculinux) payload_voculinux "$PKGFILE" ;;
|
|
python3-pywhispercpp) payload_pywhispercpp "$PKGFILE" ;;
|
|
python3-pynput | python3-PyAudio | python3-pysocks) ;; # checked by import below
|
|
*) fail "no smoke recipe for package $PKG; add one alongside its template" ;;
|
|
esac
|
|
done
|
|
|
|
echo "== imports, without a display or hardware"
|
|
# pynput: the dummy backend proves the package and its loader. The X backend
|
|
# needs an X server and is covered by the manual hardware checklist.
|
|
PYNPUT_BACKEND=dummy python3 -c "import pynput.keyboard, pynput.mouse; print('pynput dummy backend ok')"
|
|
python3 -c "import evdev; print('evdev ok')"
|
|
python3 -c "import gi; gi.require_version('Gtk', '3.0'); from gi.repository import Gtk; print('gi ok: Gtk', Gtk.get_major_version(), Gtk.get_minor_version())"
|
|
python3 -c "from pywhispercpp.model import Model; print('pywhispercpp.model ok')"
|
|
echo "== pywhispercpp reports the version it is pinned to (#43)"
|
|
PWC_VERSION=$(python3 -c "import importlib.metadata as m; print(m.version('pywhispercpp'))") ||
|
|
fail "pywhispercpp has no installed version"
|
|
[ "$PWC_VERSION" = "1.5.0" ] || fail "pywhispercpp reports $PWC_VERSION, not 1.5.0 (#43)"
|
|
echo "pywhispercpp version: $PWC_VERSION"
|
|
echo "== the extension loads its CPU and Vulkan backends from the package (#44)"
|
|
python3 - <<'PY' || fail "the dynamic ggml backends did not load from the package"
|
|
import ctypes
|
|
import os
|
|
|
|
import _pywhispercpp
|
|
import pywhispercpp.model # noqa: F401 (importing the extension loads the backends)
|
|
|
|
libdir = os.path.dirname(_pywhispercpp.__file__)
|
|
base = ctypes.CDLL(os.path.join(libdir, "libggml.so.0"))
|
|
base.ggml_backend_reg_count.restype = ctypes.c_size_t
|
|
base.ggml_backend_reg_get.argtypes = [ctypes.c_size_t]
|
|
base.ggml_backend_reg_get.restype = ctypes.c_void_p
|
|
base.ggml_backend_reg_name.argtypes = [ctypes.c_void_p]
|
|
base.ggml_backend_reg_name.restype = ctypes.c_char_p
|
|
names = [
|
|
base.ggml_backend_reg_name(base.ggml_backend_reg_get(i)).decode()
|
|
for i in range(base.ggml_backend_reg_count())
|
|
]
|
|
print("ggml backends registered:", ", ".join(names))
|
|
assert "CPU" in names, names
|
|
loaded = set()
|
|
for line in open("/proc/self/maps"):
|
|
parts = line.split()
|
|
if len(parts) >= 6 and "libggml-" in parts[-1]:
|
|
loaded.add(os.path.basename(parts[-1]))
|
|
cpu = sorted(name for name in loaded if name.startswith("libggml-cpu-"))
|
|
assert cpu, "no CPU variant was loaded"
|
|
print("CPU variant loaded:", ", ".join(cpu))
|
|
# The Vulkan module registers only when a GPU driver answers, and this image
|
|
# has none, so prove what it needs instead: its NEEDED libraries (libvulkan,
|
|
# the ggml base) resolve from the installed layout.
|
|
ctypes.CDLL(os.path.join(libdir, "libggml-vulkan.so"))
|
|
print("Vulkan backend library loads from the package")
|
|
PY
|
|
python3 -c "import pyaudio; print('pyaudio ok')"
|
|
python3 -c "import socks; print('pysocks ok')"
|
|
python3 -c "import vocalinux.main, vocalinux.speech_recognition.recognition_manager; print('vocalinux imports ok')"
|
|
echo "INSTALL SMOKE OK: voculinux $VERSION installed from the signed repository"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
build) phase_build ;;
|
|
sign) phase_sign ;;
|
|
install) phase_install ;;
|
|
*) fail "usage: container-release.sh build|sign|install" ;;
|
|
esac
|