xavierk/voculinux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

chore: import vocalinux project
CI / security (pull_request) Failing after 1m25s
Details
CI / lint (pull_request) Failing after 34s
Details
CI / ai-review (pull_request) Successful in 25s
Details

Browse Source 
Import the vocalinux source tree (tray app, Next.js site in web/,
packaging, installers, docs and tests) on top of the Gitea template
commit. The project README replaces the template README, and the
template CI files (.gitea/workflows/ci.yml, .jscpd.json, CI.md) are kept.
This commit is contained in:
xavierk
2026-11-06 11:51:44 +05:30
parent 4a90b559ca
commit 80a3792576
597 changed files with 206764 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build_reproducible_wheel.py
Executable
+331
View File
@@ -0,0 +1,331 @@
#!/usr/bin/env python3
"""Build independent wheels with locked tooling and stage only verified distributions.
The source snapshot includes tracked working-tree edits and non-ignored new files,
so local review exercises uncommitted changes too. Each build gets a separate copy
and virtualenv; no build/ or egg-info from the developer's checkout is reused.
Reproducibility is checked for the same Python/platform/toolchain, not across OSes.
"""
from __future__ import annotations
import argparse
import fcntl
import hashlib
import os
import shutil
import subprocess
import sys
import tempfile
import zipfile
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def run(argv: list[str], cwd: Path, env: dict[str, str], log: Path) -> None:
"""Run a build step, retaining its output even when the step fails."""
with log.open("a", encoding="utf-8") as stream:
print(f"$ {' '.join(argv)}", file=stream, flush=True)
result = subprocess.run(argv, cwd=cwd, env=env, stdout=stream, stderr=stream)
if result.returncode:
raise RuntimeError(f"command failed ({result.returncode}): {' '.join(argv)}; see {log}")
def snapshot(source: Path, target: Path) -> None:
"""Copy Git-visible source files, excluding ignored build and venv state."""
files = subprocess.check_output(
["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], cwd=source
)
for name in sorted(set(files.decode().split("\0")) - {""}):
origin = source / name
if not origin.exists() and not origin.is_symlink():
continue # A tracked file deleted in the working tree.
destination = target / name
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(origin, destination, follow_symlinks=False)
def one_file(directory: Path, pattern: str) -> Path:
"""Require one output, rather than accidentally comparing stale artifacts."""
files = sorted(directory.glob(pattern))
if len(files) != 1:
raise RuntimeError(f"expected exactly one {pattern} in {directory}, found {len(files)}")
return files[0]
def digest(path: Path) -> str:
"""Return the SHA-256 of the original archive bytes."""
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def compare_wheels(first: Path, second: Path, report: Path) -> None:
"""Reject any byte difference, including ZIP metadata, and explain the mismatch."""
lines = [f"{digest(path)} {path}" for path in (first, second)]
# Compare bytes, not re-packed/normalized ZIPs: metadata is part of the wheel.
equal = first.name == second.name and first.read_bytes() == second.read_bytes()
if not equal:
lines.append("FAIL: wheel filenames or bytes differ")
if first.name != second.name:
lines.append(f"filenames: {first.name} != {second.name}")
try:
with zipfile.ZipFile(first) as left, zipfile.ZipFile(second) as right:
if left.namelist() != right.namelist():
lines.append("ZIP member names/order differ")
for name in sorted(set(left.namelist()) | set(right.namelist())):
if name not in left.namelist() or name not in right.namelist():
lines.append(f"member present in only one wheel: {name}")
elif left.read(name) != right.read(name):
lines.append(f"member content differs: {name}")
lines.append("Inspect ZIP timestamps, permissions, compression and headers too.")
except zipfile.BadZipFile as error:
lines.append(f"invalid wheel archive: {error}")
else:
lines.append("PASS: independently built wheels are byte-for-byte identical")
report.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(report.read_text(encoding="utf-8"), end="")
if not equal:
raise RuntimeError(f"wheel reproducibility failed; see {report}")
def prepare_outputs(output: Path, diagnostics: Path) -> None:
"""Remove only recognizable gate outputs, validating everything before deletion.
The retained canonical archives identify staged files by name AND digest.
Never recursively wipe build/ or dist/: unrelated/modified files require the
user to move them aside. Failed builds can be retried even with partial logs
and archives, provided no unexpected files are present.
"""
diagnostic_files: list[Path] = []
staged_files: list[Path] = []
directories: list[Path] = []
archives: dict[str, Path] = {}
resolved_output = output.resolve()
resolved_diagnostics = diagnostics.resolve()
if (
resolved_output == resolved_diagnostics
or resolved_output in resolved_diagnostics.parents
or resolved_diagnostics in resolved_output.parents
):
raise RuntimeError("output and diagnostics directories must not overlap")
for directory in (output, diagnostics):
for component in (directory, *directory.parents):
if component.is_symlink():
raise RuntimeError(f"refusing to clean through symlink: {component}")
if directory.exists() and not directory.is_dir():
raise RuntimeError(f"refusing to clean non-directory: {directory}")
if diagnostics.exists():
for entry in diagnostics.iterdir():
if entry.is_symlink():
raise RuntimeError(f"refusing to clean symlink: {entry}")
if entry.name in {"canonical", "independent"} and entry.is_dir():
for artifact in entry.iterdir():
if (
artifact.is_symlink()
or not artifact.is_file()
or not artifact.name.endswith((".whl", ".tar.gz"))
):
raise RuntimeError(
f"unrecognized diagnostic output; move aside: {artifact}"
)
diagnostic_files.append(artifact)
if entry.name == "canonical":
archives[artifact.name] = artifact
directories.append(entry)
elif entry.is_file() and entry.name in {
"inputs.log",
"canonical.log",
"independent.log",
"comparison.txt",
}:
diagnostic_files.append(entry)
else:
raise RuntimeError(f"unrecognized diagnostic output; move aside: {entry}")
if output.exists():
for artifact in output.iterdir():
canonical = archives.get(artifact.name)
if (
artifact.is_symlink()
or not artifact.is_file()
or canonical is None
or digest(artifact) != digest(canonical)
):
raise RuntimeError(f"unrecognized or modified output; move aside: {artifact}")
staged_files.append(artifact)
# Delete staged copies first. Their canonical originals are the ownership
# evidence needed by the next run if cleanup is interrupted midway through.
files = staged_files + diagnostic_files
if files:
print(f"Cleaning {len(files)} previous reproducible-wheel output files")
for path in files:
path.unlink()
for directory in directories:
directory.rmdir()
diagnostics.mkdir(parents=True, exist_ok=True)
@contextmanager
def checkout_lock() -> Iterator[None]:
"""Reject overlapping builds in this checkout before touching their outputs."""
# Keep the inode after releasing the lock: unlinking it could let another
# process lock a new inode while a contender still holds the original one.
with (ROOT / ".reproducible-wheel.lock").open("a") as stream:
try:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as error:
raise RuntimeError(
"another reproducible-wheel build is active in this checkout"
) from error
try:
yield
finally:
fcntl.flock(stream, fcntl.LOCK_UN)
def stage_distributions(artifacts: tuple[Path, ...], output: Path) -> None:
"""Copy outside dist/, then atomically publish each complete archive."""
output.mkdir(parents=True, exist_ok=True)
# A sibling stays on the output filesystem for atomic renames. If killed
# during copying, only an ignored temporary directory can contain partials.
with tempfile.TemporaryDirectory(
prefix=".reproducible-wheel-stage-", dir=output.parent
) as temporary:
staging = Path(temporary)
for artifact in artifacts:
shutil.copy2(artifact, staging / artifact.name)
for artifact in artifacts:
(staging / artifact.name).replace(output / artifact.name)
def build(python: str, output: Path, diagnostics: Path) -> None:
"""Build twice and expose the canonical wheel/sdist only after verification."""
with checkout_lock():
_build(python, output, diagnostics)
def _build(python: str, output: Path, diagnostics: Path) -> None:
"""Run the gate while holding the checkout lock through final staging."""
prepare_outputs(output, diagnostics)
# Always derive the timestamp from this commit, not an inherited shell value.
epoch = subprocess.check_output(
["git", "log", "-1", "--format=%ct", "HEAD"], cwd=ROOT, text=True
).strip()
env = dict(os.environ)
for name in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"):
env.pop(name, None)
# Override .python-version (the dev interpreter) for the offline lock/export
# checks too. A clean release runner only needs the selected build Python.
env.update(
SOURCE_DATE_EPOCH=epoch,
PYTHONNOUSERSITE="1",
UV_PYTHON=python,
TZ="UTC",
LC_ALL="C.UTF-8",
)
run(["uv", "lock", "--check", "--offline"], ROOT, env, diagnostics / "inputs.log")
run(
[sys.executable, str(ROOT / "scripts/check_exports.py")],
ROOT,
env,
diagnostics / "inputs.log",
)
with tempfile.TemporaryDirectory(prefix="vocalinux-wheel-") as temporary:
scratch = Path(temporary)
source = scratch / "snapshot"
source.mkdir()
snapshot(ROOT, source)
for label, seed in (("canonical", "1"), ("independent", "2")):
tree = scratch / label / "source"
shutil.copytree(source, tree, symlinks=True)
venv = scratch / label / "venv"
dist = diagnostics / label
dist.mkdir()
log = diagnostics / f"{label}.log"
build_env = {**env, "PYTHONHASHSEED": seed}
run(
[
"uv",
"venv",
"--no-config",
"--python",
python,
str(venv),
],
scratch,
build_env,
log,
)
interpreter = str(venv / "bin/python")
# Force every exported tool into the venv, even if the system already
# has that version. Only wheels, verified by the committed hashes.
run(
[
"uv",
"pip",
"install",
"--no-config",
"--python",
interpreter,
"--require-hashes",
"--no-deps",
"--only-binary=:all:",
"--reinstall",
"-r",
str(source / "requirements/release-build.txt"),
],
scratch,
build_env,
log,
)
run(
[interpreter, "-I", "-m", "build", "--no-isolation", "--outdir", str(dist)],
tree,
build_env,
log,
)
run(
[
interpreter,
"-I",
"-m",
"twine",
"check",
"--strict",
str(one_file(dist, "*.whl")),
str(one_file(dist, "*.tar.gz")),
],
tree,
build_env,
log,
)
canonical = one_file(diagnostics / "canonical", "*.whl")
compare_wheels(
canonical,
one_file(diagnostics / "independent", "*.whl"),
diagnostics / "comparison.txt",
)
stage_distributions((canonical, one_file(diagnostics / "canonical", "*.tar.gz")), output)
def main() -> int:
"""Run the local and CI release gate."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--python", default=sys.executable, help="build interpreter (CI: Python 3.11)"
)
parser.add_argument("--output", type=Path, default=ROOT / "dist")
parser.add_argument("--diagnostics", type=Path, default=ROOT / "build/reproducible-wheel")
args = parser.parse_args()
try:
build(args.python, args.output.absolute(), args.diagnostics.absolute())
except (OSError, RuntimeError, subprocess.CalledProcessError) as error:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/check-system-deps.sh
Executable
+210
View File
@@ -0,0 +1,210 @@
#!/bin/bash
# Check for system dependencies regardless of distro
# Returns helpful error messages with install commands
check_deps() {
MISSING=0
WARNINGS=0
echo "Checking Vocalinux system dependencies..."
echo ""
# Check for Python 3.11+ (requires-python in pyproject.toml)
if command -v python3 >/dev/null 2>&1; then
PY_VERSION=$(python3 -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
echo "✓ Python $PY_VERSION"
if [ "$(echo "$PY_VERSION" | cut -d. -f1)" -lt 3 ] || \
[ "$(echo "$PY_VERSION" | cut -d. -f2)" -lt 11 ]; then
echo " ✗ Python 3.11+ required, found $PY_VERSION"
MISSING=1
fi
else
echo "✗ Python 3 not found"
MISSING=1
fi
# Check for pip
if command -v pip3 >/dev/null 2>&1; then
echo "✓ pip3"
else
echo "✗ pip3 not found"
MISSING=1
fi
# Check for pkg-config
if command -v pkg-config >/dev/null 2>&1; then
echo "✓ pkg-config"
else
echo "✗ pkg-config not found"
MISSING=1
fi
# Check for GTK3
if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists gtk+-3.0 2>/dev/null; then
GTK_VERSION=$(pkg-config --modversion gtk+-3.0 2>/dev/null)
echo "✓ GTK3 $GTK_VERSION"
else
echo "✗ GTK3 development files not found"
MISSING=1
fi
# Check for PyGObject
if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists gobject-introspection-1.0 2>/dev/null; then
echo "✓ GObject Introspection"
GI_TYPELIB_PATH=$(pkg-config --variable=typelibdir gobject-introspection-1.0 2>/dev/null)
echo " Typelib path: $GI_TYPELIB_PATH"
else
echo "✗ GObject Introspection not found"
MISSING=1
fi
# Check for AppIndicator/Ayatana AppIndicator GI runtime
if python3 - <<'PY' >/dev/null 2>&1
import importlib
import gi
for namespace in ("AppIndicator3", "AyatanaAppIndicator3", "AyatanaAppindicator3"):
try:
gi.require_version(namespace, "0.1")
importlib.import_module(f"gi.repository.{namespace}")
raise SystemExit(0)
except (ImportError, ValueError):
pass
raise SystemExit(1)
PY
then
echo "✓ AppIndicator/Ayatana GI runtime"
else
echo "✗ AppIndicator/Ayatana GI runtime not found"
MISSING=1
fi
# Check for PortAudio
if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists portaudio-2.0 2>/dev/null; then
PA_VERSION=$(pkg-config --modversion portaudio-2.0 2>/dev/null)
echo "✓ PortAudio $PA_VERSION"
else
echo "⚠ PortAudio development files not found (required for audio)"
MISSING=1
fi
# Check for text injection tools
echo ""
echo "Text injection tools (at least one required):"
if command -v xdotool >/dev/null 2>&1; then
echo "✓ xdotool (X11)"
else
echo " xdotool not found (X11)"
fi
if command -v wtype >/dev/null 2>&1; then
echo "✓ wtype (Wayland)"
else
echo " wtype not found (Wayland)"
fi
if command -v ydotool >/dev/null 2>&1; then
echo "✓ ydotool (Wayland universal)"
else
echo " ydotool not found (Wayland)"
fi
echo ""
echo "Clipboard tools (recommended for Wayland layout-independent paste fallback):"
if command -v wl-copy >/dev/null 2>&1; then
echo "✓ wl-copy (Wayland clipboard)"
else
echo " wl-copy not found (Wayland clipboard)"
fi
if command -v xclip >/dev/null 2>&1; then
echo "✓ xclip (X11/XWayland clipboard)"
else
echo " xclip not found (X11/XWayland clipboard)"
fi
if command -v xsel >/dev/null 2>&1; then
echo "✓ xsel (X11/XWayland clipboard fallback)"
else
echo " xsel not found (X11/XWayland clipboard fallback)"
fi
# Detect distribution for install hints
echo ""
DETECTED_DISTRO="unknown"
if [ -f /etc/os-release ]; then
. /etc/os-release
DETECTED_DISTRO="$ID"
DETECTED_DISTRO_LIKE="$ID_LIKE"
fi
# Summary
echo ""
echo "===================================="
if [ $MISSING -eq 0 ]; then
echo "✓ All critical dependencies found!"
return 0
else
echo "✗ Some dependencies are missing"
echo ""
echo "Install based on your distribution:"
echo ""
# Determine distro family for install hints
if [[ "$DETECTED_DISTRO" == "ubuntu" || "$DETECTED_DISTRO_LIKE" == *"ubuntu"* || "$DETECTED_DISTRO" == "pop" || "$DETECTED_DISTRO" == "linuxmint" || "$DETECTED_DISTRO" == "elementary" || "$DETECTED_DISTRO" == "zorin" ]]; then
echo "Ubuntu/Debian-based:"
echo " sudo apt update"
echo " sudo apt install -y python3-gi gir1.2-gtk-3.0 gir1.2-gdkpixbuf-2.0 portaudio19-dev python3-dev python3-venv pkg-config xdotool wtype wl-clipboard xclip xsel"
elif [[ "$DETECTED_DISTRO" == "fedora" || "$DETECTED_DISTRO_LIKE" == *"fedora"* || "$DETECTED_DISTRO" == "rhel" || "$DETECTED_DISTRO" == "centos" || "$DETECTED_DISTRO" == "rocky" || "$DETECTED_DISTRO" == "almalinux" ]]; then
echo "Fedora/RHEL-based:"
echo " sudo dnf install -y python3-gobject gtk3-devel libayatana-appindicator-gtk3 gobject-introspection-devel portaudio-devel python3-devel python3-virtualenv pkg-config xdotool wtype wl-clipboard xclip xsel"
elif [[ "$DETECTED_DISTRO" == "arch" || "$DETECTED_DISTRO_LIKE" == *"arch"* || "$DETECTED_DISTRO" == "manjaro" || "$DETECTED_DISTRO" == "endeavouros" ]]; then
echo "Arch Linux-based:"
echo " sudo pacman -S --needed python-gobject gtk3 libayatana-appindicator gobject-introspection portaudio python pkg-config xdotool wtype wl-clipboard xclip xsel"
elif [[ "$DETECTED_DISTRO" == "opensuse" || "$DETECTED_DISTRO_LIKE" == *"suse"* ]]; then
echo "openSUSE:"
echo " PYVER=\$(python3 -c 'import sys; print(f\"python{sys.version_info.major}{sys.version_info.minor}\")')"
echo " sudo zypper install -y \"\${PYVER}-gobject\" \"\${PYVER}-gobject-cairo\" gtk3 typelib-1_0-AyatanaAppIndicator3-0_1 libayatana-appindicator3-1 typelib-1_0-Notify-0_7 libnotify4 gobject-introspection-devel portaudio-devel \"\${PYVER}-devel\" \"\${PYVER}-virtualenv\" pkg-config xdotool wtype wl-clipboard xclip xsel"
echo " If \"\${PYVER}-virtualenv\" is unavailable, try \"\${PYVER}-venv\"."
elif [[ "$DETECTED_DISTRO" == "gentoo" ]]; then
echo "Gentoo:"
echo " sudo emerge dev-python/pygobject:3 x11-libs/gtk+:3 dev-libs/libayatana-appindicator media-libs/portaudio dev-lang/python:3.11 xdotool wtype"
elif [[ "$DETECTED_DISTRO" == "alpine" ]]; then
echo "Alpine Linux:"
echo " sudo apk add py3-gobject3 py3-pip gtk+3.0 py3-cairo portaudio-dev py3-virtualenv pkgconf xdotool wtype"
elif [[ "$DETECTED_DISTRO" == "void" ]]; then
echo "Void Linux:"
echo " sudo xbps-install -Sy python3-pip python3-gobject gtk+3 libappindicator gobject-introspection portaudio-devel python3-devel pkg-config xdotool wtype wl-clipboard xclip xsel"
elif [[ "$DETECTED_DISTRO" == "solus" ]]; then
echo "Solus:"
echo " sudo eopkg install python3-pip python3-gobject gtk3 libappindicator gobject-introspection-devel portaudio-devel python3-virtualenv pkg-config xdotool wtype wl-clipboard xclip xsel"
elif [[ "$DETECTED_DISTRO" == "mageia" ]]; then
echo "Mageia:"
if command -v dnf >/dev/null 2>&1; then
echo " sudo dnf install -y python3-gobject gtk3-devel gobject-introspection-devel portaudio-devel python3-devel python3-virtualenv pkg-config xdotool wtype wl-clipboard xclip xsel"
else
echo " sudo urpmi -y python3-gobject gtk3-devel gobject-introspection-devel portaudio-devel python3-devel python3-virtualenv pkg-config xdotool wtype wl-clipboard xclip xsel"
fi
else
echo "Ubuntu/Debian-based:"
echo " sudo apt install -y python3-gi gir1.2-gtk-3.0 portaudio19-dev python3-dev pkg-config xdotool wtype wl-clipboard xclip xsel"
echo ""
echo "Fedora/RHEL-based:"
echo " sudo dnf install -y python3-gobject gtk3-devel libayatana-appindicator-gtk3 portaudio-devel python3-devel pkg-config xdotool wtype wl-clipboard xclip xsel"
echo ""
echo "Arch Linux:"
echo " sudo pacman -S python-gobject gtk3 libayatana-appindicator portaudio python pkg-config xdotool wtype wl-clipboard xclip xsel"
fi
echo ""
echo "After installing system dependencies, run the installer again:"
echo " ./install.sh"
return 1
fi
}
check_deps
scripts/check_build_deps.py
Executable
+212
View File
@@ -0,0 +1,212 @@
#!/usr/bin/env python3
"""Fail fast when the native headers `just deps` needs are missing.
`uv sync` builds pyaudio, pycairo and PyGObject from source on Linux. Without
their development headers it fails deep inside a C compiler invocation, which
reads as a uv or dependency problem even though the fix is a distro package.
This check runs before the sync and names what is missing.
"""
from __future__ import annotations
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
VENV_PYTHON = ROOT / ".venv" / "bin" / "python"
# pkg-config modules the source builds query. Labels explain the module to a
# contributor who does not know which package owns it.
BUILD_MODULES = (("portaudio-2.0", "PortAudio headers (pyaudio)"),)
# PyGObject and its pycairo dependency are selected only on Linux by the
# sys_platform marker in pyproject.toml. They must not gate macOS tooling.
PYGOBJECT_BUILD_MODULES = (
("cairo", "cairo headers (pycairo)"),
("cairo-gobject", "cairo-gobject headers (PyGObject)"),
("girepository-2.0", "GObject Introspection headers (glib 2.80+)"),
("libffi", "libffi headers (PyGObject)"),
)
MIN_GIREPOSITORY_VERSION = "2.80"
APT_HINT = [
"Ubuntu 24.04+ / Debian 13+:",
" sudo apt install -y build-essential pkg-config libcairo2-dev portaudio19-dev \\",
" libgirepository-2.0-dev libffi-dev",
]
DEBIAN_12_NOTE = [
"",
"Debian 12 cannot build PyGObject 3.56 (glib is too old). Use ./install.sh --dev",
"there and run the suite with venv/bin/pytest.",
]
DNF_HINT = [
"Fedora 40+ / RHEL-based systems with glib 2.80+:",
" sudo dnf install -y gcc gcc-c++ make pkg-config cairo-devel \\",
" cairo-gobject-devel portaudio-devel glib2-devel \\",
" gobject-introspection-devel libffi-devel",
"If your system's glib is older than 2.80, use ./install.sh --dev instead.",
]
FEDORA_OLD_NOTE = [
"Fedora 39 and older cannot build PyGObject 3.56 (glib is too old).",
"Use ./install.sh --dev there and run the suite with venv/bin/pytest.",
]
PACMAN_HINT = [
"Arch / Manjaro:",
" sudo pacman -S --needed base-devel pkgconf cairo portaudio gobject-introspection",
]
ZYPPER_HINT = [
"openSUSE:",
" sudo zypper install -y gcc gcc-c++ make pkg-config cairo-devel \\",
" portaudio-devel gobject-introspection-devel libffi-devel",
]
GENERIC_HINT = [
"Install the development packages for PortAudio, cairo, GObject Introspection",
"(glib 2.80+) and libffi, plus a C compiler and pkg-config.",
]
MACOS_HINT = [
"macOS (Homebrew):",
" brew install portaudio pkg-config",
"Install the Xcode Command Line Tools if a C compiler is missing:",
" xcode-select --install",
]
def read_os_release(path: Path = Path("/etc/os-release")) -> dict[str, str]:
"""Parse os-release without executing it as shell."""
values: dict[str, str] = {}
try:
lines = path.read_text(encoding="utf-8").splitlines()
except OSError:
return values
for line in lines:
key, separator, value = line.partition("=")
if not separator or not key or key.startswith("#"):
continue
values[key] = value.strip().strip('"').strip("'")
return values
def install_hint(release: dict[str, str]) -> list[str]:
"""Return distro-specific install commands for the missing build headers."""
distro = release.get("ID", "").lower()
like = release.get("ID_LIKE", "").lower()
if distro == "ubuntu" or "ubuntu" in like:
return APT_HINT
if distro == "debian" or "debian" in like:
return [*APT_HINT, *DEBIAN_12_NOTE]
if distro in {"fedora", "rhel", "centos", "rocky", "almalinux"} or any(
family in like for family in ("fedora", "rhel")
):
if distro == "fedora" and release.get("VERSION_ID", "").isdigit():
if int(release["VERSION_ID"]) < 40:
return FEDORA_OLD_NOTE
return DNF_HINT
if distro in {"arch", "manjaro", "endeavouros"} or "arch" in like:
return PACMAN_HINT
if distro == "opensuse" or distro.startswith("opensuse-") or "suse" in like:
return ZYPPER_HINT
return GENERIC_HINT
def venv_has_compiled_packages() -> bool:
"""True when `.venv` has the native packages selected for this platform."""
if not VENV_PYTHON.is_file():
return False
packages = ("pyaudio",)
if sys.platform == "linux":
packages += ("pycairo", "pygobject")
check = "import importlib.metadata as metadata; " + "; ".join(
f"metadata.version({package!r})" for package in packages
)
result = subprocess.run(
[str(VENV_PYTHON), "-c", check],
capture_output=True,
check=False,
)
return result.returncode == 0
def missing_build_dependencies() -> list[str]:
"""Return the compiler or pkg-config modules uv cannot find, if any."""
missing: list[str] = []
if shutil.which("pkg-config") is None:
missing.append("pkg-config")
if not any(shutil.which(compiler) for compiler in ("cc", "gcc", "clang")):
missing.append("a C compiler")
# Module probes need pkg-config; report the missing tool instead of a list
# of modules the user cannot check yet.
if missing:
return missing
modules = BUILD_MODULES
if sys.platform == "linux":
modules += PYGOBJECT_BUILD_MODULES
for module, description in modules:
# PyGObject 3.56 requires girepository-2.0 >= 2.80; finding an older
# .pc file is not enough to build it successfully.
args = ["pkg-config", "--exists", module]
if module == "girepository-2.0":
args = ["pkg-config", f"--atleast-version={MIN_GIREPOSITORY_VERSION}", module]
result = subprocess.run(
args,
capture_output=True,
check=False,
)
if result.returncode != 0:
missing.append(description)
return missing
def main() -> int:
"""Print an actionable dependency report when the build cannot succeed."""
if venv_has_compiled_packages():
return 0
missing = missing_build_dependencies()
if not missing:
return 0
print("error: missing native build dependencies for .venv/", file=sys.stderr)
print(file=sys.stderr)
print(
"`just deps` builds native Python packages from source, so uv needs their",
file=sys.stderr,
)
print("development headers before it can create the environment.", file=sys.stderr)
print(file=sys.stderr)
print("Missing:", file=sys.stderr)
for item in missing:
print(f" - {item}", file=sys.stderr)
print(file=sys.stderr)
print("Install them for your system:", file=sys.stderr)
hints = MACOS_HINT if sys.platform == "darwin" else install_hint(read_os_release())
for line in hints:
print(line, file=sys.stderr)
print(file=sys.stderr)
print(
"See the Manual setup section in CONTRIBUTING.md for the full package list.",
file=sys.stderr,
)
return 1
if __name__ == "__main__":
raise SystemExit(main())
scripts/check_exports.py
Executable
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env python3
"""Fail if any `requirements/*.txt` export is behind `uv.lock`.
`just lock` writes the exports and nothing ever re-ran it, so a committed export
stays whatever it was on the day it was written. `just lock-check` and
`just flatpak-deps-check` are both in the justfile and neither is wired to a
workflow, which is how three engine exports could be added with nothing
comparing them to the lock they came from. The lock itself is covered:
`uv sync --locked` and `uv run --locked` in the pipeline fail when it drifts
from pyproject.toml. Nothing covered the step after that one.
The export lines are read out of the `lock` recipe rather than listed here, so
an export added there is checked without this file changing.
`uv export` reads `uv.lock` and nothing else, so this runs under UV_OFFLINE=1
and reproduces every file byte for byte on a cold cache. The env var rather than
`--offline`: uv echoes its own argv into the header of the file it writes, and
the flag would change the bytes this exists to compare.
`uv pip compile` lines stay out of scope. requirements/whisper.txt and the
AppImage files resolve against live indexes on purpose (see
requirements/whisper.in), so they are neither reproducible offline nor derived
from the lock.
"""
from __future__ import annotations
import os
import re
import shlex
import subprocess
import sys
import tempfile
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
JUSTFILE = REPO_ROOT / "justfile"
def _lock_recipe() -> list[str]:
"""The body of the `lock` recipe, one command per entry.
Continuations are joined. The recipe already wraps its `uv pip compile`
lines, so a wrapped `uv export` is a matter of time, and without this it
would reach `shlex.split` with a dangling backslash: that raises `ValueError:
No escaped character` instead of naming a file, and the `-o` sitting on the
next physical line would be invisible either way.
"""
body: list[str] = []
inside = False
for line in JUSTFILE.read_text(encoding="utf-8").splitlines():
if re.match(r"^lock:\s*$", line):
inside = True
continue
if not inside:
continue
if line.strip() and not line.startswith((" ", "\t")):
break
command = line.strip()
if body and body[-1].endswith("\\"):
body[-1] = body[-1][:-1].rstrip() + " " + command
else:
body.append(command)
return body
def export_commands() -> list[tuple[list[str], Path]]:
"""Every `uv export` line in `just lock`, with the file it writes.
A line with no `-o` writes to stdout, which no export in the recipe does and
which this could not compare: reported rather than skipped, because skipping
it is the failure this whole script is for.
"""
commands = []
for line in _lock_recipe():
if not line.startswith("uv export"):
continue
argv = shlex.split(line)
if "-o" not in argv:
raise SystemExit(f"`just lock` exports to stdout, so nothing can check it:\n {line}")
commands.append((argv, REPO_ROOT / argv[argv.index("-o") + 1]))
return commands
def regenerate(argv: list[str], target: Path, scratch: Path) -> str:
"""Re-run one export into `scratch` and return what it would have written.
uv records the command it ran in the file's header, so the scratch path is
put back to the declared one before comparing: the header is part of the
export and worth checking, just not the part that says where it went.
"""
rerun = list(argv)
rerun[rerun.index("-o") + 1] = str(scratch)
result = subprocess.run(
rerun,
cwd=REPO_ROOT,
env={**os.environ, "UV_OFFLINE": "1"},
capture_output=True,
text=True,
)
if result.returncode != 0:
raise SystemExit(f"{shlex.join(rerun)}\n{result.stderr.strip()}")
written = scratch.read_text(encoding="utf-8")
return written.replace(str(scratch), target.relative_to(REPO_ROOT).as_posix())
def main() -> int:
"""Check every lock-generated export and report whether any are stale."""
commands = export_commands()
if not commands:
print("no `uv export` lines in the lock recipe; nothing was checked", file=sys.stderr)
return 1
stale = []
with tempfile.TemporaryDirectory() as tmp:
for argv, target in commands:
expected = regenerate(argv, target, Path(tmp) / target.name)
name = target.relative_to(REPO_ROOT)
if not target.is_file():
stale.append(f"{name} is named in `just lock` but is not committed")
elif target.read_text(encoding="utf-8") != expected:
stale.append(f"{name} is behind uv.lock")
if stale:
print("\n".join(stale), file=sys.stderr)
print("re-run `just lock` and commit the result", file=sys.stderr)
return 1
print(f"{len(commands)} exports match uv.lock")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/distro-package-map.yaml
Executable
+202
View File
@@ -0,0 +1,202 @@
# Authoritative distro package inventory for install.sh.
#
# Edit this file, then run `just distro-packages`. The installer consumes the
# generated install.d/package_map.sh and never needs a YAML parser at runtime.
# The remote bootstrap's git package is the one deliberate exception: it must
# be installed before the tagged repository and this generated map are present.
schema_version: 1
distributions:
ubuntu:
# libssl/autotools/libtool/patchelf support pywhispercpp source builds.
system:
- python3-pip
- python3-gi
- python3-gi-cairo
- gir1.2-gtk-3.0
- gir1.2-ibus-1.0
- libcairo2-dev
- cmake
- python3-dev
- build-essential
- portaudio19-dev
- python3-venv
- pkg-config
- wget
- curl
- unzip
- vulkan-tools
- libvulkan-dev
- xclip
- xsel
- wl-clipboard
- libssl-dev
- autoconf
- automake
- libtool
- patchelf
appindicator: [gir1.2-appindicator3-0.1, gir1.2-ayatanaappindicator3-0.1]
gi_development: [libgirepository-2.0-dev, libgirepository1.0-dev]
shader_compiler: [glslc, glslang-tools]
optional_system: [util-linux-extra]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
debian_12:
system:
- python3-pip
- python3-gi
- python3-gi-cairo
- gir1.2-gtk-3.0
- gir1.2-ibus-1.0
- libcairo2-dev
- cmake
- python3-dev
- build-essential
- portaudio19-dev
- python3-venv
- pkg-config
- wget
- curl
- unzip
- vulkan-tools
- libvulkan-dev
- xclip
- xsel
- wl-clipboard
- libssl-dev
- autoconf
- automake
- libtool
- patchelf
- libgirepository1.0-dev
- gir1.2-ayatanaappindicator3-0.1
shader_compiler: [glslc, glslang-tools]
optional_system: [util-linux-extra]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
debian_13_plus:
# Used only to distinguish Debian 13-based derivatives whose own
# VERSION_ID does not match the Debian base release.
selection_probe: libgirepository-2.0-dev
system:
- python3-pip
- python3-gi
- python3-gi-cairo
- gir1.2-gtk-3.0
- gir1.2-ibus-1.0
- libcairo2-dev
- cmake
- python3-dev
- build-essential
- portaudio19-dev
- python3-venv
- pkg-config
- wget
- curl
- unzip
- vulkan-tools
- libvulkan-dev
- xclip
- xsel
- wl-clipboard
- libssl-dev
- autoconf
- automake
- libtool
- patchelf
- libgirepository-2.0-dev
- gir1.2-ayatanaappindicator3-0.1
shader_compiler: [glslc, glslang-tools]
optional_system: [util-linux-extra]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
fedora:
# cairo-devel is needed because the pinned pycairo builds from source.
# gcc-c++/make compile the pywhispercpp sdist (the wheel SIGILLs on old CPUs)
# and the evdev/PyAudio sdists; python3-devel only guarantees gcc.
system: [python3-pip, python3-gobject, gtk3, ibus-devel, gobject-introspection-devel, python3-devel, portaudio-devel, cairo-devel, python3-virtualenv, pkg-config, cmake, gcc-c++, make, wget, curl, unzip, vulkan-tools, vulkan-loader-devel, glslc, patchelf, xclip, xsel, wl-clipboard]
appindicator: [libayatana-appindicator-gtk3, libappindicator-gtk3]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
arch:
system: [python-pip, python-gobject, gtk3, ibus, gobject-introspection, python-cairo, portaudio, python-virtualenv, pkg-config, cmake, wget, curl, unzip, base-devel, vulkan-tools, vulkan-headers, shaderc, patchelf, xclip, xsel, wl-clipboard]
appindicator: [libayatana-appindicator, libappindicator-gtk3]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
suse:
# IBus is a runtime dependency; ibus-devel drags in an unsatisfiable gtk-doc
# chain on Tumbleweed. The compilers build evdev and PyAudio from sdists.
# cairo-devel is needed because the pinned pycairo builds from source.
system: [gtk3, ibus, typelib-1_0-IBus-1_0, gobject-introspection-devel, portaudio-devel, cairo-devel, pkg-config, cmake, gcc, gcc-c++, make, wget, curl, unzip, xclip, xsel, wl-clipboard, typelib-1_0-Notify-0_7, libnotify4, patchelf]
appindicator: [typelib-1_0-AyatanaAppIndicator3-0_1, typelib-1_0-AppIndicator3-0_1, typelib-1_0-AyatanaAppIndicator-0_1, libayatana-appindicator3-1, libappindicator3-1, libappindicator-gtk3]
python_packages:
pip: pip
gobject: gobject
gobject_cairo: gobject-cairo
devel: devel
virtualenv: virtualenv
venv: venv
vulkan: [vulkan-tools, vulkan-devel]
shader_compiler: [shaderc, glslang-devel, glslang]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
gentoo:
system: [dev-python/pygobject:3, x11-libs/gtk+:3, dev-libs/libayatana-appindicator, media-libs/portaudio, dev-lang/python:3.11, pkgconf, cmake, media-libs/shaderc, dev-util/patchelf, x11-misc/xclip, x11-misc/xsel, gui-apps/wl-clipboard]
text_input:
xdotool: [x11-misc/xdotool]
wtype: [x11-misc/wtype]
ydotool: [app-misc/ydotool]
alpine:
# cairo-dev is needed because the pinned pycairo builds from source.
# build-base compiles the pywhispercpp sdist and the evdev/PyAudio sdists.
system: [py3-gobject3, py3-pip, gtk+3.0, py3-cairo, portaudio-dev, cairo-dev, py3-virtualenv, pkgconf, cmake, build-base, wget, curl, unzip, shaderc, patchelf, vulkan-tools, xclip, xsel, wl-clipboard]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
void:
# cairo-devel is needed because the pinned pycairo builds from source.
# base-devel compiles the pywhispercpp sdist and the evdev/PyAudio sdists.
system: [python3-pip, python3-gobject, gtk+3, libappindicator-gtk3, gobject-introspection, portaudio-devel, cairo-devel, python3-devel, pkg-config, cmake, base-devel, wget, curl, unzip, shaderc, patchelf, Vulkan-Tools, xclip, xsel, wl-clipboard]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
solus:
# cairo-devel is needed because the pinned pycairo builds from source.
# gcc/g++/make compile the pywhispercpp sdist and the evdev/PyAudio sdists.
system: [python3-pip, python3-gobject, gtk3, libappindicator, gobject-introspection-devel, portaudio-devel, cairo-devel, python3-virtualenv, pkg-config, cmake, gcc, g++, make, wget, curl, unzip, shaderc, patchelf, vulkan-tools, xclip, xsel, wl-clipboard]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
mageia:
# The existing installer deliberately follows Fedora's package inventory,
# including gcc-c++/make for the pywhispercpp sdist compile.
system: [python3-pip, python3-gobject, gtk3, ibus-devel, gobject-introspection-devel, python3-devel, portaudio-devel, cairo-devel, python3-virtualenv, pkg-config, cmake, gcc-c++, make, wget, curl, unzip, vulkan-tools, vulkan-loader-devel, glslc, patchelf, xclip, xsel, wl-clipboard]
text_input:
xdotool: [xdotool]
wtype: [wtype]
ydotool: [ydotool]
scripts/generate-model-checksums.py
Executable
+377
View File
@@ -0,0 +1,377 @@
#!/usr/bin/env python3
"""Regenerate src/vocalinux/utils/model_checksums.txt from upstream metadata.
Vocalinux downloads 40MB-2GB speech models at install time and on first run,
then hands them to native code (whisper.cpp loads its ggml files through
``ctypes.cdll``). Nothing about that is safe unless we know what we received,
so every downloader checks the file against a hash pinned *in this repository*
rather than one fetched alongside the download.
Where that hash comes from differs by registry, and so does what generating it
costs:
* whisper.cpp ships on Hugging Face, whose API exposes the sha256 of every LFS
blob (``lfs.oid``) plus the commit the listing was taken at, so nothing is
downloaded for it. That commit is pinned too -- the download URLs use it
instead of ``main``, so upstream retagging a file cannot turn every user's
install into a checksum failure.
* VOSK is different: ``model-list.json`` carries an md5 per model, but that is a
metadata field from the same host we are pinning against, so copying it pins
nothing an attacker could not change alongside the file. Those models are
downloaded here and pinned by the sha256 of the bytes we actually received;
the published md5 is checked against them on the way through, so a listing
that disagrees with its own file is caught at generation time. Downloading is
incremental -- a model already pinned by sha256 is not fetched again, so only
new models cost bandwidth. A first run, or any run with --refresh, therefore
pulls every VOSK zip: ~21.9GB, roughly half an hour on a fast connection.
The model names come from the package itself, so a model added to
vosk_model_info.py, whispercpp_model_info.py, parakeet_model_info.py or
faster_whisper_model_info.py is picked up on the next run and a model that
upstream never published fails the generation loudly.
Usage: just model-checksums (or: python scripts/generate-model-checksums.py)
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import sys
import urllib.request
from pathlib import Path
from typing import Iterable, NamedTuple
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC = REPO_ROOT / "src"
MANIFEST = SRC / "vocalinux" / "utils" / "model_checksums.txt"
HF_REPO = "ggerganov/whisper.cpp"
HF_API = f"https://huggingface.co/api/models/{HF_REPO}?blobs=true"
VOSK_MODEL_LIST = "https://alphacephei.com/vosk/models/model-list.json"
TIMEOUT = 60
class Entry(NamedTuple):
"""One verifiable artifact: the file name is the manifest's lookup key."""
filename: str
algo: str
digest: str
size: int
def _fetch_json(url: str):
request = urllib.request.Request(url, headers={"User-Agent": "vocalinux-checksum-generator"})
with urllib.request.urlopen(request, timeout=TIMEOUT) as response: # noqa: S310
return json.load(response)
def _existing_sha256_entries() -> dict:
"""sha256 entries already in the manifest, so a rerun re-downloads nothing."""
if not MANIFEST.exists():
return {}
entries = {}
for line in MANIFEST.read_text(encoding="utf-8").splitlines():
if line.startswith("#") or not line.strip():
continue
fields = line.split()
if len(fields) == 4 and fields[1] == "sha256":
entries[fields[0]] = Entry(fields[0], "sha256", fields[2], int(fields[3]))
return entries
def _download_and_hash(url: str, expected_md5: str) -> tuple[str, int]:
"""Stream ``url``, returning the sha256 of the bytes and their length.
The published md5 is verified on the way through: it is not what we pin, but
a mismatch means the listing and the file disagree, which is worth stopping
for rather than pinning either one.
"""
sha256, md5, size = hashlib.sha256(), hashlib.md5(), 0
request = urllib.request.Request(url, headers={"User-Agent": "vocalinux-checksum-generator"})
with urllib.request.urlopen(request, timeout=TIMEOUT) as response: # noqa: S310
for chunk in iter(lambda: response.read(1024 * 1024), b""):
sha256.update(chunk)
md5.update(chunk)
size += len(chunk)
if expected_md5 and md5.hexdigest() != expected_md5:
raise SystemExit(
f"{url}\n"
f" published md5: {expected_md5}\n"
f" actual md5: {md5.hexdigest()}\n"
"The listing does not describe the file it points at. Refusing to pin either."
)
return sha256.hexdigest(), size
def _model_names() -> tuple[list[str], list[str]]:
"""Model names the application can actually download, straight from the package."""
sys.path.insert(0, str(SRC))
from vocalinux.utils.vosk_model_info import VOSK_MODEL_INFO
from vocalinux.utils.whispercpp_model_info import WHISPERCPP_MODEL_INFO
vosk = {
name for info in VOSK_MODEL_INFO.values() for name in info["languages"].values() if name
}
return sorted(WHISPERCPP_MODEL_INFO), sorted(vosk)
def _parakeet_entries() -> list[Entry]:
"""Digests for every file of every Parakeet bundle, at its pinned revision.
The ONNX files are LFS, so Hugging Face publishes their sha256; tokens.txt
is a plain blob and is hashed from the bytes this script downloads, the same
way VOSK archives are.
"""
sys.path.insert(0, str(SRC))
from vocalinux.utils.parakeet_model_info import (
PARAKEET_MODEL_INFO,
get_model_file_url,
manifest_key,
model_files,
)
entries = []
for model_name, info in PARAKEET_MODEL_INFO.items():
repo, revision = info["repo"], info["revision"]
payload = _fetch_json(
f"https://huggingface.co/api/models/{repo}/revision/{revision}?blobs=true"
)
published = {}
for sibling in payload.get("siblings", []):
lfs = sibling.get("lfs") or {}
digest = lfs.get("oid") or lfs.get("sha256")
if digest:
published[sibling["rfilename"]] = (digest, int(lfs.get("size", 0)))
for filename in model_files(model_name):
key = manifest_key(model_name, filename)
remote_name = "/".join(part for part in (info.get("subdir", ""), filename) if part)
if remote_name in published:
digest, size = published[remote_name]
else:
url = get_model_file_url(model_name, filename)
print(f" {key} (not LFS, hashing bytes)...", flush=True)
digest, size = _download_and_hash(url, "")
entries.append(Entry(key, "sha256", digest, size))
return entries
def _faster_whisper_entries() -> list[Entry]:
"""Digests for every file of every faster-whisper bundle, at its pinned revision.
``model.bin`` is LFS (Hugging Face publishes sha256 on these repos); config,
tokenizer and vocabulary files are plain blobs hashed from the bytes this
script downloads, the same way Parakeet's tokens.txt is.
"""
sys.path.insert(0, str(SRC))
from vocalinux.utils.faster_whisper_model_info import (
FASTER_WHISPER_MODEL_INFO,
manifest_key,
model_files,
)
entries = []
for model_name, info in FASTER_WHISPER_MODEL_INFO.items():
repo, revision = info["repo"], info["revision"]
payload = _fetch_json(
f"https://huggingface.co/api/models/{repo}/revision/{revision}?blobs=true"
)
published = {}
for sibling in payload.get("siblings", []):
lfs = sibling.get("lfs") or {}
digest = lfs.get("sha256") or lfs.get("oid")
if digest:
published[sibling["rfilename"]] = (digest, int(lfs.get("size", 0)))
for filename in model_files(model_name):
key = manifest_key(model_name, filename)
if filename in published:
digest, size = published[filename]
else:
url = f"https://huggingface.co/{repo}/resolve/{revision}/{filename}?download=true"
print(f" {key} (not LFS, hashing bytes)...", flush=True)
digest, size = _download_and_hash(url, "")
entries.append(Entry(key, "sha256", digest, size))
return entries
def _hf_lfs_digests(repo: str, revision: str) -> dict[str, tuple[str, int]]:
"""sha256/size per LFS file in ``repo`` at ``revision``, from the HF API."""
payload = _fetch_json(
f"https://huggingface.co/api/models/{repo}/revision/{revision}?blobs=true"
)
published = {}
for sibling in payload.get("siblings", []):
lfs = sibling.get("lfs") or {}
digest = lfs.get("sha256") or lfs.get("oid")
if digest:
published[sibling["rfilename"]] = (digest, int(lfs.get("size", 0)))
return published
def _whispercpp_entries(wanted: Iterable[str]) -> tuple[list[Entry], str]:
sys.path.insert(0, str(SRC))
from vocalinux.utils.whispercpp_model_info import (
whispercpp_model_file,
whispercpp_model_source,
)
payload = _fetch_json(HF_API)
revision = payload.get("sha")
if not revision:
raise SystemExit(f"{HF_API} returned no commit sha; refusing to pin an unknown revision")
by_name = {}
for sibling in payload.get("siblings", []):
lfs = sibling.get("lfs") or {}
digest = lfs.get("oid") or lfs.get("sha256")
if digest:
by_name[sibling["rfilename"]] = (digest, int(lfs.get("size", 0)))
# Catalog entries hosted outside the main repository pin their own commit
# in whispercpp_model_info.py (same contract parakeet bundles follow).
side_repo_digests: dict[str, dict[str, tuple[str, int]]] = {}
entries, missing = [], []
for model_name in wanted:
filename = whispercpp_model_file(model_name)
repo, pinned_revision = whispercpp_model_source(model_name)
if repo != HF_REPO:
if repo not in side_repo_digests:
side_repo_digests[repo] = _hf_lfs_digests(repo, pinned_revision)
found = side_repo_digests[repo].get(filename)
if found is None:
missing.append(f"{filename} ({repo})")
continue
digest, size = found
elif filename not in by_name:
missing.append(filename)
continue
else:
digest, size = by_name[filename]
entries.append(Entry(filename, "sha256", digest, size))
if missing:
raise SystemExit(
f"Hugging Face publishes no sha256 for: {', '.join(missing)}.\n"
"Either the model was renamed upstream or whispercpp_model_info.py lists "
"a file that does not exist."
)
entries.sort(key=lambda entry: entry.filename)
return entries, revision
def _vosk_entries(wanted: Iterable[str], refresh: bool) -> list[Entry]:
payload = _fetch_json(VOSK_MODEL_LIST)
by_name = {item["name"]: item for item in payload}
existing = {} if refresh else _existing_sha256_entries()
missing = [name for name in wanted if name not in by_name]
if missing:
raise SystemExit(
f"{VOSK_MODEL_LIST} does not list: {', '.join(missing)}.\n"
"Either the model was withdrawn upstream or vosk_model_info.py names "
"a model Alphacephei never shipped."
)
todo = [name for name in wanted if f"{name}.zip" not in existing]
if todo:
total = sum(int(by_name[name].get("size", 0)) for name in todo)
print(f" downloading {len(todo)} VOSK model(s), {total / 1e9:.1f} GB, to hash the bytes")
entries = []
for name in wanted:
filename = f"{name}.zip"
if filename in existing:
entries.append(existing[filename])
continue
item = by_name[name]
print(f" {name} ({int(item.get('size', 0)) / 1e6:.0f} MB)...", flush=True)
digest, size = _download_and_hash(item["url"], item.get("md5", ""))
entries.append(Entry(filename, "sha256", digest, size))
return entries
def _render(entries: list[Entry], revision: str) -> str:
width = max(len(entry.filename) for entry in entries)
lines = [
"# Checksums for every speech model Vocalinux downloads.",
"#",
"# Generated by scripts/generate-model-checksums.py -- do not edit by hand.",
"# Regenerate with `just model-checksums` after adding a model to",
"# vosk_model_info.py, whispercpp_model_info.py, parakeet_model_info.py",
"# or faster_whisper_model_info.py.",
"#",
"# Read by src/vocalinux/utils/model_checksums.py and by install.sh; keep the",
"# format (filename, algorithm, digest, size in bytes) parseable by awk.",
"#",
"# whisper.cpp models are pinned to this Hugging Face commit so upstream",
"# replacing a file cannot invalidate the digests below.",
f"# whispercpp-revision: {revision}",
"#",
"# OpenAI Whisper (.pt) models are absent on purpose: their download URLs embed",
"# the sha256 as a path segment, so the expected digest travels with the URL.",
"#",
"# VOSK digests are the sha256 of bytes this script downloaded, not the md5",
"# Alphacephei publishes; that md5 is only checked against them at that point.",
"#",
"# Parakeet and faster-whisper bundles are pinned per model in their",
"# model_info modules, so their keys carry the model name: several",
"# bundles ship the same file names.",
"",
]
for entry in entries:
lines.append(
f"{entry.filename.ljust(width)} {entry.algo:<6} {entry.digest} {entry.size}"
)
return "\n".join(lines) + "\n"
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--refresh",
action="store_true",
help="re-download every VOSK model instead of reusing pinned sha256 entries",
)
args = parser.parse_args()
whispercpp_files, vosk_names = _model_names()
print(f"Resolving {len(whispercpp_files)} whisper.cpp and {len(vosk_names)} VOSK models...")
whispercpp_entries, revision = _whispercpp_entries(whispercpp_files)
print(f" whisper.cpp: {len(whispercpp_entries)} sha256 digests at {HF_REPO}@{revision[:12]}")
vosk_entries = _vosk_entries(vosk_names, args.refresh)
print(f" VOSK: {len(vosk_entries)} sha256 digests")
parakeet_entries = _parakeet_entries()
print(f" Parakeet: {len(parakeet_entries)} sha256 digests")
faster_whisper_entries = _faster_whisper_entries()
print(f" faster-whisper: {len(faster_whisper_entries)} sha256 digests")
MANIFEST.write_text(
_render(
whispercpp_entries + vosk_entries + parakeet_entries + faster_whisper_entries,
revision,
),
encoding="utf-8",
)
print(f"Wrote {MANIFEST.relative_to(REPO_ROOT)}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/generate_distro_package_map.py
Executable
+246
View File
@@ -0,0 +1,246 @@
#!/usr/bin/env python3
"""Generate the installer's shell package map from its YAML source."""
from __future__ import annotations
import argparse
import shlex
import sys
from pathlib import Path
from typing import Any
import yaml
ROOT = Path(__file__).resolve().parents[1]
SOURCE = ROOT / "scripts" / "distro-package-map.yaml"
TARGET = ROOT / "install.d" / "package_map.sh"
REQUIRED_DISTRIBUTIONS = {
"alpine",
"arch",
"debian_12",
"debian_13_plus",
"fedora",
"gentoo",
"mageia",
"solus",
"suse",
"ubuntu",
"void",
}
LIST_FIELDS = {
"appindicator",
"gi_development",
"optional_system",
"shader_compiler",
"system",
"vulkan",
}
SCALAR_FIELDS = {"selection_probe"}
TEXT_TOOLS = {"wtype", "xdotool", "ydotool"}
SUSE_PYTHON_ROLES = {"pip", "gobject", "gobject_cairo", "devel", "virtualenv", "venv"}
class UniqueKeyLoader(yaml.SafeLoader):
"""Reject duplicate mapping keys instead of silently dropping inventory."""
def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict:
self.flatten_mapping(node)
result = {}
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
if not isinstance(key, str):
raise ValueError("package map keys must be strings")
if key in result:
raise ValueError(f"duplicate YAML key: {key}")
result[key] = self.construct_object(value_node, deep=deep)
return result
def _package_list(value: Any, location: str, *, required: bool = False) -> list[str]:
"""Validate and return one package list."""
if value is None and not required:
return []
if not isinstance(value, list) or (required and not value):
raise ValueError(f"{location} must be a non-empty list")
if any(not isinstance(item, str) or not item for item in value):
raise ValueError(f"{location} must contain non-empty strings")
if len(value) != len(set(value)):
raise ValueError(f"{location} contains duplicate packages")
return value
def load_map() -> dict[str, dict[str, Any]]:
"""Load and validate the authoritative YAML package map."""
document = yaml.load(SOURCE.read_text(encoding="utf-8"), Loader=UniqueKeyLoader)
if not isinstance(document, dict) or document.get("schema_version") != 1:
raise ValueError("distro package map must use schema_version: 1")
distributions = document.get("distributions")
if not isinstance(distributions, dict):
raise ValueError("distro package map must define a distributions mapping")
names = set(distributions)
if names != REQUIRED_DISTRIBUTIONS:
missing = ", ".join(sorted(REQUIRED_DISTRIBUTIONS - names)) or "none"
extra = ", ".join(sorted(names - REQUIRED_DISTRIBUTIONS)) or "none"
raise ValueError(f"distribution keys differ (missing: {missing}; extra: {extra})")
for distro, config in distributions.items():
if not isinstance(config, dict):
raise ValueError(f"distributions.{distro} must be a mapping")
unknown = set(config) - LIST_FIELDS - SCALAR_FIELDS - {"python_packages", "text_input"}
if unknown:
raise ValueError(f"distributions.{distro} has unknown fields: {sorted(unknown)}")
_package_list(config.get("system"), f"distributions.{distro}.system", required=True)
for field in LIST_FIELDS - {"system"}:
_package_list(config.get(field), f"distributions.{distro}.{field}")
selection_probe = config.get("selection_probe")
if selection_probe is not None and (
distro != "debian_13_plus"
or not isinstance(selection_probe, str)
or not selection_probe
):
raise ValueError("selection_probe must be a non-empty string on debian_13_plus only")
python_packages = config.get("python_packages")
if python_packages is not None:
if distro != "suse" or not isinstance(python_packages, dict):
raise ValueError("python_packages must be a mapping on suse only")
if set(python_packages) != SUSE_PYTHON_ROLES:
raise ValueError(
f"distributions.suse.python_packages must define {sorted(SUSE_PYTHON_ROLES)}"
)
if any(not isinstance(value, str) or not value for value in python_packages.values()):
raise ValueError("distributions.suse.python_packages values must be strings")
text_input = config.get("text_input")
if not isinstance(text_input, dict) or set(text_input) != TEXT_TOOLS:
raise ValueError(f"distributions.{distro}.text_input must define {sorted(TEXT_TOOLS)}")
for tool in TEXT_TOOLS:
_package_list(
text_input.get(tool),
f"distributions.{distro}.text_input.{tool}",
required=True,
)
if len(text_input[tool]) != 1:
raise ValueError(
f"distributions.{distro}.text_input.{tool} needs exactly one package"
)
for distro in ("ubuntu", "fedora", "arch"):
if len(distributions[distro].get("appindicator") or []) != 2:
raise ValueError(f"distributions.{distro}.appindicator must define two fallbacks")
if len(distributions["suse"].get("appindicator") or []) < 4:
raise ValueError("distributions.suse.appindicator must define at least four alternatives")
if not distributions["debian_13_plus"].get("selection_probe"):
raise ValueError("distributions.debian_13_plus.selection_probe is required")
if len(distributions["ubuntu"].get("gi_development") or []) != 2:
raise ValueError("distributions.ubuntu.gi_development must define modern and legacy")
for distro in ("ubuntu", "debian_12", "debian_13_plus"):
if len(distributions[distro].get("shader_compiler") or []) < 2:
raise ValueError(f"distributions.{distro}.shader_compiler needs a fallback")
if "python_packages" not in distributions["suse"]:
raise ValueError("distributions.suse.python_packages is required")
for field in ("vulkan", "shader_compiler"):
_package_list(distributions["suse"].get(field), f"suse.{field}", required=True)
return distributions
def _array_assignment(name: str, values: list[str], indent: str = " ") -> str:
quoted = " ".join(shlex.quote(value) for value in values)
return f"{indent}{name}=({quoted})"
def render(distributions: dict[str, dict[str, Any]]) -> str:
"""Render a deterministic Bash module from the validated map."""
debian_13_probe = distributions["debian_13_plus"]["selection_probe"]
lines = [
"#!/bin/bash",
"# Generated by scripts/generate_distro_package_map.py -- do not edit.",
"# Regenerate with `just distro-packages` after editing",
"# scripts/distro-package-map.yaml.",
"",
f"DEBIAN_13_PLUS_PROBE_PACKAGE={shlex.quote(debian_13_probe)}",
"",
"load_distro_package_map() {",
' local package_map_key="$1"',
"",
" SYSTEM_PACKAGES=()",
" APPINDICATOR_PACKAGES=()",
" GI_DEVELOPMENT_PACKAGES=()",
" OPTIONAL_SYSTEM_PACKAGES=()",
' PYTHON_PIP_SUFFIX=""',
' PYTHON_GOBJECT_SUFFIX=""',
' PYTHON_GOBJECT_CAIRO_SUFFIX=""',
' PYTHON_DEVEL_SUFFIX=""',
' PYTHON_VIRTUALENV_SUFFIX=""',
' PYTHON_VENV_SUFFIX=""',
" SHADER_COMPILER_PACKAGES=()",
" VULKAN_PACKAGES=()",
" XDOTOOL_PACKAGES=()",
" WTYPE_PACKAGES=()",
" YDOTOOL_PACKAGES=()",
"",
' case "$package_map_key" in',
]
names = {
"system": "SYSTEM_PACKAGES",
"appindicator": "APPINDICATOR_PACKAGES",
"gi_development": "GI_DEVELOPMENT_PACKAGES",
"optional_system": "OPTIONAL_SYSTEM_PACKAGES",
"shader_compiler": "SHADER_COMPILER_PACKAGES",
"vulkan": "VULKAN_PACKAGES",
}
for distro, config in distributions.items():
lines.append(f" {distro})")
for field, variable in names.items():
values = _package_list(config.get(field), f"{distro}.{field}")
if values:
lines.append(_array_assignment(variable, values))
python_packages = config.get("python_packages")
if python_packages:
for role in sorted(SUSE_PYTHON_ROLES):
variable = f"PYTHON_{role.upper()}_SUFFIX"
lines.append(f" {variable}={shlex.quote(python_packages[role])}")
text_input = config["text_input"]
lines.append(_array_assignment("XDOTOOL_PACKAGES", text_input["xdotool"]))
lines.append(_array_assignment("WTYPE_PACKAGES", text_input["wtype"]))
lines.append(_array_assignment("YDOTOOL_PACKAGES", text_input["ydotool"]))
lines.extend([" ;;", ""])
lines.extend(
[
" *)",
' echo "Unknown distro package map key: $package_map_key" >&2',
" return 1",
" ;;",
" esac",
"}",
"",
]
)
return "\n".join(lines)
def main() -> int:
"""Generate the map, or check that the committed output is current."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--check", action="store_true", help="fail if the output is stale")
args = parser.parse_args()
try:
generated = render(load_map())
except (OSError, ValueError, yaml.YAMLError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
if args.check:
current = TARGET.read_text(encoding="utf-8") if TARGET.exists() else ""
if current != generated:
print(
f"{TARGET.relative_to(ROOT)} is stale; run `just distro-packages`",
file=sys.stderr,
)
return 1
return 0
TARGET.write_text(generated, encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/generate_sounds.py
Executable
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env python3
"""Generate Vocalinux notification WAVs.
Writes 16-bit PCM mono files at 44.1 kHz into resources/sounds/:
- start_recording.wav / stop_recording.wav / error.wav: the original Linux
pair. Kept on disk; not used as the implicit default anymore.
Family catalog pairs ({id}_start.wav / {id}_stop.wav) are downloaded preview
bytes. This script never synthesizes or overwrites them.
Usage:
python scripts/generate_sounds.py
"""
from __future__ import annotations
import math
import os
import struct
import wave
SAMPLE_RATE = 44100
# Named pitches used by the family catalog.
G2 = 98.00
C3 = 130.81
D3 = 146.83
F3 = 174.61
G3 = 196.00
C4 = 261.63
E4 = 329.63
F4 = 349.23
G4 = 392.00
A4 = 440.00
C5 = 523.25
A5 = 880.00
C6 = 1046.50
E7 = 2637.02
C7 = 2093.00
def _smoothstep(progress: float) -> float:
return progress * progress * (3.0 - 2.0 * progress)
def _sine_envelope(progress: float) -> float:
return (math.sin(math.pi * progress) ** 2) * math.exp(-0.5 * progress)
def generate_glide_tone(
filename: str,
freq_start: float,
freq_end: float,
duration: float = 0.6,
amplitude: float = 0.16,
sample_rate: int = SAMPLE_RATE,
) -> None:
"""Generate a smooth pitch glide tone (original Linux pair recipe)."""
num_samples = int(sample_rate * duration)
samples = []
for i in range(num_samples):
t = i / sample_rate
progress = i / num_samples
envelope = _sine_envelope(progress)
glide = _smoothstep(progress)
freq_current = freq_start + (freq_end - freq_start) * glide
phase = 2 * math.pi * freq_current * t
samples.append(amplitude * envelope * math.sin(phase))
_write_wav(filename, samples, sample_rate)
def _write_wav(filename: str, samples: list[float], sample_rate: int = SAMPLE_RATE) -> None:
wav_samples = [int(max(-1.0, min(1.0, sample)) * 32767) for sample in samples]
os.makedirs(os.path.dirname(filename) or ".", exist_ok=True)
with wave.open(filename, "w") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(sample_rate)
wav_file.writeframes(struct.pack("<" + "h" * len(wav_samples), *wav_samples))
print(f"Generated: {filename}")
def _tone_burst(
freq: float,
duration: float,
amplitude: float = 0.12,
decay: float = 8.0,
) -> list[float]:
n = int(SAMPLE_RATE * duration)
samples = []
for i in range(n):
t = i / SAMPLE_RATE
progress = i / max(n - 1, 1)
attack = min(1.0, i / max(int(0.004 * SAMPLE_RATE), 1))
env = attack * math.exp(-decay * t) * (1.0 - 0.15 * progress)
samples.append(amplitude * env * math.sin(2 * math.pi * freq * t))
return samples
def _silence(duration: float) -> list[float]:
return [0.0] * int(SAMPLE_RATE * duration)
def generate_ticks(
filename: str,
freqs: tuple[float, ...],
tick_duration: float = 0.07,
gap: float = 0.045,
amplitude: float = 0.11,
decay: float = 14.0,
) -> None:
samples: list[float] = []
for index, freq in enumerate(freqs):
if index:
samples.extend(_silence(gap))
samples.extend(_tone_burst(freq, tick_duration, amplitude, decay))
_write_wav(filename, samples)
def generate_fifth_swell(
filename: str,
low: float,
high: float,
duration: float = 0.42,
amplitude: float = 0.13,
opening: bool = True,
) -> None:
"""Open-fifth swell. Start opens C4→G4; stop closes G4→C4."""
n = int(SAMPLE_RATE * duration)
samples = []
phase_low = 0.0
phase_high = 0.0
dt = 1.0 / SAMPLE_RATE
for i in range(n):
progress = i / max(n - 1, 1)
env = math.sin(math.pi * progress) ** 2
blend = _smoothstep(progress)
if opening:
low_amt = 1.0
high_amt = blend
else:
low_amt = 1.0
high_amt = 1.0 - blend
sample = (
amplitude
* env
* (low_amt * math.sin(phase_low) + 0.85 * high_amt * math.sin(phase_high))
)
samples.append(sample)
phase_low += 2 * math.pi * low * dt
phase_high += 2 * math.pi * high * dt
_write_wav(filename, samples)
def generate_ping(
filename: str, freq: float, duration: float = 0.32, amplitude: float = 0.12
) -> None:
n = int(SAMPLE_RATE * duration)
samples = []
for i in range(n):
t = i / SAMPLE_RATE
env = math.exp(-14.0 * t)
samples.append(amplitude * env * math.sin(2 * math.pi * freq * t))
_write_wav(filename, samples)
def generate_scale(
filename: str,
freqs: tuple[float, ...],
note_duration: float = 0.085,
gap: float = 0.02,
amplitude: float = 0.12,
) -> None:
generate_ticks(filename, freqs, note_duration, gap, amplitude, decay=9.0)
def _maybe_write_original(path: str, writer) -> None:
if os.path.exists(path):
print(f"Keeping existing: {path}")
return
writer()
def main() -> None:
script_dir = os.path.dirname(os.path.abspath(__file__))
repo_root = os.path.dirname(script_dir)
sounds_dir = os.path.join(repo_root, "resources", "sounds")
os.makedirs(sounds_dir, exist_ok=True)
print(f"Generating sounds in: {sounds_dir}\n")
start_path = os.path.join(sounds_dir, "start_recording.wav")
stop_path = os.path.join(sounds_dir, "stop_recording.wav")
error_path = os.path.join(sounds_dir, "error.wav")
_maybe_write_original(
start_path,
lambda: generate_glide_tone(start_path, F4, A4, duration=0.6, amplitude=0.16),
)
_maybe_write_original(
stop_path,
lambda: generate_glide_tone(stop_path, A4, F4, duration=0.6, amplitude=0.16),
)
_maybe_write_original(
error_path,
lambda: generate_glide_tone(error_path, E4, C4, duration=0.7, amplitude=0.14),
)
# Family catalog pairs are downloaded preview bytes. Never synthesize or
# overwrite them.
family_ids = (
"lift",
"flick",
"ember",
"step",
"voca",
"soft",
"chirp",
"scale",
"drop",
"glass",
)
family_names = {f"{tone_id}_{kind}.wav" for tone_id in family_ids for kind in ("start", "stop")}
for name in sorted(family_names):
path = os.path.join(sounds_dir, name)
if os.path.exists(path):
print(f"Keeping downloaded catalog: {path}")
else:
print(f"Missing catalog WAV (not synthesizing): {path}")
print("\nOriginal start/stop/error files were left in place.")
print("Family catalog WAVs were not written.")
if __name__ == "__main__":
main()
scripts/install-remote-test.sh
Executable
+327
View File
@@ -0,0 +1,327 @@
#!/usr/bin/env bash
# Run install.sh's remote path end to end in a distro container and check what
# it installed.
#
# Usage (a read-only mount is enough; the tag is published from git, not the
# tree):
# docker run --rm -v "$PWD:/repo:ro" debian:12 bash /repo/scripts/install-remote-test.sh
# or: just remote-install-gate debian:12
#
# The path under test is the one a `curl | bash` user takes and
# scripts/install-test.sh never reaches: bootstrap outside a checkout, --tag
# selection, clone of the selected tag, handoff to the tagged installer with
# the original arguments plus the remote venv path, then a second run's
# fetch+reset over the existing clone.
#
# The remote is a local mirror so the gate answers for *this commit*: the tag
# is published from HEAD's tree and VOCALINUX_REPO_URL points the bootstrap at
# it. A gate that cloned the public repo would install the last release, not
# the checkout under test. The tag's own install.sh is the bootstrap: the #701
# guardrail allows the bootstrap to come from main, and this commit's copy is
# what a merged PR would serve anyway.
#
# Flags the installer sees: --auto, --skip-models (a model is 40-75 MB
# per run and the download path is checksum-verified elsewhere), --tag (the
# selection under test). Not --skip-system-deps: the per-distro package
# installation is the point. Run 1 is headless; run 2 pipes the bootstrap in
# under a `script` pty so the real `curl | bash` fd-0 shape stays covered.
#
# Fail closed: after the install, the clone's tree must equal HEAD's tree —
# the #701 guardrail that installer, sourced modules and requirement exports
# all come from the selected tag is only as strong as that equality.
set -euo pipefail
REPO="${REPO:-/repo}"
INSTALL_USER="${INSTALL_USER:-installer}"
INSTALL_HOME="/home/$INSTALL_USER"
# Opaque on purpose: a name the public repository cannot already carry, so a
# silent fallback to GitHub fails loudly instead of installing the wrong ref.
GATE_TAG="${GATE_TAG:-v0.0.0-remote-gate}"
MIRROR="/srv/vocalinux-remote.git"
BOOTSTRAP="/tmp/vl.sh"
CLONE_DIR="$INSTALL_HOME/.local/share/vocalinux-install"
REMOTE_VENV="$INSTALL_HOME/.local/share/vocalinux/venv"
fail() {
echo "FAIL: $*" >&2
dump_install_log
exit 1
}
# install.sh writes a full transcript under ~/.local/state/vocalinux/. It is the
# only place the per-distro package manager output survives.
dump_install_log() {
local log
log="$(ls -1t "$INSTALL_HOME"/.local/state/vocalinux/install-*.log 2>/dev/null | head -n1 || true)"
if [ -n "$log" ]; then
echo "== last 200 lines of $log ==" >&2
tail -n 200 "$log" >&2
else
echo "== no install log was written ==" >&2
fi
}
# A mirror going slow mid-transaction leaves a half-populated cache, so
# re-running the same transaction resumes rather than restarts.
retry() {
local attempt
for attempt in 1 2 3; do
"$@" && return 0
echo " '$1' failed (attempt $attempt); retrying" >&2
sleep 5
done
return 1
}
[ -f "$REPO/install.sh" ] || fail "$REPO/install.sh not found; mount the checkout at \$REPO"
. /etc/os-release
echo "== ${NAME:-unknown} ${VERSION_ID:-} =="
echo "== Bootstrap: sudo, git and an unprivileged user =="
# The minimum for install.sh to start, and nothing it should install itself.
# git is for the mirror below; the remote path ensures it again for real users.
# Same arms as scripts/install-test.sh so `just remote-install-gate <distro>`
# works everywhere the local gate does.
case "$ID" in
ubuntu | debian)
export DEBIAN_FRONTEND=noninteractive
retry apt-get update -qq
retry apt-get install -y -qq sudo git ca-certificates >/dev/null
;;
fedora | rhel | centos)
retry dnf install -y -q sudo git shadow-utils >/dev/null
;;
arch)
# -Syu, never -Sy: archlinux:latest lags the mirrors, and a partial upgrade
# would be this script's breakage rather than the installer's.
retry pacman -Syu --noconfirm --needed sudo git >/dev/null
;;
opensuse-tumbleweed | opensuse-leap | sles)
retry zypper --non-interactive --gpg-auto-import-keys refresh >/dev/null
retry zypper --non-interactive install -y sudo git >/dev/null
;;
*)
fail "no bootstrap arm for ID='$ID'; add one when adding the container"
;;
esac
id -u "$INSTALL_USER" >/dev/null 2>&1 || useradd -m -s /bin/bash "$INSTALL_USER"
echo "$INSTALL_USER ALL=(ALL) NOPASSWD: ALL" >"/etc/sudoers.d/$INSTALL_USER"
chmod 0440 "/etc/sudoers.d/$INSTALL_USER"
echo "== Publish HEAD's tree as tag $GATE_TAG in a local mirror =="
# safe.directory: the mounted checkout and, for a linked worktree, its mounted
# common gitdir are owned by another uid.
git config --global --add safe.directory '*'
# CI checks out with fetch-depth 1, and a shallow boundary commit can neither
# be pushed nor fetched into a ref ("shallow update not allowed"). The tag
# therefore carries an orphan commit built from HEAD's tree: a root commit is
# not a shallow root, and tree equality below still binds the gate to this
# commit's exact contents.
SOURCE_TREE="$(git -C "$REPO" rev-parse 'HEAD^{tree}')" \
|| fail "$REPO is not a git checkout; the gate publishes the tag from git"
git clone --quiet --depth 1 "file://$REPO" /tmp/gate-source
# commit-tree needs an identity the bare container does not carry.
EXPECTED_SHA="$(GIT_AUTHOR_NAME=gate GIT_AUTHOR_EMAIL=gate@local \
GIT_COMMITTER_NAME=gate GIT_COMMITTER_EMAIL=gate@local \
git -C /tmp/gate-source commit-tree "$SOURCE_TREE" -m "remote-install gate")"
git init --bare "$MIRROR" >/dev/null
git -C /tmp/gate-source push "$MIRROR" "$EXPECTED_SHA:refs/tags/$GATE_TAG" >/dev/null
[ "$(git -C "$MIRROR" rev-parse "$GATE_TAG^{tree}")" = "$SOURCE_TREE" ] \
|| fail "the mirror's $GATE_TAG does not carry HEAD's tree"
# The installer clones as $INSTALL_USER, and git refuses a root-owned remote
# for anyone else as "dubious ownership".
chown -R "$INSTALL_USER:$INSTALL_USER" "$MIRROR"
# A remote user downloads install.sh over HTTPS; the gate takes the same file
# out of the tag instead. The tagged copy is this commit's bootstrap, and the
# handoff target is identical either way.
git -C "$MIRROR" show "$GATE_TAG:install.sh" >"$BOOTSTRAP"
chmod 0644 "$BOOTSTRAP"
run_remote_install() {
# cd to $INSTALL_HOME so the bootstrap sees no checkout: local mode is chosen
# by a pyproject.toml naming vocalinux in the working directory, and the
# remote path is the one under test.
su - "$INSTALL_USER" -c "cd '$INSTALL_HOME' && VOCALINUX_REPO_URL='file://$MIRROR' bash '$BOOTSTRAP' --auto --skip-models '--tag=$GATE_TAG'"
}
# The user's literal `curl | bash`: installer code reaches bash on a pipe while
# a controlling terminal exists. `script` allocates that pty — a saved file on
# stdin or a headless pipe takes the headless --auto branch instead, which is
# exactly how the fd-0 redirect regression in #933 slipped through. bsdutils /
# util-linux `script` is in the base package set of every gated distro.
command -v script >/dev/null 2>&1 || fail "script(1) not found; needed to give the piped run a terminal"
#
# The producer half matters too: a `cat` of the whole file finishes inside the
# pipe buffer long before bash reaches the tagged-installer handoff, so it can
# never be killed by a consumer that abandons the pipe the way a still-
# streaming curl can (the error 23 the #933 fix still had to drain for). The
# helper below trickles the bootstrap so it is mid-write at the handoff, and
# pipefail makes its exit status count — a consumer that closes the pipe early
# kills it with SIGPIPE and fails the run instead of hiding behind bash's own
# exit status.
PIPED_FEED=/tmp/vl-piped-feed.sh
cat >"$PIPED_FEED" <<'EOF'
#!/usr/bin/env bash
# Usage: vl-piped-feed.sh BOOTSTRAP [installer args...]
set -o pipefail
bootstrap=$1; shift
# ~16 KiB/s: still writing (and soon blocked on the full pipe buffer) when the
# bootstrap reaches the tagged-installer handoff, the way curl is mid-download.
drip() {
local file=$1 chunk rc=0
split -b 8192 "$file" /tmp/vl-feed-
for chunk in /tmp/vl-feed-*; do
# A cat killed by SIGPIPE means the consumer closed the pipe early — the
# download failure this arm exists to catch. Keep the status through the
# cleanup so pipefail above sees it.
cat "$chunk" || { rc=$?; break; }
sleep 0.5
done
rm -f /tmp/vl-feed-*
return "$rc"
}
drip "$bootstrap" | bash -s -- "$@"
EOF
chmod 0755 "$PIPED_FEED"
run_remote_install_piped() {
su - "$INSTALL_USER" -c "cd '$INSTALL_HOME' && VOCALINUX_REPO_URL='file://$MIRROR' script -qec \"'$PIPED_FEED' '$BOOTSTRAP' --auto --skip-models '--tag=$GATE_TAG'\" /dev/null"
}
assert_remote_install() {
echo "== Assert: the clone is exactly the selected tag =="
[ -d "$CLONE_DIR/.git" ] || fail "remote install left no clone at $CLONE_DIR"
local cloned_sha cloned_tree
cloned_sha="$(git -C "$CLONE_DIR" rev-parse HEAD)" \
|| fail "the clone at $CLONE_DIR is not a git checkout"
[ "$cloned_sha" = "$EXPECTED_SHA" ] || fail \
"installed from the wrong revision: clone HEAD is $cloned_sha, $GATE_TAG is $EXPECTED_SHA"
[ "$(git -C "$CLONE_DIR" rev-parse "$GATE_TAG^{commit}" 2>/dev/null || true)" = "$EXPECTED_SHA" ] \
|| fail "the clone does not carry the selected tag ref $GATE_TAG"
# The tag's commit is an orphan; tree equality is the check that installer,
# modules and exports are this commit's files and not a neighbour revision's.
cloned_tree="$(git -C "$CLONE_DIR" rev-parse 'HEAD^{tree}')"
[ "$cloned_tree" = "$SOURCE_TREE" ] || fail \
"the clone's tree $cloned_tree is not HEAD's tree $SOURCE_TREE"
# The artifacts the #701 guardrail names: the tagged installer, its sourced
# modules, and the exports pip installs with --require-hashes.
[ -f "$CLONE_DIR/install.d/system_dependencies.sh" ] \
|| fail "the clone ships no install.d/system_dependencies.sh for the tagged installer to source"
[ -f "$CLONE_DIR/requirements/runtime.txt" ] \
|| fail "the clone ships no requirements/runtime.txt for the tagged installer to install"
# Same shape as the local gate's #736 check, now reachable through the remote
# path: --skip-models means the run itself never reads the manifest.
[ -f "$CLONE_DIR/src/vocalinux/utils/model_checksums.txt" ] \
|| fail "the tag ships no src/vocalinux/utils/model_checksums.txt; install.sh pins model downloads against it"
echo "== Assert: the handoff carried the original arguments =="
# handoff_to_tagged_installer appends --venv-dir; a venv anywhere else means
# the appended argument was dropped.
[ -x "$REMOTE_VENV/bin/python" ] \
|| fail "no remote venv at $REMOTE_VENV; the handoff's --venv-dir did not reach the tagged installer"
# The remote marker relocates the activation helper to ~/.local/bin only when
# VOCALINUX_REMOTE_INSTALL survives the exec.
[ -f "$INSTALL_HOME/.local/bin/activate-vocalinux.sh" ] \
|| fail "no ~/.local/bin/activate-vocalinux.sh; the remote marker did not survive the handoff"
# --skip-models forwarded: a model in the data dir means the tagged installer
# re-parsed arguments the handoff did not deliver.
if [ -d "$INSTALL_HOME/.local/share/vocalinux/models" ]; then
[ -z "$(find "$INSTALL_HOME/.local/share/vocalinux/models" -type f -print -quit)" ] \
|| fail "models were downloaded; --skip-models did not survive the handoff"
fi
echo "== Assert: the installed app is there and answers =="
[ -x "$INSTALL_HOME/.local/bin/vocalinux" ] || fail "no wrapper at ~/.local/bin/vocalinux"
[ -x "$INSTALL_HOME/.local/bin/vocalinux-gui" ] || fail "no wrapper at ~/.local/bin/vocalinux-gui"
[ -f "$INSTALL_HOME/.local/share/applications/vocalinux.desktop" ] \
|| fail "no desktop entry; install_desktop_entry did not run"
# The version the wrapper reports must be the tag's own, not an assertion
# about a version string composed locally.
local expected_version version_output
expected_version="$(grep -oP '__version__\s*=\s*"\K[^"]+' "$CLONE_DIR/src/vocalinux/version.py")" \
|| fail "cannot read __version__ from the clone's version.py"
version_output="$(su - "$INSTALL_USER" -c "'$INSTALL_HOME/.local/bin/vocalinux' --version")" \
|| fail "the installed wrapper cannot report a version"
case "$version_output" in
*"$expected_version"*) ;;
*) fail "wrapper reports '$version_output', expected the tag's $expected_version" ;;
esac
echo " vocalinux --version: $version_output"
# Same blind spots as the local gate: the injection binaries are reached
# through subprocess, and the IBus runtime only where the package list
# promises it.
for tool in xclip xsel wl-copy; do
command -v "$tool" >/dev/null 2>&1 || fail "$tool is not on PATH; install.sh lists it for every distro"
done
case "$ID" in
arch | opensuse-tumbleweed | opensuse-leap | sles)
command -v ibus-daemon >/dev/null 2>&1 \
|| fail "no ibus-daemon: the package list ships the IBus typelib without the runtime ibus_engine.py spawns"
;;
esac
su - "$INSTALL_USER" -c "'$REMOTE_VENV/bin/python' - " <<'PY' || fail "the installation does not import"
import vocalinux.main
import vocalinux.ui.tray_indicator
import vocalinux.speech_recognition.recognition_manager
import vocalinux.text_injection.text_injector
from vocalinux.ui.keyboard_backends import EVDEV_AVAILABLE, PYNPUT_AVAILABLE
import gi
# keyboard_backends imports each backend under `except ImportError`, so the
# import above stays clean when evdev and pynput both failed to build. Assert
# the flags rather than the import.
assert EVDEV_AVAILABLE or PYNPUT_AVAILABLE, (
"no keyboard backend is importable: evdev and pynput both failed to install"
)
# The typelibs the installer is responsible for putting on the system. Both are
# loaded lazily at runtime, so importing the modules above proves nothing about
# them: a missing one would surface on a user's first dictation instead.
gi.require_version("Gtk", "3.0")
gi.require_version("IBus", "1.0")
from gi.repository import Gtk, IBus # noqa: F401
print(
" main, tray, recognition, injection, Gtk and IBus import;"
f" keyboard backends: evdev={EVDEV_AVAILABLE} pynput={PYNPUT_AVAILABLE}"
)
PY
}
echo "== Run 1: remote install (fresh clone) as $INSTALL_USER =="
START=$(date +%s)
run_remote_install || fail "the remote install exited non-zero on a fresh clone"
echo " install took $(( $(date +%s) - START ))s"
assert_remote_install
echo "== Run 2: re-run over the existing clone (fetch + reset), piped under a terminal =="
# The second bootstrap sees $CLONE_DIR/.git and updates it instead of cloning:
# the arm a user re-running curl|bash takes on every release after the first —
# served here literally through `cat | bash` on a pty so the terminal-backed
# pipe keeps regression coverage (#933).
# Move the clone off the tag and drop its local tag ref first: a run that only
# had to notice the right commit was already checked out would pass even if
# the fetch and reset were both no-ops.
AWAY_SHA="$(su - "$INSTALL_USER" -c "cd '$CLONE_DIR' && \
GIT_AUTHOR_NAME=gate GIT_AUTHOR_EMAIL=gate@local \
GIT_COMMITTER_NAME=gate GIT_COMMITTER_EMAIL=gate@local \
git commit-tree 'HEAD^{tree}' -p HEAD -m moved")"
su - "$INSTALL_USER" -c "git -C '$CLONE_DIR' reset --hard '$AWAY_SHA' \
&& git -C '$CLONE_DIR' update-ref -d 'refs/tags/$GATE_TAG'" \
|| fail "could not move the clone off $GATE_TAG for the re-run arm"
[ "$(git -C "$CLONE_DIR" rev-parse HEAD)" = "$AWAY_SHA" ] \
|| fail "the clone did not move off $GATE_TAG; the re-run proves nothing"
[ -z "$(git -C "$CLONE_DIR" tag -l "$GATE_TAG")" ] \
|| fail "the clone kept refs/tags/$GATE_TAG; the re-run proves nothing"
run_remote_install_piped || fail "the terminal-backed piped install exited non-zero over the existing clone"
grep -l "Updating existing clone" "$INSTALL_HOME"/.local/state/vocalinux/install-*.log >/dev/null 2>&1 \
|| fail "no install log shows 'Updating existing clone'; the fetch arm did not run"
assert_remote_install
echo "PASS: the remote install path (bootstrap -> tag -> clone/fetch -> handoff) installs ${NAME:-this distro}'s app"
scripts/install-test.sh
Executable
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env bash
# Run install.sh unattended in a distro container and check what it installed.
#
# Usage (a read-only mount is enough; the tree is taken via git archive):
# docker run --rm -v "$PWD:/repo:ro" debian:12 bash /repo/scripts/install-test.sh
# or: just install-gate debian:12
#
# Flags:
# --auto the only non-interactive mode; without it the script blocks
# on prompts that CI has no TTY to answer.
# --skip-models a model is 40-75 MB per container per run, and the download
# path is verified by checksum elsewhere.
# Not --skip-system-deps: the per-distro package installation is the point.
#
# install.sh refuses to run as root and calls sudo directly, so the container
# needs an unprivileged user with passwordless sudo. Running it as root would
# exercise a path no user takes.
#
# Local mode only: install.sh chooses it by finding a pyproject.toml naming
# vocalinux in the working directory, so extracting the tree and running from it
# tests this commit. Remote mode (curl | bash, which clones the latest tag) is
# not covered here.
#
# Failure mode to watch: the install pip-installs pywhispercpp against whatever
# CPython the distro ships. Wheels exist for manylinux_2_28, but when a rolling
# distro moves to a new interpreter before upstream publishes wheels for it, pip
# falls back to the sdist and this goes slow or red for an unrelated reason.
set -euo pipefail
REPO="${REPO:-/repo}"
INSTALL_USER="${INSTALL_USER:-installer}"
INSTALL_HOME="/home/$INSTALL_USER"
# Local mode puts the venv inside the tree, which is why the checkout is copied
# rather than written to: a bind-mounted host tree would come back owned by a
# container uid with a venv in it.
TREE="$INSTALL_HOME/vocalinux"
fail() {
echo "FAIL: $*" >&2
dump_install_log
exit 1
}
# install.sh writes a full transcript under ~/.local/state/vocalinux/. It is the
# only place the per-distro package manager output survives.
dump_install_log() {
local log
log="$(ls -1t "$INSTALL_HOME"/.local/state/vocalinux/install-*.log 2>/dev/null | head -n1 || true)"
if [ -n "$log" ]; then
echo "== last 200 lines of $log ==" >&2
tail -n 200 "$log" >&2
else
echo "== no install log was written ==" >&2
fi
}
# A mirror going slow mid-transaction leaves a half-populated cache, so
# re-running the same transaction resumes rather than restarts.
retry() {
local attempt
for attempt in 1 2 3; do
"$@" && return 0
echo " '$1' failed (attempt $attempt); retrying" >&2
sleep 5
done
return 1
}
[ -f "$REPO/install.sh" ] || fail "$REPO/install.sh not found; mount the checkout at \$REPO"
. /etc/os-release
echo "== ${NAME:-unknown} ${VERSION_ID:-} =="
echo "== Bootstrap: sudo, git and an unprivileged user =="
# The minimum for install.sh to start, and nothing it should install itself.
# git is for the archive below; local mode never calls it.
case "$ID" in
ubuntu | debian)
export DEBIAN_FRONTEND=noninteractive
retry apt-get update -qq
retry apt-get install -y -qq sudo git ca-certificates >/dev/null
;;
fedora | rhel | centos)
retry dnf install -y -q sudo git shadow-utils >/dev/null
;;
arch)
# -Syu, never -Sy: archlinux:latest lags the mirrors, and a partial upgrade
# would be this script's breakage rather than the installer's.
retry pacman -Syu --noconfirm --needed sudo git >/dev/null
;;
opensuse-tumbleweed | opensuse-leap | sles)
retry zypper --non-interactive --gpg-auto-import-keys refresh >/dev/null
retry zypper --non-interactive install -y sudo git >/dev/null
;;
*)
fail "no bootstrap arm for ID='$ID'; add one when adding the container"
;;
esac
id -u "$INSTALL_USER" >/dev/null 2>&1 || useradd -m -s /bin/bash "$INSTALL_USER"
echo "$INSTALL_USER ALL=(ALL) NOPASSWD: ALL" >"/etc/sudoers.d/$INSTALL_USER"
chmod 0440 "/etc/sudoers.d/$INSTALL_USER"
echo "== Extract this commit into $TREE =="
# git archive, not cp: it takes exactly the tracked files a user gets from a
# clone, so a build artefact or stray venv in the working tree cannot make this
# pass. safe.directory because the mount is owned by another uid.
git config --global --add safe.directory '*'
git -C "$REPO" rev-parse --verify HEAD >/dev/null \
|| fail "$REPO is not a git checkout; git archive needs it"
install -d -o "$INSTALL_USER" -g "$INSTALL_USER" "$TREE"
git -C "$REPO" archive HEAD | tar -x -C "$TREE"
chown -R "$INSTALL_USER:$INSTALL_USER" "$TREE"
# --skip-models below means install.sh never reads this manifest, so nothing in
# this run would notice it leaving the tree. #736 was the release-tag half of
# that shape: an installer verifying models against a manifest the tree it had
# just cloned did not carry. This is the half a local-mode gate can see.
[ -f "$TREE/src/vocalinux/utils/model_checksums.txt" ] \
|| fail "the tree ships no src/vocalinux/utils/model_checksums.txt; install.sh pins model downloads against it"
echo "== Run install.sh --auto --skip-models as $INSTALL_USER =="
# su -, so HOME and PATH are the user's own: install.sh writes wrappers into
# $HOME/.local/bin and reads $HOME throughout, and a run with root's HOME would
# install into the wrong place and still succeed.
START=$(date +%s)
if ! su - "$INSTALL_USER" -c "cd '$TREE' && bash install.sh --auto --skip-models"; then
fail "install.sh exited non-zero"
fi
echo " install took $(( $(date +%s) - START ))s"
echo "== Smoke: is there an installation, and does it import =="
VENV="$TREE/venv"
[ -x "$VENV/bin/python" ] || fail "no venv interpreter at $VENV/bin/python"
[ -x "$INSTALL_HOME/.local/bin/vocalinux" ] || fail "no wrapper at ~/.local/bin/vocalinux"
[ -f "$INSTALL_HOME/.local/share/applications/vocalinux.desktop" ] \
|| fail "no desktop entry; install_desktop_entry did not run"
# The injection backends are reached through subprocess, not GI, so no typelib
# import can see one of these missing. These are the binaries every distro's
# package list promises.
for tool in xclip xsel wl-copy; do
command -v "$tool" >/dev/null 2>&1 || fail "$tool is not on PATH; install.sh lists it for every distro"
done
# ibus-daemon and the ibus CLI are what ibus_engine.py actually spawns, and only
# the arch and suse lists carry the package that provides them. Fedora's
# ibus-devel and Ubuntu's gir1.2-ibus-1.0 pull the library and the typelib
# alone, so this records the split rather than hiding it: on the two distros
# that do promise the runtime, a swap to a typelib-only package fails here
# instead of on a user's first dictation.
case "$ID" in
arch | opensuse-tumbleweed | opensuse-leap | sles)
command -v ibus-daemon >/dev/null 2>&1 \
|| fail "no ibus-daemon: the package list ships the IBus typelib without the runtime ibus_engine.py spawns"
;;
esac
# Through the wrapper rather than the venv directly: the wrapper is what the
# desktop entry and the user's PATH invoke, and it carries the GI_TYPELIB_PATH
# and pywhispercpp library-path resolution the bare console script does not.
su - "$INSTALL_USER" -c "'$INSTALL_HOME/.local/bin/vocalinux' --version" \
|| fail "the installed wrapper cannot report a version"
su - "$INSTALL_USER" -c "'$VENV/bin/python' - " <<'PY' || fail "the installation does not import"
import vocalinux.main
import vocalinux.ui.tray_indicator
import vocalinux.speech_recognition.recognition_manager
import vocalinux.text_injection.text_injector
from vocalinux.ui.keyboard_backends import EVDEV_AVAILABLE, PYNPUT_AVAILABLE
import gi
# keyboard_backends imports each backend under `except ImportError`, so the
# import above stays clean when evdev and pynput both failed to build. That is
# the openSUSE failure this gate was extended for, so assert the flags rather
# than the import.
assert EVDEV_AVAILABLE or PYNPUT_AVAILABLE, (
"no keyboard backend is importable: evdev and pynput both failed to install"
)
# The typelibs the installer is responsible for putting on the system. Both are
# loaded lazily at runtime, so importing the modules above proves nothing about
# them: a missing one would surface on a user's first dictation instead.
gi.require_version("Gtk", "3.0")
gi.require_version("IBus", "1.0")
from gi.repository import Gtk, IBus # noqa: F401
print(
" main, tray, recognition, injection, Gtk and IBus import;"
f" keyboard backends: evdev={EVDEV_AVAILABLE} pynput={PYNPUT_AVAILABLE}"
)
PY
echo "PASS: install.sh installs on ${NAME:-this distro} (this commit, unattended)"
scripts/installer_requirements.py
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Select committed requirement blocks without changing pins, markers or hashes.
The installer uses complete exports for normal installs and just pywhispercpp's
block for backend rebuilds. Missing exports or packages are errors, never an
empty successful install. Only the standard library is needed before bootstrap.
"""
from __future__ import annotations
import argparse
import re
from pathlib import Path
def select_requirements(source: Path, package: str | None = None) -> str:
"""Return an export, optionally retaining only one distribution's blocks."""
text = source.read_text(encoding="utf-8")
blocks = re.split(r"(?=^[A-Za-z0-9_.-]+==)", text, flags=re.MULTILINE)
requirements = blocks[1:]
if not requirements:
raise ValueError(f"No pinned requirements in {source}")
if package is not None:
normalize = lambda name: re.sub(r"[-_.]+", "-", name).lower()
requirements = [
block
for block in requirements
if normalize(block.split("==", 1)[0]) == normalize(package)
]
if not requirements:
raise ValueError(f"No pin for {package} in {source}")
return blocks[0] + "".join(requirements)
def main() -> None:
"""Write the selected export to the installer's scratch directory."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("source", type=Path)
parser.add_argument("destination", type=Path)
parser.add_argument("--package")
args = parser.parse_args()
try:
text = select_requirements(args.source, args.package)
except (OSError, ValueError) as error:
parser.exit(1, f"Cannot install pinned dependencies: {error}\n")
args.destination.write_text(text, encoding="utf-8")
if __name__ == "__main__":
main()
scripts/prepare_aur_source.py
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Prepare the exact AUR source inputs, verifying published release provenance first.
Run with the locked release-build dependency group (provides packaging). Release
mode requires SHA256SUMS and the expected source commit. --local-unattested is
only for PR/local fixtures built by the reproducible-wheel gate, never publishing.
"""
from __future__ import annotations
import argparse
import hashlib
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
from email.parser import BytesParser
from pathlib import Path
from packaging.version import Version
ROOT = Path(__file__).resolve().parents[1]
TEMPLATE = ROOT / "packaging/aur/vocalinux/PKGBUILD"
REPOSITORY = "VocaHQ/vocalinux"
def digest(path: Path) -> str:
"""Hash the original archive bytes, without extracting or repacking them."""
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def sdist_version(sdist: Path) -> str:
"""Require one canonical Vocalinux sdist with matching top-level metadata."""
match = re.fullmatch(r"vocalinux-([0-9][A-Za-z0-9.+!]*)\.tar\.gz", sdist.name)
if not match:
raise ValueError("expected a canonical vocalinux-VERSION.tar.gz sdist")
version = match[1]
if str(Version(version)) != version:
raise ValueError("sdist filename version is not canonical")
root = f"vocalinux-{version}"
with tarfile.open(sdist, "r:gz") as archive:
members = archive.getmembers()
if any(
member.name.split("/", 1)[0] != root or ".." in member.name.split("/")
for member in members
):
raise ValueError("sdist has an unexpected archive root or unsafe member path")
metadata = [member for member in members if member.name == f"{root}/PKG-INFO"]
if len(metadata) != 1 or not metadata[0].isfile():
raise ValueError("sdist must contain exactly one regular top-level PKG-INFO")
stream = archive.extractfile(metadata[0])
assert stream is not None
with stream:
info = BytesParser().parse(stream)
if info.get_all("Name") != ["vocalinux"] or info.get_all("Version") != [version]:
raise ValueError("sdist metadata does not match its filename")
return version
def verify_release(sdist: Path, manifest: Path, tag: str, commit: str) -> str:
"""Fail closed on missing/mismatched hashes or untrusted release provenance."""
if not re.fullmatch(r"[0-9a-f]{40}", commit):
raise ValueError("release verification requires a full source commit SHA")
entries = []
for line in manifest.read_text(encoding="utf-8").splitlines():
match = re.fullmatch(r"([0-9a-f]{64}) (\S+)", line)
if not match:
raise ValueError("malformed SHA256SUMS entry")
if match[2] == sdist.name:
entries.append(match[1])
if len(entries) != 1:
raise ValueError("SHA256SUMS must pin the sdist exactly once")
if digest(sdist) != entries[0]:
raise ValueError("sdist checksum does not match SHA256SUMS")
subprocess.run(
[
"gh",
"attestation",
"verify",
str(sdist),
"--repo",
REPOSITORY,
"--signer-workflow",
f"{REPOSITORY}/.github/workflows/release.yml",
"--source-ref",
f"refs/tags/{tag}",
"--source-digest",
commit,
],
check=True,
)
return entries[0]
def prepare(
sdist: Path,
output: Path,
tag: str | None = None,
manifest: Path | None = None,
commit: str | None = None,
local_unattested: bool = False,
) -> None:
"""Stage an unchanged sdist and its fully pinned PKGBUILD only after validation."""
version = sdist_version(sdist)
tag = tag or (f"v{version}" if local_unattested else "")
if not re.fullmatch(r"v[0-9][A-Za-z0-9.+!-]*", tag) or Version(tag[1:]) != Version(version):
raise ValueError("release tag and sdist version do not match")
if local_unattested:
if manifest is not None or commit is not None:
raise ValueError("local-unattested mode cannot take release verification inputs")
print("UNATTESTED local/PR fixture: not authorized for AUR publication")
checksum = digest(sdist)
else:
if manifest is None or commit is None:
raise ValueError("release mode requires --manifest and --source-commit")
checksum = verify_release(sdist, manifest, tag, commit)
text = TEMPLATE.read_text(encoding="utf-8")
for key, value in {
"pkgver": tag[1:].replace("-", ""),
"_tag": tag[1:],
"_version": version,
"pkgrel": "1",
"sha256sums": f"('{checksum}')",
}.items():
text, count = re.subn(rf"^{key}=.*$", f"{key}={value}", text, flags=re.MULTILINE)
if count != 1:
raise ValueError(f"PKGBUILD template must assign {key} exactly once")
if output.exists() or output.is_symlink():
raise ValueError(f"output already exists; choose a fresh directory: {output}")
output.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".aur-source-", dir=output.parent) as temporary:
staging = Path(temporary) / "source"
staging.mkdir()
shutil.copyfile(sdist, staging / sdist.name)
if digest(staging / sdist.name) != checksum:
raise ValueError("sdist changed while staging")
(staging / "PKGBUILD").write_text(text, encoding="utf-8")
staging.rename(output)
print(f"Prepared {tag}: {checksum} {sdist.name} in {output}")
def main() -> int:
"""Prepare verified release inputs or an explicitly unattested local fixture."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--sdist", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--tag", help="exact v* tag; defaults to the sdist version in local mode")
parser.add_argument("--manifest", type=Path, help="published SHA256SUMS")
parser.add_argument("--source-commit", help="expected full commit SHA of the release tag")
parser.add_argument("--local-unattested", action="store_true", help="PR/local fixture only")
args = parser.parse_args()
try:
prepare(
args.sdist,
args.output,
args.tag,
args.manifest,
args.source_commit,
args.local_unattested,
)
except (OSError, ValueError, tarfile.TarError, subprocess.CalledProcessError) as error:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/sync_flatpak_deps.py
Executable
+355
View File
@@ -0,0 +1,355 @@
#!/usr/bin/env python3
"""Point the Flatpak's dependency manifest at the versions uv.lock resolved.
`packaging/flatpak/python3-dependencies.yaml` was produced once by
flatpak-pip-generator, from a package list typed out by hand on the command
line recorded in its first line, and then hand-edited. Nothing regenerated it
afterwards and no test compared it to anything, so it drifted: by 2026-09-10 ten
of its fifteen shared packages were behind `requirements/runtime.txt`, including
pywhispercpp, which sat at 1.4.1 under the project's own `>=1.5.0` floor. The
Flatpak installs the project with `pip3 install --no-deps`, so that constraint
is never evaluated and nothing there could notice. It now reads
`requirements/vad.txt`, the runtime dependencies plus the `[vad]` extra, so
Silero VAD's onnxruntime ships too (#969).
This script closes the loop the other way round: it reads the hash-pinned export
that `just lock` produces and rewrites every source in the manifest to the
artifact whose digest uv already recorded. The bytes the Flatpak downloads are
then the bytes in `uv.lock`, verified by hash rather than by convention. The
`onnxruntime` sources are selected as CPython-ABI wheels for each Flatpak
architecture because PyPI publishes no sdist for it.
It edits in place rather than regenerating, because two things in that file are
hand-written and no generator produces them: pywhispercpp's Vulkan build with
its version injection, and the symlinking of whisper.cpp's shared libraries onto
the loader path. Keeping the structure keeps the diff to versions and digests.
Usage:
python scripts/sync_flatpak_deps.py # rewrite the manifest
python scripts/sync_flatpak_deps.py --check # report drift, change nothing
`tests/test_flatpak_packaging.py` asserts the same invariant offline, so CI
catches drift without reaching PyPI.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
import urllib.request
from pathlib import Path
from packaging.markers import Marker
from packaging.utils import canonicalize_name
REPO_ROOT = Path(__file__).resolve().parents[1]
EXPORT = REPO_ROOT / "requirements" / "vad.txt"
DEPS_YAML = REPO_ROOT / "packaging" / "flatpak" / "python3-dependencies.yaml"
MANIFEST = REPO_ROOT / "packaging" / "flatpak" / "com.vocalinux.Vocalinux.yml"
#: The environment the Flatpak build resolves for. GNOME 50 ships Python 3.13.
#: numpy's lock entry splits at `python_full_version >= '3.12'`; onnxruntime's
#: CPython wheel tag also uses this Python version. Bump it when the manifest's
#: runtime-version moves.
TARGET_ENV = {
"sys_platform": "linux",
"platform_system": "Linux",
"platform_machine": "x86_64",
"os_name": "posix",
"python_version": "3.13",
"python_full_version": "3.13.0",
"implementation_name": "cpython",
"platform_python_implementation": "CPython",
"extra": "",
}
#: org.gnome.Platform//50 ships these; a second copy under /app would shadow the
#: runtime's own GI stack. PyGObject is already absent from the export, which
#: `just lock` builds with --no-emit-package pygobject for the same reason.
RUNTIME_PROVIDED = {"pycairo"}
#: onnxruntime has no sdist on PyPI, and building it from source is out of reach
#: in the sandbox. These come as CPython-ABI wheels, one per Flatpak arch, tagged
#: for TARGET_ENV's python_version; a runtime Python bump makes pip --no-index
#: fail loudly at build time rather than silently.
BINARY_ONLY = {"onnxruntime"}
FLATPAK_ARCHES = ("x86_64", "aarch64")
#: Build backends for the sdists the manifest builds offline under
#: --no-build-isolation. They are not runtime dependencies, so they never appear
#: in requirements/vad.txt and stay pinned here by hand.
BUILD_BACKENDS = {"meson-python", "pyproject-metadata"}
PYPI_JSON = "https://pypi.org/pypi/{name}/{version}/json"
def parse_export(path: Path) -> dict[str, tuple[str, set[str]]]:
"""Map every package the Flatpak target selects to its version and digests.
uv writes one `name==version [; marker]` line followed by indented
`--hash=sha256:` lines, continued with backslashes. A package can appear
more than once under mutually exclusive markers, which is how numpy is
split across Python versions, so the marker decides rather than the order.
"""
selected: dict[str, tuple[str, set[str]]] = {}
current: str | None = None
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.strip().rstrip("\\").strip()
if not line or line.startswith("#"):
continue
head = re.match(r"^([A-Za-z0-9._-]+)==([^ ;]+)\s*(?:;\s*(.*))?$", line)
if head:
name, version, marker = head.group(1), head.group(2), head.group(3)
current = None
if marker and not Marker(marker).evaluate(TARGET_ENV):
continue
key = canonicalize_name(name)
if key in selected:
raise SystemExit(
f"{path.name}: {key} resolves twice for the Flatpak target "
f"({selected[key][0]} and {version}); the markers overlap"
)
selected[key] = (version, set())
current = key
continue
digest = re.match(r"^--hash=sha256:([0-9a-f]{64})$", line)
if digest and current:
selected[current][1].add(digest.group(1))
return selected
def pypi_artifacts(name: str, version: str) -> list[dict]:
url = PYPI_JSON.format(name=name, version=version)
with urllib.request.urlopen(url, timeout=30) as response: # noqa: S310
return json.load(response)["urls"]
def choose_artifact(
name: str, version: str, digests: set[str], arch: str | None = None
) -> tuple[str, str]:
"""The one file the Flatpak should download, as (url, sha256).
Binary-only packages use an architecture-specific CPython wheel. For every
other package, prefer a universal wheel and fall back to the sdist so the
sandbox builds compiled dependencies against the SDK's interpreter.
"""
artifacts = pypi_artifacts(name, version)
if name in BINARY_ONLY:
if arch not in FLATPAK_ARCHES:
raise SystemExit(
f"{name} {version}: expected one of the Flatpak architectures "
f"{', '.join(FLATPAK_ARCHES)}, got {arch!r}"
)
python_tag = TARGET_ENV["python_version"].replace(".", "")
wheel_pattern = re.compile(
rf"-cp{python_tag}-cp{python_tag}-manylinux_.+_{re.escape(arch)}\.whl$"
)
candidates = [
artifact
for artifact in artifacts
if artifact["packagetype"] == "bdist_wheel"
and wheel_pattern.search(artifact["filename"])
]
if not candidates:
raise SystemExit(
f"{name} {version} publishes no CPython cp{python_tag} manylinux "
f"wheel for {arch}; update the Flatpak Python runtime or package"
)
else:
wheels = [
artifact
for artifact in artifacts
if artifact["packagetype"] == "bdist_wheel"
and artifact["filename"].endswith("-none-any.whl")
]
sdists = [artifact for artifact in artifacts if artifact["packagetype"] == "sdist"]
candidates = wheels or sdists
if not candidates:
raise SystemExit(
f"{name} {version} publishes neither a universal wheel nor an sdist; "
"the Flatpak cannot build it without an ABI-tagged wheel"
)
# A release can publish more than one matching wheel; take the first whose
# digest uv actually locked.
picked = next((a for a in candidates if a["digests"]["sha256"] in digests), None)
if picked is None:
first = candidates[0]
raise SystemExit(
f"{name} {version}: PyPI serves {first['filename']} with digest "
f"{first['digests']['sha256']}, which is not among the hashes uv "
"locked. Re-run `just lock`."
)
return picked["url"], picked["digests"]["sha256"]
def artifact_name(filename: str) -> str:
if filename.endswith(".whl"):
return canonicalize_name(filename.split("-")[0])
base = re.sub(r"\.(tar\.gz|zip|tar\.bz2)$", "", filename)
return canonicalize_name(base.rpartition("-")[0])
def manifest_sources(text: str) -> list[tuple[int, str, str]]:
"""Every PyPI source in the manifest, as (line index, filename, sha256)."""
lines = text.splitlines()
found = []
for index, line in enumerate(lines):
url = re.match(r"^\s*url: (https://files\.pythonhosted\.org/\S+/([^/\s]+))$", line)
if not url:
continue
digest = re.match(r"^\s*sha256: ([0-9a-f]{64})\s*$", lines[index + 1])
if not digest:
raise SystemExit(
f"{DEPS_YAML.name}:{index + 2}: a PyPI source with no sha256 beneath it"
)
found.append((index, url.group(2), digest.group(1)))
return found
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--check",
action="store_true",
help="report what would change and exit non-zero, without writing",
)
args = parser.parse_args()
locked = parse_export(EXPORT)
text = DEPS_YAML.read_text(encoding="utf-8")
lines = text.splitlines(keepends=True)
sources = manifest_sources(text)
in_manifest = {artifact_name(name) for _, name, _ in sources}
stale = sorted(in_manifest - set(locked) - BUILD_BACKENDS)
missing = sorted(set(locked) - in_manifest - RUNTIME_PROVIDED)
binary_arches = {name: set() for name in BINARY_ONLY}
for _, filename, _ in sources:
name = artifact_name(filename)
if name in BINARY_ONLY:
arch_match = re.search(r"_(x86_64|aarch64)\.whl$", filename)
if arch_match:
binary_arches[name].add(arch_match.group(1))
for name in BINARY_ONLY & set(locked):
missing.extend(
f"{name} ({arch})" for arch in FLATPAK_ARCHES if arch not in binary_arches[name]
)
missing.sort()
if stale:
print(
f"{DEPS_YAML.name} builds packages the lock does not resolve: "
f"{', '.join(stale)}\n"
"Delete their modules and sources by hand: whether a package is a "
"build backend the sandbox needs or a leftover this script cannot "
"tell.",
file=sys.stderr,
)
if missing:
print(
f"{DEPS_YAML.name} is missing packages the lock resolves: "
f"{', '.join(missing)}\n"
"The Flatpak installs with --no-deps, so a missing one fails at "
"runtime rather than at build time. Add a module for each.",
file=sys.stderr,
)
if stale or missing:
return 1
changes: list[str] = []
for index, filename, old_sha in sources:
name = artifact_name(filename)
if name in BUILD_BACKENDS:
# Absent from the lock by definition: nothing imports them at
# runtime, they only let the sandbox build the sdists offline.
continue
version, digests = locked[name]
arch = None
if name in BINARY_ONLY:
arch_match = re.search(r"_(x86_64|aarch64)\.whl$", filename)
if not arch_match:
raise SystemExit(f"{name}: cannot determine a Flatpak architecture from {filename}")
arch = arch_match.group(1)
url, sha = choose_artifact(name, version, digests, arch)
if sha == old_sha:
continue
indent = re.match(r"^(\s*)", lines[index]).group(1)
lines[index] = f"{indent}url: {url}\n"
lines[index + 1] = f"{indent}sha256: {sha}\n"
changes.append(f" {name}: {filename} -> {url.rsplit('/', 1)[-1]}")
updated = "".join(lines)
# pywhispercpp's version is spelled out in its build commands: the sdist
# carries no version.txt and setup() takes no version= argument, so the
# module rewrites setup.py before building. setuptools-scm would otherwise
# read a git tree that does not exist in the sandbox.
pywhispercpp_version = locked["pywhispercpp"][0]
updated, injected = re.subn(
# The version appears as a tarball name, a directory name and a
# literal inside the setup.py rewrite. Refuse to match rather than
# match short: a partial match would leave the tail of the old version
# glued to the new one and rename the directory tar just wrote.
r"pywhispercpp-\d+(?:\.\d+)*(?![\d.]\d)",
f"pywhispercpp-{pywhispercpp_version}",
updated,
)
updated = re.sub(
r"PYWHISPERCPP_VERSION=\d+(?:\.\d+)*",
f"PYWHISPERCPP_VERSION={pywhispercpp_version}",
updated,
)
updated = re.sub(
r'(version=\\")\d+(?:\.\d+)*(\\")',
rf"\g<1>{pywhispercpp_version}\g<2>",
updated,
)
if injected and updated != text and not changes:
changes.append(f" pywhispercpp: build commands -> {pywhispercpp_version}")
manifest_text = MANIFEST.read_text(encoding="utf-8")
manifest_updated = re.sub(
r'(PYWHISPERCPP_VERSION: ")\d+(?:\.\d+)*(")',
rf"\g<1>{pywhispercpp_version}\g<2>",
manifest_text,
)
if updated == text and manifest_updated == manifest_text:
print(
"Flatpak dependencies already match requirements/vad.txt "
"(runtime dependencies plus the [vad] extra)"
)
return 0
if args.check:
print(
"Flatpak dependencies are behind requirements/vad.txt "
"(runtime dependencies plus the [vad] extra):",
file=sys.stderr,
)
for change in changes:
print(change, file=sys.stderr)
if manifest_updated != manifest_text:
print(
f" {MANIFEST.name}: PYWHISPERCPP_VERSION -> {pywhispercpp_version}",
file=sys.stderr,
)
print("Run `just flatpak-deps` to update.", file=sys.stderr)
return 1
DEPS_YAML.write_text(updated, encoding="utf-8")
MANIFEST.write_text(manifest_updated, encoding="utf-8")
print(f"Updated {DEPS_YAML.relative_to(REPO_ROOT)}:")
for change in changes:
print(change)
if manifest_updated != manifest_text:
print(
f"Updated {MANIFEST.relative_to(REPO_ROOT)}: "
f"PYWHISPERCPP_VERSION={pywhispercpp_version}"
)
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/test_remote_server.py
Executable
+219
View File
@@ -0,0 +1,219 @@
#!/usr/bin/env python3
"""
Vocalinux Remote API Test Server
A simple mock server for testing Vocalinux's remote API feature.
Supports whisper.cpp (/inference) and OpenAI/FunASR-compatible
(/v1/audio/transcriptions) formats.
Usage:
python test_remote_server.py [--port PORT] [--delay SECONDS]
Examples:
python test_remote_server.py # Default: port 8080, no delay
python test_remote_server.py --port 9000 # Custom port
python test_remote_server.py --delay 2 # Simulate 2s processing delay
"""
import argparse
import json
import time
from http.server import BaseHTTPRequestHandler, HTTPServer
class RemoteAPITestHandler(BaseHTTPRequestHandler):
"""HTTP request handler for testing Vocalinux remote API."""
def log_message(self, format, *args):
"""Custom log format with timestamp."""
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] {args[0]}")
def do_GET(self):
"""Handle GET requests (health check and OpenAI models endpoint)."""
if self.path == "/":
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
response = {
"status": "ok",
"message": "Vocalinux Remote API Test Server",
"endpoints": [
"/inference (whisper.cpp format)",
"/v1/audio/transcriptions (OpenAI/FunASR format)",
"/v1/models (OpenAI models list)",
],
}
self.wfile.write(json.dumps(response, indent=2).encode())
elif self.path == "/v1/models":
# OpenAI-compatible models endpoint
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
response = {
"data": [
{
"id": "whisper-1",
"object": "model",
"created": 1234567890,
"owned_by": "openai",
},
{
"id": "sensevoice",
"object": "model",
"created": 1767225600,
"owned_by": "mock",
},
],
"object": "list",
}
self.wfile.write(json.dumps(response, indent=2).encode())
elif self.path == "/inference":
# whisper.cpp server health check
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
response = {"status": "ok", "model": "mock-whisper"}
self.wfile.write(json.dumps(response, indent=2).encode())
else:
self.send_response(404)
self.send_header("Content-Type", "application/json")
self.end_headers()
response = {"error": "Not found"}
self.wfile.write(json.dumps(response, indent=2).encode())
def do_POST(self):
"""Handle POST requests (transcription endpoints)."""
# Simulate processing delay if configured
if self.server.delay > 0:
time.sleep(self.server.delay)
if self.path in ["/inference", "/v1/audio/transcriptions"]:
# Parse multipart form data (simplified - just read the body)
content_length = int(self.headers.get("Content-Length", 0))
body = self.rfile.read(content_length)
# Extract filename if present
filename = "audio.wav"
if b"filename=" in body:
# Simple extraction from multipart data
try:
start = body.index(b'filename="') + 10
end = body.index(b'"', start)
filename = body[start:end].decode("utf-8", errors="ignore")
except (ValueError, IndexError):
pass
# Generate mock transcription based on filename
transcription = self._generate_mock_transcription(filename)
# Send response
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
if self.path == "/inference":
# whisper.cpp format
response = {"text": transcription}
else:
# OpenAI/FunASR-compatible format
response = {"text": transcription}
self.wfile.write(json.dumps(response, indent=2).encode())
print(f" → Transcribed '{filename}': {transcription}")
else:
self.send_response(404)
self.send_header("Content-Type", "application/json")
self.end_headers()
response = {"error": "Not found"}
self.wfile.write(json.dumps(response, indent=2).encode())
def _generate_mock_transcription(self, filename: str) -> str:
"""Generate a mock transcription based on the filename."""
# Different responses based on filename to make testing more interesting
if "test" in filename.lower():
return "This is a test transcription from the mock server."
elif "hello" in filename.lower():
return "Hello, this is Vocalinux testing the remote API."
elif "long" in filename.lower():
return (
"This is a longer transcription to test how the system handles "
"multiple sentences and proper text injection into applications. "
"The quick brown fox jumps over the lazy dog."
)
else:
return "Mock transcription from test server."
class DelayedHTTPServer(HTTPServer):
"""HTTP Server with configurable delay."""
def __init__(self, server_address, RequestHandlerClass, delay=0):
self.delay = delay
super().__init__(server_address, RequestHandlerClass)
def main():
parser = argparse.ArgumentParser(
description="Vocalinux Remote API Test Server"
)
parser.add_argument(
"--port",
type=int,
default=8080,
help="Port to listen on (default: 8080)",
)
parser.add_argument(
"--delay",
type=float,
default=0,
help="Simulated processing delay in seconds (default: 0)",
)
parser.add_argument(
"--host",
type=str,
default="0.0.0.0",
help="Host to bind to (default: 0.0.0.0)",
)
args = parser.parse_args()
print("=" * 60)
print("Vocalinux Remote API Test Server")
print("=" * 60)
print(f"Listening on: http://{args.host}:{args.port}")
print(f"Processing delay: {args.delay}s")
print()
print("Endpoints:")
print(" GET / - Server info")
print(" GET /v1/models - OpenAI models list")
print(" GET /inference - whisper.cpp health check")
print(" POST /inference - whisper.cpp transcription")
print(" POST /v1/audio/transcriptions - OpenAI/FunASR transcription")
print()
print("Press Ctrl+C to stop")
print("=" * 60)
print()
try:
server = DelayedHTTPServer(
(args.host, args.port), RemoteAPITestHandler, delay=args.delay
)
server.serve_forever()
except KeyboardInterrupt:
print("\n\nServer stopped.")
except Exception as e:
print(f"\nError: {e}")
if args.port < 1024:
print("Note: Ports below 1024 require root privileges.")
return 1
return 0
if __name__ == "__main__":
exit(main())
scripts/verify_release.py
Executable
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env python3
"""Check that a published release verifies as published.
The guards in tests/ read workflow files, so none of them can see what a
release looks like once its run has finished. This reads the release:
manifest SHA256SUMS is attached, lists every other asset, and its digests
match the bytes GitHub stores
provenance every artifact the manifest lists resolves in the attestations API
notes the body still tells users how to check both
pypi the wheel and the sdist on PyPI are the bytes on the release
Nothing is downloaded: GitHub reports a sha256 for every asset it stores, and an
asset it reports none for fails rather than being skipped.
Usage: scripts/verify_release.py [tag] [--if-published]
Default tag: the latest stable release. --if-published passes a tag that
published nothing.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
MANIFEST = "SHA256SUMS"
PYPI_PROJECT = "vocalinux"
PYPI_SUFFIXES = (".whl", ".tar.gz")
PYPI_TIMEOUT = 30
#: Both commands the notes should hand the user, plus the file they act on.
NOTES_MUST_MENTION = ("sha256sum -c", "gh attestation verify", MANIFEST)
#: What `gh attestation verify` asks for, so this asks for the same. Pre-encoded:
#: `gh api` reads an unescaped `://` in a query as a protocol and refuses.
SLSA_PROVENANCE = "https%3A%2F%2Fslsa.dev%2Fprovenance%2Fv1"
class ReleaseNotFound(SystemExit):
"""The API confirmed no release for the tag, with a 404.
Only this is forgivable. A rate limit or an auth failure stays a plain
SystemExit, or --if-published would pass a release nothing has read.
"""
def _gh(*args: str) -> subprocess.CompletedProcess:
return subprocess.run(["gh", *args], capture_output=True, text=True)
def _gh_ok(*args: str) -> str:
done = _gh(*args)
if done.returncode != 0:
raise SystemExit(f"gh {' '.join(args)} failed:\n{done.stderr.strip()}")
return done.stdout
def repo_slug() -> str:
slug = os.environ.get("GITHUB_REPOSITORY")
return slug or json.loads(_gh_ok("repo", "view", "--json", "nameWithOwner"))["nameWithOwner"]
def _release_from_api(payload: dict) -> dict:
"""Map a Releases REST payload onto the field names the rest of this script uses."""
return {
"tagName": payload["tag_name"],
"body": payload.get("body"),
"isDraft": payload["draft"],
"isPrerelease": payload["prerelease"],
"assets": payload.get("assets") or [],
}
def fetch_release(tag: str | None) -> dict:
"""The latest stable release, or the one named by tag. `0.16.2` is accepted
as well as `v0.16.2`, since that is how the version is usually written.
Digests come from the Releases REST API (`gh api`), not `gh release view
--json assets`. On gh 2.46 (common in distro packages, and what
`just verify-release` often runs) view's asset objects have no digest
field at all, so every asset would fail as undigested. The REST payload
includes `digest` for assets GitHub stores a hash for.
"""
slug = repo_slug()
candidates = [tag] + ([f"v{tag}"] if tag and tag[0].isdigit() else [])
errors: list[str] = []
absent = True
for candidate in candidates:
path = (
f"/repos/{slug}/releases/tags/{candidate}"
if candidate
else f"/repos/{slug}/releases/latest"
)
done = _gh("api", path)
if done.returncode == 0:
return _release_from_api(json.loads(done.stdout))
detail = (done.stderr or done.stdout or "").strip()
absent = absent and "HTTP 404" in detail
errors.append(f"{path}: {detail or f'exit {done.returncode}'}")
named = " or ".join(c for c in candidates if c) or "the latest stable release"
hint = "\n".join(errors)
raise (ReleaseNotFound if absent else SystemExit)(
f"no release found for {named}\n{hint}\nusage: verify_release.py [tag], e.g. v0.16.2"
)
def parse_manifest(text: str) -> dict[str, str]:
"""`<digest> <name>` per line, as sha256sum writes and reads it."""
entries = {}
for number, line in enumerate(text.splitlines(), 1):
if not line.strip():
continue
parts = line.split(None, 1)
if len(parts) != 2:
raise ValueError(f"{MANIFEST} line {number} is not `<digest> <name>`: {line!r}")
# Binary mode marks the name with a leading star, which is not the name.
entries[parts[1].strip().lstrip("*")] = parts[0].strip().lower()
return entries
def asset_digests(assets: list[dict]) -> tuple[dict[str, str], list[str]]:
"""(name -> sha256, names GitHub reports no digest for)."""
digests: dict[str, str] = {}
undigested: list[str] = []
for asset in assets:
raw = (asset.get("digest") or "").strip().lower()
if raw.startswith("sha256:"):
digests[asset["name"]] = raw.split(":", 1)[1]
else:
undigested.append(asset["name"])
return digests, undigested
def check_manifest(stored: dict, manifest: dict, undigested: list) -> list[str]:
"""Both directions: a missing line reads as tampering to anyone running
`sha256sum -c`, and an extra one is a promise nothing keeps."""
covered, listed = set(stored) - {MANIFEST}, set(manifest)
return (
[f"GitHub reports no digest for {name}" for name in undigested if name != MANIFEST]
+ [f"{name} is published but absent from {MANIFEST}" for name in sorted(covered - listed)]
+ [f"{MANIFEST} lists {name}, which is not an asset" for name in sorted(listed - covered)]
+ [
f"{name}: {MANIFEST} says {manifest[name]}, GitHub stores {stored[name]}"
for name in sorted(covered & listed)
if manifest[name] != stored[name]
]
)
def check_provenance(slug: str, manifest: dict[str, str]) -> list[str]:
"""The manifest is the subject-checksums input, so it is not a subject itself."""
problems = []
for name, digest in sorted(manifest.items()):
query = f"/repos/{slug}/attestations/sha256:{digest}"
done = _gh("api", f"{query}?predicate_type={SLSA_PROVENANCE}")
try:
bundles = json.loads(done.stdout).get("attestations") if done.returncode == 0 else None
except json.JSONDecodeError:
bundles = None
if not bundles:
problems.append(f"{name} has no build provenance")
return problems
def check_notes(body: str | None) -> list[str]:
# GitHub returns JSON null for an empty body; treat that as missing notes.
text = body or ""
return [f"the notes never mention `{p}`" for p in NOTES_MUST_MENTION if p not in text]
def check_pypi(version: str, stored: dict[str, str]) -> list[str]:
expected = {n: d for n, d in stored.items() if n.endswith(PYPI_SUFFIXES)}
# Per suffix: a release that lost its sdist would otherwise pass on the wheel.
problems = [
f"the release carries no {suffix} to compare against PyPI"
for suffix in PYPI_SUFFIXES
if not any(name.endswith(suffix) for name in expected)
]
if not expected:
return problems
# Past the early return, and not at module scope: urllib.error reaches
# tempfile, which two tests here leave as a MagicMock in sys.modules.
import urllib.error
import urllib.request
try:
url = f"https://pypi.org/pypi/{PYPI_PROJECT}/{version}/json"
with urllib.request.urlopen(url, timeout=PYPI_TIMEOUT) as response:
payload = json.load(response)
except urllib.error.HTTPError as error:
if error.code == 404:
return problems + [f"PyPI has no {PYPI_PROJECT} {version}"]
raise
published = {i["filename"]: i["digests"]["sha256"].lower() for i in payload["urls"]}
for name, digest in sorted(expected.items()):
if name not in published:
problems.append(f"{name} is on the release but not on PyPI")
elif published[name] != digest:
problems.append(f"{name}: PyPI serves {published[name]}, the release serves {digest}")
for name in sorted(published):
if name.endswith(PYPI_SUFFIXES) and name not in expected:
problems.append(f"{name} is on PyPI but not on the release")
return problems
def report(results: list[tuple[str, list[str]]]) -> bool:
width = max(len(label) for label, _ in results)
for label, problems in results:
print(f" {label.ljust(width)} {'FAIL' if problems else 'PASS'}")
for problem in problems:
print(f" {' ' * width} {problem}")
return not any(problems for _, problems in results)
def main(argv: list[str]) -> int:
args = [arg for arg in argv[1:] if arg]
if_published = "--if-published" in args
positional = [arg for arg in args if arg != "--if-published"]
tag = positional[0] if positional else None
slug = repo_slug()
try:
release = fetch_release(tag)
except ReleaseNotFound:
if not if_published:
raise
# Nothing published, nothing to be wrong about, and the Release run
# has already gone red.
print(f"no release published for {tag or 'the latest stable tag'}, nothing to verify")
return 0
if release["isDraft"] or release["isPrerelease"]:
# Nightlies never go through publish-checksums and carry no manifest.
print(f"{release['tagName']} is not a published stable release")
return 1
tag = release["tagName"]
stored, undigested = asset_digests(release["assets"])
print(f"Verifying {slug} {tag}, {len(release['assets'])} assets\n")
if MANIFEST not in stored and MANIFEST not in undigested:
results = [("manifest", [f"{tag} has no {MANIFEST} attached"])]
else:
text = _gh_ok("release", "download", tag, "--pattern", MANIFEST, "--output", "-")
manifest = parse_manifest(text)
results = [
("manifest", check_manifest(stored, manifest, undigested)),
("provenance", check_provenance(slug, manifest)),
("notes", check_notes(release["body"])),
("pypi", check_pypi(tag.removeprefix("v"), stored)),
]
ok = report(results)
print(f"\n{tag} {'verifies' if ok else 'does not verify'} as published")
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main(sys.argv))