xavierk
180cd4d512
Model recommendation keyed to the installed build and free memory, fork-sourced Void templates, signed xbps release workflow on Gitea with the pinned repository key, niri injection guard and wtype-first order, tightened naming sweep, web copy fixes, and CI security and duplication fixes.
357 lines
14 KiB
Python
Executable File
357 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Point the Flatpak's dependency manifest at the versions uv.lock resolved.
|
|
|
|
`packaging/flatpak/python3-dependencies.yaml` was produced once by
|
|
flatpak-pip-generator, from a package list typed out by hand on the command
|
|
line recorded in its first line, and then hand-edited. Nothing regenerated it
|
|
afterwards and no test compared it to anything, so it drifted: by 2026-09-10 ten
|
|
of its fifteen shared packages were behind `requirements/runtime.txt`, including
|
|
pywhispercpp, which sat at 1.4.1 under the project's own `>=1.5.0` floor. The
|
|
Flatpak installs the project with `pip3 install --no-deps`, so that constraint
|
|
is never evaluated and nothing there could notice. It now reads
|
|
`requirements/vad.txt`, the runtime dependencies plus the `[vad]` extra, so
|
|
Silero VAD's onnxruntime ships too (#969).
|
|
|
|
This script closes the loop the other way round: it reads the hash-pinned export
|
|
that `just lock` produces and rewrites every source in the manifest to the
|
|
artifact whose digest uv already recorded. The bytes the Flatpak downloads are
|
|
then the bytes in `uv.lock`, verified by hash rather than by convention. The
|
|
`onnxruntime` sources are selected as CPython-ABI wheels for each Flatpak
|
|
architecture because PyPI publishes no sdist for it.
|
|
|
|
It edits in place rather than regenerating, because two things in that file are
|
|
hand-written and no generator produces them: pywhispercpp's Vulkan build with
|
|
its version injection, and the symlinking of whisper.cpp's shared libraries onto
|
|
the loader path. Keeping the structure keeps the diff to versions and digests.
|
|
|
|
Usage:
|
|
python scripts/sync_flatpak_deps.py # rewrite the manifest
|
|
python scripts/sync_flatpak_deps.py --check # report drift, change nothing
|
|
|
|
`tests/test_flatpak_packaging.py` asserts the same invariant offline, so CI
|
|
catches drift without reaching PyPI.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from packaging.markers import Marker
|
|
from packaging.utils import canonicalize_name
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
EXPORT = REPO_ROOT / "requirements" / "vad.txt"
|
|
DEPS_YAML = REPO_ROOT / "packaging" / "flatpak" / "python3-dependencies.yaml"
|
|
MANIFEST = REPO_ROOT / "packaging" / "flatpak" / "com.voculinux.Voculinux.yml"
|
|
|
|
#: The environment the Flatpak build resolves for. GNOME 50 ships Python 3.13.
|
|
#: numpy's lock entry splits at `python_full_version >= '3.12'`; onnxruntime's
|
|
#: CPython wheel tag also uses this Python version. Bump it when the manifest's
|
|
#: runtime-version moves.
|
|
TARGET_ENV = {
|
|
"sys_platform": "linux",
|
|
"platform_system": "Linux",
|
|
"platform_machine": "x86_64",
|
|
"os_name": "posix",
|
|
"python_version": "3.13",
|
|
"python_full_version": "3.13.0",
|
|
"implementation_name": "cpython",
|
|
"platform_python_implementation": "CPython",
|
|
"extra": "",
|
|
}
|
|
|
|
#: org.gnome.Platform//50 ships these; a second copy under /app would shadow the
|
|
#: runtime's own GI stack. PyGObject is already absent from the export, which
|
|
#: `just lock` builds with --no-emit-package pygobject for the same reason.
|
|
RUNTIME_PROVIDED = {"pycairo"}
|
|
|
|
#: onnxruntime has no sdist on PyPI, and building it from source is out of reach
|
|
#: in the sandbox. These come as CPython-ABI wheels, one per Flatpak arch, tagged
|
|
#: for TARGET_ENV's python_version; a runtime Python bump makes pip --no-index
|
|
#: fail loudly at build time rather than silently.
|
|
BINARY_ONLY = {"onnxruntime"}
|
|
FLATPAK_ARCHES = ("x86_64", "aarch64")
|
|
|
|
#: Build backends for the sdists the manifest builds offline under
|
|
#: --no-build-isolation. They are not runtime dependencies, so they never appear
|
|
#: in requirements/vad.txt and stay pinned here by hand.
|
|
BUILD_BACKENDS = {"meson-python", "pyproject-metadata"}
|
|
|
|
PYPI_JSON = "https://pypi.org/pypi/{name}/{version}/json"
|
|
|
|
|
|
def parse_export(path: Path) -> dict[str, tuple[str, set[str]]]:
|
|
"""Map every package the Flatpak target selects to its version and digests.
|
|
|
|
uv writes one `name==version [; marker]` line followed by indented
|
|
`--hash=sha256:` lines, continued with backslashes. A package can appear
|
|
more than once under mutually exclusive markers, which is how numpy is
|
|
split across Python versions, so the marker decides rather than the order.
|
|
"""
|
|
selected: dict[str, tuple[str, set[str]]] = {}
|
|
current: str | None = None
|
|
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
line = raw.strip().rstrip("\\").strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
head = re.match(r"^([A-Za-z0-9._-]+)==([^ ;]+)\s*(?:;\s*(.*))?$", line)
|
|
if head:
|
|
name, version, marker = head.group(1), head.group(2), head.group(3)
|
|
current = None
|
|
if marker and not Marker(marker).evaluate(TARGET_ENV):
|
|
continue
|
|
key = canonicalize_name(name)
|
|
if key in selected:
|
|
raise SystemExit(
|
|
f"{path.name}: {key} resolves twice for the Flatpak target "
|
|
f"({selected[key][0]} and {version}); the markers overlap"
|
|
)
|
|
selected[key] = (version, set())
|
|
current = key
|
|
continue
|
|
digest = re.match(r"^--hash=sha256:([0-9a-f]{64})$", line)
|
|
if digest and current:
|
|
selected[current][1].add(digest.group(1))
|
|
return selected
|
|
|
|
|
|
def pypi_artifacts(name: str, version: str) -> list[dict]:
|
|
url = PYPI_JSON.format(name=name, version=version)
|
|
# nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected -- fixed https://pypi.org endpoint; name and version come from uv.lock
|
|
with urllib.request.urlopen(url, timeout=30) as response: # noqa: S310
|
|
return json.load(response)["urls"]
|
|
|
|
|
|
def choose_artifact(
|
|
name: str, version: str, digests: set[str], arch: str | None = None
|
|
) -> tuple[str, str]:
|
|
"""The one file the Flatpak should download, as (url, sha256).
|
|
|
|
Binary-only packages use an architecture-specific CPython wheel. For every
|
|
other package, prefer a universal wheel and fall back to the sdist so the
|
|
sandbox builds compiled dependencies against the SDK's interpreter.
|
|
"""
|
|
artifacts = pypi_artifacts(name, version)
|
|
if name in BINARY_ONLY:
|
|
if arch not in FLATPAK_ARCHES:
|
|
raise SystemExit(
|
|
f"{name} {version}: expected one of the Flatpak architectures "
|
|
f"{', '.join(FLATPAK_ARCHES)}, got {arch!r}"
|
|
)
|
|
python_tag = TARGET_ENV["python_version"].replace(".", "")
|
|
wheel_pattern = re.compile(
|
|
rf"-cp{python_tag}-cp{python_tag}-manylinux_.+_{re.escape(arch)}\.whl$"
|
|
)
|
|
candidates = [
|
|
artifact
|
|
for artifact in artifacts
|
|
if artifact["packagetype"] == "bdist_wheel"
|
|
and wheel_pattern.search(artifact["filename"])
|
|
]
|
|
if not candidates:
|
|
raise SystemExit(
|
|
f"{name} {version} publishes no CPython cp{python_tag} manylinux "
|
|
f"wheel for {arch}; update the Flatpak Python runtime or package"
|
|
)
|
|
else:
|
|
wheels = [
|
|
artifact
|
|
for artifact in artifacts
|
|
if artifact["packagetype"] == "bdist_wheel"
|
|
and artifact["filename"].endswith("-none-any.whl")
|
|
]
|
|
sdists = [artifact for artifact in artifacts if artifact["packagetype"] == "sdist"]
|
|
candidates = wheels or sdists
|
|
if not candidates:
|
|
raise SystemExit(
|
|
f"{name} {version} publishes neither a universal wheel nor an sdist; "
|
|
"the Flatpak cannot build it without an ABI-tagged wheel"
|
|
)
|
|
# A release can publish more than one matching wheel; take the first whose
|
|
# digest uv actually locked.
|
|
picked = next((a for a in candidates if a["digests"]["sha256"] in digests), None)
|
|
if picked is None:
|
|
first = candidates[0]
|
|
raise SystemExit(
|
|
f"{name} {version}: PyPI serves {first['filename']} with digest "
|
|
f"{first['digests']['sha256']}, which is not among the hashes uv "
|
|
"locked. Re-run `just lock`."
|
|
)
|
|
return picked["url"], picked["digests"]["sha256"]
|
|
|
|
|
|
def artifact_name(filename: str) -> str:
|
|
if filename.endswith(".whl"):
|
|
return canonicalize_name(filename.split("-")[0])
|
|
base = re.sub(r"\.(tar\.gz|zip|tar\.bz2)$", "", filename)
|
|
return canonicalize_name(base.rpartition("-")[0])
|
|
|
|
|
|
def manifest_sources(text: str) -> list[tuple[int, str, str]]:
|
|
"""Every PyPI source in the manifest, as (line index, filename, sha256)."""
|
|
lines = text.splitlines()
|
|
found = []
|
|
for index, line in enumerate(lines):
|
|
url = re.match(r"^\s*url: (https://files\.pythonhosted\.org/\S+/([^/\s]+))$", line)
|
|
if not url:
|
|
continue
|
|
digest = re.match(r"^\s*sha256: ([0-9a-f]{64})\s*$", lines[index + 1])
|
|
if not digest:
|
|
raise SystemExit(
|
|
f"{DEPS_YAML.name}:{index + 2}: a PyPI source with no sha256 beneath it"
|
|
)
|
|
found.append((index, url.group(2), digest.group(1)))
|
|
return found
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--check",
|
|
action="store_true",
|
|
help="report what would change and exit non-zero, without writing",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
locked = parse_export(EXPORT)
|
|
text = DEPS_YAML.read_text(encoding="utf-8")
|
|
lines = text.splitlines(keepends=True)
|
|
|
|
sources = manifest_sources(text)
|
|
in_manifest = {artifact_name(name) for _, name, _ in sources}
|
|
stale = sorted(in_manifest - set(locked) - BUILD_BACKENDS)
|
|
missing = sorted(set(locked) - in_manifest - RUNTIME_PROVIDED)
|
|
binary_arches = {name: set() for name in BINARY_ONLY}
|
|
for _, filename, _ in sources:
|
|
name = artifact_name(filename)
|
|
if name in BINARY_ONLY:
|
|
arch_match = re.search(r"_(x86_64|aarch64)\.whl$", filename)
|
|
if arch_match:
|
|
binary_arches[name].add(arch_match.group(1))
|
|
for name in BINARY_ONLY & set(locked):
|
|
missing.extend(
|
|
f"{name} ({arch})" for arch in FLATPAK_ARCHES if arch not in binary_arches[name]
|
|
)
|
|
missing.sort()
|
|
if stale:
|
|
print(
|
|
f"{DEPS_YAML.name} builds packages the lock does not resolve: "
|
|
f"{', '.join(stale)}\n"
|
|
"Delete their modules and sources by hand: whether a package is a "
|
|
"build backend the sandbox needs or a leftover this script cannot "
|
|
"tell.",
|
|
file=sys.stderr,
|
|
)
|
|
if missing:
|
|
print(
|
|
f"{DEPS_YAML.name} is missing packages the lock resolves: "
|
|
f"{', '.join(missing)}\n"
|
|
"The Flatpak installs with --no-deps, so a missing one fails at "
|
|
"runtime rather than at build time. Add a module for each.",
|
|
file=sys.stderr,
|
|
)
|
|
if stale or missing:
|
|
return 1
|
|
|
|
changes: list[str] = []
|
|
for index, filename, old_sha in sources:
|
|
name = artifact_name(filename)
|
|
if name in BUILD_BACKENDS:
|
|
# Absent from the lock by definition: nothing imports them at
|
|
# runtime, they only let the sandbox build the sdists offline.
|
|
continue
|
|
version, digests = locked[name]
|
|
arch = None
|
|
if name in BINARY_ONLY:
|
|
arch_match = re.search(r"_(x86_64|aarch64)\.whl$", filename)
|
|
if not arch_match:
|
|
raise SystemExit(f"{name}: cannot determine a Flatpak architecture from {filename}")
|
|
arch = arch_match.group(1)
|
|
url, sha = choose_artifact(name, version, digests, arch)
|
|
if sha == old_sha:
|
|
continue
|
|
indent = re.match(r"^(\s*)", lines[index]).group(1)
|
|
lines[index] = f"{indent}url: {url}\n"
|
|
lines[index + 1] = f"{indent}sha256: {sha}\n"
|
|
changes.append(f" {name}: {filename} -> {url.rsplit('/', 1)[-1]}")
|
|
|
|
updated = "".join(lines)
|
|
|
|
# pywhispercpp's version is spelled out in its build commands: the sdist
|
|
# carries no version.txt and setup() takes no version= argument, so the
|
|
# module rewrites setup.py before building. setuptools-scm would otherwise
|
|
# read a git tree that does not exist in the sandbox.
|
|
pywhispercpp_version = locked["pywhispercpp"][0]
|
|
updated, injected = re.subn(
|
|
# The version appears as a tarball name, a directory name and a
|
|
# literal inside the setup.py rewrite. Refuse to match rather than
|
|
# match short: a partial match would leave the tail of the old version
|
|
# glued to the new one and rename the directory tar just wrote.
|
|
r"pywhispercpp-\d+(?:\.\d+)*(?![\d.]\d)",
|
|
f"pywhispercpp-{pywhispercpp_version}",
|
|
updated,
|
|
)
|
|
updated = re.sub(
|
|
r"PYWHISPERCPP_VERSION=\d+(?:\.\d+)*",
|
|
f"PYWHISPERCPP_VERSION={pywhispercpp_version}",
|
|
updated,
|
|
)
|
|
updated = re.sub(
|
|
r'(version=\\")\d+(?:\.\d+)*(\\")',
|
|
rf"\g<1>{pywhispercpp_version}\g<2>",
|
|
updated,
|
|
)
|
|
if injected and updated != text and not changes:
|
|
changes.append(f" pywhispercpp: build commands -> {pywhispercpp_version}")
|
|
|
|
manifest_text = MANIFEST.read_text(encoding="utf-8")
|
|
manifest_updated = re.sub(
|
|
r'(PYWHISPERCPP_VERSION: ")\d+(?:\.\d+)*(")',
|
|
rf"\g<1>{pywhispercpp_version}\g<2>",
|
|
manifest_text,
|
|
)
|
|
|
|
if updated == text and manifest_updated == manifest_text:
|
|
print(
|
|
"Flatpak dependencies already match requirements/vad.txt "
|
|
"(runtime dependencies plus the [vad] extra)"
|
|
)
|
|
return 0
|
|
|
|
if args.check:
|
|
print(
|
|
"Flatpak dependencies are behind requirements/vad.txt "
|
|
"(runtime dependencies plus the [vad] extra):",
|
|
file=sys.stderr,
|
|
)
|
|
for change in changes:
|
|
print(change, file=sys.stderr)
|
|
if manifest_updated != manifest_text:
|
|
print(
|
|
f" {MANIFEST.name}: PYWHISPERCPP_VERSION -> {pywhispercpp_version}",
|
|
file=sys.stderr,
|
|
)
|
|
print("Run `just flatpak-deps` to update.", file=sys.stderr)
|
|
return 1
|
|
|
|
DEPS_YAML.write_text(updated, encoding="utf-8")
|
|
MANIFEST.write_text(manifest_updated, encoding="utf-8")
|
|
print(f"Updated {DEPS_YAML.relative_to(REPO_ROOT)}:")
|
|
for change in changes:
|
|
print(change)
|
|
if manifest_updated != manifest_text:
|
|
print(
|
|
f"Updated {MANIFEST.relative_to(REPO_ROOT)}: "
|
|
f"PYWHISPERCPP_VERSION={pywhispercpp_version}"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|