fix(model): detect CUDA toolkit outside PATH and rank by installed build

cuda_toolkit_ready() only accepted nvcc on PATH, so a session started
before the CUDA profile snippet ranked Vulkan above a CUDA-compiled build.
A complete toolkit is now also recognised under CUDA_HOME, CUDA_PATH or
/usr/local/cuda*, requiring nvcc, include/cuda_runtime.h and libcudart in
the architecture-matching target. The backend order only names backends
the installed build compiled in, so a Vulkan-only build never lists CUDA.

Refs #20

src/vocalinux/speech_recognition/model_ladder.py

@@ -19,14 +19,20 @@ that change is deliberate.
 
 Backend order follows the same readiness rule: CUDA ranks before Vulkan
 only when a complete architecture-matching CUDA toolkit is installed
-(``nvidia-smi`` sees a GPU, ``nvcc`` answers, compute capability >= 7.0);
-otherwise the order is Vulkan, then CPU. A driver without the toolkit is
+(``nvidia-smi`` sees a GPU, compute capability >= 7.0, and nvcc answers on
+PATH or a complete toolkit root under ``CUDA_HOME``, ``CUDA_PATH`` or
+``/usr/local/cuda*``); otherwise the order is Vulkan, then CPU. The order only
+names backends the installed build compiled in. A driver without the toolkit is
 exactly what made the old VRAM-aware recommendation unreachable.
 """
 
 import logging
+import os
+import platform
 import shutil
 import subprocess
+from collections.abc import Mapping
+from pathlib import Path
 
 logger = logging.getLogger(__name__)
 
@@ -59,6 +65,14 @@ MIN_CUDA_COMPUTE_CAPABILITY: tuple[int, int] = (7, 0)
 #: discrete card. A CPU build keeps its model in system RAM.
 GPU_BACKENDS: tuple[str, ...] = ("vulkan", "cuda")
 
+#: Environment variables that may name a CUDA toolkit root, in probe order.
+_CUDA_ROOT_ENV_VARS: tuple[str, ...] = ("CUDA_HOME", "CUDA_PATH")
+#: Directory whose ``cuda*`` entries are probed as toolkit roots. Tests redirect it.
+_CUDA_USR_LOCAL_ROOT: Path = Path("/usr/local")
+#: Toolkit ``targets/`` directory for each host machine. Voculinux publishes
+#: only glibc x86_64, so any other machine has no matching toolkit target.
+_CUDA_TARGET_FOR_MACHINE: dict[str, str] = {"x86_64": "x86_64-linux"}
+
 
 def _floor_rung(language_is_english: bool) -> str:
     """Return the floor rung for the language: English-only pairs with ``.en``."""
@@ -168,6 +182,67 @@ def cuda_toolkit_ready_from_probe(
     return compute_cap >= MIN_CUDA_COMPUTE_CAPABILITY
 
 
+def _toolkit_is_complete(root: Path, machine: str) -> bool:
+    """Return whether ``root`` holds an nvcc, the runtime header and libcudart.
+
+    ``machine`` must have a toolkit target: the libraries are looked up in that
+    target's ``lib`` directory (and ``lib64`` on x86_64, where it is a link to
+    the same place in the runfile layout).
+    """
+    target = _CUDA_TARGET_FOR_MACHINE.get(machine)
+    if target is None:
+        return False
+    nvcc = root / "bin" / "nvcc"
+    if not (nvcc.is_file() and os.access(nvcc, os.X_OK)):
+        return False
+    if not (root / "include" / "cuda_runtime.h").is_file():
+        return False
+    lib_dirs = [root / "targets" / target / "lib"]
+    if machine == "x86_64":
+        lib_dirs.append(root / "lib64")
+    return any(lib.is_dir() and any(lib.glob("libcudart.so*")) for lib in lib_dirs)
+
+
+def _cuda_toolkit_roots(environ: Mapping[str, str]) -> list[Path]:
+    """Return candidate toolkit roots: CUDA_HOME, CUDA_PATH, then /usr/local/cuda*."""
+    roots: list[Path] = []
+    for name in _CUDA_ROOT_ENV_VARS:
+        value = environ.get(name, "").strip()
+        if value:
+            roots.append(Path(value))
+    if _CUDA_USR_LOCAL_ROOT.is_dir():
+        roots.extend(
+            sorted(
+                (path for path in _CUDA_USR_LOCAL_ROOT.glob("cuda*") if path.is_dir()),
+                reverse=True,
+            )
+        )
+    return roots
+
+
+def _toolkit_root_is_complete(environ: Mapping[str, str], machine: str) -> bool:
+    """Return whether any candidate root is a complete toolkit for ``machine``."""
+    checked: set[Path] = set()
+    for root in _cuda_toolkit_roots(environ):
+        try:
+            resolved = root.resolve()
+        except OSError:
+            continue
+        if resolved in checked:
+            continue
+        checked.add(resolved)
+        if _toolkit_is_complete(resolved, machine):
+            return True
+    return False
+
+
+def _cuda_toolkit_installed() -> bool:
+    """Return whether a CUDA toolkit is usable: nvcc on PATH, or a complete root."""
+    if _run_probe(["nvcc", "--version"]) is not None:
+        return True
+    return _toolkit_root_is_complete(os.environ, platform.machine())
+
+
 def cuda_toolkit_ready() -> bool:
     """Probe the host for a complete architecture-matching CUDA toolkit."""
     smi = _run_probe(
@@ -175,10 +250,9 @@ def cuda_toolkit_ready() -> bool:
     )
     if not smi:
         return False
-    nvcc = _run_probe(["nvcc", "--version"])
     return cuda_toolkit_ready_from_probe(
         has_nvidia_gpu=True,
-        nvcc_ok=nvcc is not None,
+        nvcc_ok=_cuda_toolkit_installed(),
         compute_cap=_parse_first_compute_cap(smi),
     )
 
@@ -232,15 +306,27 @@ def free_memory_mib() -> float | None:
     return free_system_ram_mib()
 
 
-def preferred_backend_order() -> tuple[str, ...]:
-    """Return the backend attempt order for this host.
+def preferred_backend_order(build_backend: str | None = None) -> tuple[str, ...]:
+    """Return the backend attempt order for this host and installed build.
 
-    CUDA ranks before Vulkan only when a complete architecture-matching
-    toolkit is installed; otherwise Vulkan, then CPU.
+    ``build_backend`` is the GPU backend the installed pywhispercpp build
+    compiled in (``vulkan``, ``cuda`` or ``cpu``). The runtime can only use
+    those, so the order names only them: CUDA first when a complete
+    architecture-matching toolkit is installed, then Vulkan, then CPU. A
+    Vulkan-only build never lists CUDA. ``None`` (build unknown) keeps the
+    unfiltered order: CUDA, Vulkan, CPU with a toolkit, otherwise Vulkan, CPU.
     """
-    if cuda_toolkit_ready():
-        return ("cuda", "vulkan", "cpu")
-    return ("vulkan", "cpu")
+    if build_backend is None:
+        if cuda_toolkit_ready():
+            return ("cuda", "vulkan", "cpu")
+        return ("vulkan", "cpu")
+    order: list[str] = []
+    if build_backend == "cuda" and cuda_toolkit_ready():
+        order.append("cuda")
+    if build_backend == "vulkan":
+        order.append("vulkan")
+    order.append("cpu")
+    return tuple(order)
 
 
 def describe_environment() -> str:

src/vocalinux/speech_recognition/recognition_manager.py

@@ -1801,7 +1801,7 @@ class SpeechRecognitionManager:
         ggml_devices = detect_ggml_gpu_devices()
         logger.info(
             "whisper.cpp backend selection priority: %s",
-            " -> ".join(preferred_backend_order()),
+            " -> ".join(preferred_backend_order(actual_gpu_backend)),
         )
 
         selected_gpu_device = None

tests/test_model_ladder.py

@@ -8,8 +8,12 @@ protocol headroom. Replacing them with a recorded measurement is a
 deliberate change to this test.
 """
 
+import os
+from pathlib import Path
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from vocalinux.speech_recognition import model_ladder
 from vocalinux.speech_recognition.model_ladder import (
     DEFAULT_RUNG,
@@ -130,6 +134,146 @@ class TestCudaToolkitReadiness:
         assert cuda_toolkit_ready_from_probe(True, True, None) is False
 
 
+NVCC_VERSION_OUTPUT = "Cuda compilation tools, release 13.2, V13.2.51\n"
+NVIDIA_SMI_OUTPUT = "3711, 8.6\n"
+
+
+def _write_toolkit(
+    root: Path,
+    *,
+    nvcc: bool = True,
+    header: bool = True,
+    libcudart: bool = True,
+    target: str = "x86_64-linux",
+) -> Path:
+    """Lay out a CUDA toolkit tree shaped like the NVIDIA runfile install."""
+    if nvcc:
+        (root / "bin").mkdir(parents=True, exist_ok=True)
+        binary = root / "bin" / "nvcc"
+        binary.write_text("#!/bin/sh\n")
+        binary.chmod(0o755)
+    if header:
+        (root / "include").mkdir(parents=True, exist_ok=True)
+        (root / "include" / "cuda_runtime.h").write_text("/* header */\n")
+    if libcudart:
+        lib = root / "targets" / target / "lib"
+        lib.mkdir(parents=True, exist_ok=True)
+        (lib / "libcudart.so.13").write_text("")
+        if target == "x86_64-linux":
+            (root / "lib64").symlink_to(lib, target_is_directory=True)
+    return root
+
+
+def _probe_stub(nvcc_output: str | None):
+    """Stand in for ``_run_probe``: nvidia-smi sees an RTX-class GPU."""
+
+    def probe(argv, timeout=10):
+        if argv[0] == "nvidia-smi":
+            return NVIDIA_SMI_OUTPUT
+        if argv[0] == "nvcc":
+            return nvcc_output
+        return None
+
+    return probe
+
+
+@pytest.fixture
+def no_env_toolkit(tmp_path: Path):
+    """Isolate detection from the host: no CUDA env vars, empty /usr/local."""
+    usr_local = tmp_path / "usr-local"
+    usr_local.mkdir()
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        patch.object(model_ladder, "_CUDA_USR_LOCAL_ROOT", usr_local),
+        patch.object(model_ladder.platform, "machine", return_value="x86_64"),
+    ):
+        yield usr_local
+
+
+class TestCudaToolkitDetection:
+    """A toolkit counts from PATH, CUDA_HOME, CUDA_PATH or /usr/local/cuda*/."""
+
+    def test_toolkit_found_via_path(self, no_env_toolkit: Path) -> None:
+        with patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(NVCC_VERSION_OUTPUT)):
+            assert model_ladder.cuda_toolkit_ready() is True
+
+    @pytest.mark.parametrize("variable", ["CUDA_HOME", "CUDA_PATH"])
+    def test_toolkit_found_via_cuda_env_root(
+        self, tmp_path: Path, no_env_toolkit: Path, variable: str
+    ) -> None:
+        root = _write_toolkit(tmp_path / "opt" / "cuda")
+        with (
+            patch.dict(os.environ, {variable: str(root)}, clear=True),
+            patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)),
+        ):
+            assert model_ladder.cuda_toolkit_ready() is True
+
+    def test_toolkit_found_only_under_usr_local_cuda(self, no_env_toolkit: Path) -> None:
+        """The real target: /usr/local/cuda -> cuda-13.2 with nvcc off PATH."""
+        real = _write_toolkit(no_env_toolkit / "cuda-13.2")
+        (no_env_toolkit / "cuda").symlink_to(real, target_is_directory=True)
+        with patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)):
+            assert model_ladder.cuda_toolkit_ready() is True
+
+    def test_no_toolkit_anywhere_is_not_ready(self, no_env_toolkit: Path) -> None:
+        with patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)):
+            assert model_ladder.cuda_toolkit_ready() is False
+
+    @pytest.mark.parametrize(
+        "layout",
+        [
+            {"nvcc": False},
+            {"header": False},
+            {"libcudart": False},
+            {"target": "sbsa-linux"},
+        ],
+        ids=["no-nvcc", "no-cuda-runtime-header", "no-libcudart", "other-architecture"],
+    )
+    def test_incomplete_toolkit_is_rejected(
+        self, tmp_path: Path, no_env_toolkit: Path, layout: dict
+    ) -> None:
+        root = _write_toolkit(tmp_path / "partial", **layout)
+        with (
+            patch.dict(os.environ, {"CUDA_HOME": str(root)}, clear=True),
+            patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)),
+        ):
+            assert model_ladder.cuda_toolkit_ready() is False
+
+    def test_driver_without_toolkit_stays_not_ready_with_partial_tree(
+        self, no_env_toolkit: Path
+    ) -> None:
+        """A /usr/local/cuda with headers only is not a toolkit."""
+        _write_toolkit(no_env_toolkit / "cuda-13.2", nvcc=False, libcudart=False)
+        with patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)):
+            assert model_ladder.cuda_toolkit_ready() is False
+
+
+class TestBuildAwareBackendOrder:
+    """The order names only backends the installed build compiled in."""
+
+    def test_vulkan_only_build_never_picks_cuda_even_with_toolkit(
+        self, no_env_toolkit: Path
+    ) -> None:
+        _write_toolkit(no_env_toolkit / "cuda-13.2")
+        (no_env_toolkit / "cuda").symlink_to(no_env_toolkit / "cuda-13.2")
+        with patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)):
+            assert model_ladder.cuda_toolkit_ready() is True
+            assert preferred_backend_order("vulkan") == ("vulkan", "cpu")
+
+    def test_cuda_build_picks_cuda_with_nvcc_off_path(self, no_env_toolkit: Path) -> None:
+        _write_toolkit(no_env_toolkit / "cuda-13.2")
+        (no_env_toolkit / "cuda").symlink_to(no_env_toolkit / "cuda-13.2")
+        with patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)):
+            assert preferred_backend_order("cuda") == ("cuda", "cpu")
+
+    def test_cuda_build_without_toolkit_has_no_cuda_rung(self, no_env_toolkit: Path) -> None:
+        with patch.object(model_ladder, "_run_probe", side_effect=_probe_stub(None)):
+            assert preferred_backend_order("cuda") == ("cpu",)
+
+    def test_cpu_build_is_cpu_only(self) -> None:
+        assert preferred_backend_order("cpu") == ("cpu",)
+
+
 class TestPreferredBackendOrder:
     """CUDA ranks first only with a complete toolkit."""