xavierk/voculinux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix(whispercpp): map GPU picker to ggml devices and size threads by effective backend (#1001)

Browse Source 
* fix(whispercpp): map GPU picker to ggml devices and size threads by effective backend

The GPU picker and auto selection used vulkaninfo indices, but whisper.cpp's
gpu_device is an ordinal over the GPU/IGPU devices ggml registered. ggml skips
Vulkan devices it cannot use (for example a Haswell iGPU), so the indices shift
and a selected NVIDIA card could fall back to CPU while logs claimed Vulkan.

Read ggml's device registry through the loaded pywhispercpp extension, select
and label devices by that ordinal, run on CPU explicitly when ggml has no GPU,
log the effective backend after load, and pick n_threads from the effective
backend instead of the presence of GPU libraries.

Fixes #987
Fixes #990

* test(whispercpp): move new ggml probe tests into their test class
This commit is contained in:
Jatin K Malik
2026-10-21 00:07:01 -07:00
committed by GitHub
parent 5fa0db6a47
commit 1c299d2dc7
7 changed files with 693 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vocalinux/speech_recognition/recognition_manager.py
+145 -48
View File
@@ -137,6 +137,26 @@ def _pywhispercpp_distribution_version() -> Optional[tuple[int, ...]]:
return tuple(parts) if parts else None
def _select_ggml_gpu_device(
devices: list[dict[str, Any]], requested_index: Optional[int]
) -> Optional[dict[str, Any]]:
if requested_index is not None and requested_index >= 0:
selected = next((device for device in devices if device["index"] == requested_index), None)
if selected is not None:
return selected
logger.warning(
"Requested whisper.cpp GPU index %s is unavailable; usable ggml GPU devices: %s. "
"Selecting automatically.",
requested_index,
devices,
)
return next(
(device for device in devices if device["device_type"] == "discrete"),
devices[0] if devices else None,
)
def resolve_whisper_language(language: str) -> Optional[str]:
"""Map a catalog language id to a Whisper / whisper.cpp language code.
@@ -1515,12 +1535,21 @@ class SpeechRecognitionManager:
return compatible_kwargs
def _load_model_with_compatible_params(
self, model_path: str, model_kwargs: dict, gpu_device: Optional[int] = None
):
self,
model_path: str,
model_kwargs: dict[str, Any],
gpu_device: Optional[int] = None,
use_gpu: Optional[bool] = None,
) -> Any:
from pywhispercpp.model import Model
compatible_kwargs = self._filter_whispercpp_model_kwargs(model_kwargs)
context_params: dict[str, Any] = {}
if use_gpu is not None:
context_params["use_gpu"] = use_gpu
if gpu_device is not None and gpu_device >= 0:
context_params["gpu_device"] = gpu_device
if context_params:
dist_version = _pywhispercpp_distribution_version()
too_old_for_context_params = dist_version is not None and dist_version < (1, 5, 0)
try:
@@ -1540,7 +1569,7 @@ class SpeechRecognitionManager:
version_label,
)
elif supports_context_params:
compatible_kwargs["context_params"] = {"gpu_device": gpu_device}
compatible_kwargs["context_params"] = context_params
else:
logger.warning(
"pywhispercpp does not support context_params; "
@@ -1582,62 +1611,118 @@ class SpeechRecognitionManager:
_prefer_discrete_vulkan_device,
detect_compute_backend,
detect_cpu_info,
detect_ggml_gpu_devices,
detect_vulkan_devices,
get_backend_display_name,
)
host_backend, host_info = detect_compute_backend()
actual_gpu_backend = self._detect_pywhispercpp_gpu_backend()
ggml_devices = detect_ggml_gpu_devices()
logger.info("whisper.cpp backend selection priority: Vulkan -> CUDA -> CPU")
selected_gpu_device = None
use_gpu: Optional[bool] = None
runtime_backend = ComputeBackend.CPU
runtime_info = detect_cpu_info()
if actual_gpu_backend == "cuda":
# pywhispercpp CUDA uses CUDA device ordinals (0 = first NVIDIA GPU).
# Vulkan enumeration on hybrid laptops lists iGPU as GPU0 and dGPU as
# GPU1; passing that Vulkan index to CUDA selects the CPU fallback.
vulkan_gpu_index = self.whispercpp_gpu_device
if vulkan_gpu_index is None or vulkan_gpu_index < 0:
vulkan_gpu_index = _prefer_discrete_vulkan_device()
if vulkan_gpu_index not in (None, 0):
logger.info(
"pywhispercpp is CUDA-backed; using CUDA device 0 "
f"(Vulkan GPU index {vulkan_gpu_index} does not map to CUDA ordinals)"
if ggml_devices == []:
use_gpu = False
skipped_devices = [
device["name"]
for device in detect_vulkan_devices()
if device.get("device_type") != "software"
]
logger.warning(
"whisper.cpp found no usable ggml GPU device and will use CPU. "
"Vulkan hardware devices skipped by ggml: %s",
", ".join(skipped_devices) if skipped_devices else "none found",
)
else:
logger.info("pywhispercpp is CUDA-backed; using CUDA device 0")
selected_gpu_device = 0
runtime_backend = ComputeBackend.CUDA
runtime_info = host_info if host_backend == ComputeBackend.CUDA else "NVIDIA GPU"
# CUDA device ordinals are independent of vulkaninfo indices.
vulkan_gpu_index = self.whispercpp_gpu_device
if vulkan_gpu_index is None or vulkan_gpu_index < 0:
vulkan_gpu_index = _prefer_discrete_vulkan_device()
if vulkan_gpu_index not in (None, 0):
logger.info(
"pywhispercpp is CUDA-backed; using CUDA device 0 "
"(Vulkan GPU index %s does not map to CUDA ordinals)",
vulkan_gpu_index,
)
else:
logger.info("pywhispercpp is CUDA-backed; using CUDA device 0")
selected_gpu_device = 0
runtime_backend = ComputeBackend.CUDA
runtime_info = host_info if host_backend == ComputeBackend.CUDA else "NVIDIA GPU"
elif actual_gpu_backend == "vulkan":
gpu_device_index = self.whispercpp_gpu_device
if gpu_device_index is None or gpu_device_index < 0:
gpu_device_index = _prefer_discrete_vulkan_device()
devices = detect_vulkan_devices()
runtime_backend = ComputeBackend.VULKAN
if gpu_device_index is not None:
device_name = next(
(d["name"] for d in devices if d["index"] == gpu_device_index),
"unknown",
)
logger.info(f"Using Vulkan GPU [{gpu_device_index}]: {device_name}")
selected_gpu_device = gpu_device_index
runtime_info = device_name
elif devices:
runtime_info = devices[0]["name"]
logger.warning(
"pywhispercpp is Vulkan-backed but no preferred GPU was selected. "
"whisper.cpp will default to GPU 0, which may be an iGPU on hybrid laptops."
)
if ggml_devices is not None:
vulkan_devices = detect_vulkan_devices()
if ggml_devices:
selected_device = _select_ggml_gpu_device(
ggml_devices, self.whispercpp_gpu_device
)
ggml_names = {device["name"] for device in ggml_devices}
skipped_devices = [
device["name"]
for device in vulkan_devices
if device.get("device_type") != "software"
and device["name"] not in ggml_names
]
if skipped_devices:
logger.info(
"Vulkan devices not registered by ggml: %s",
", ".join(skipped_devices),
)
if selected_device is not None:
runtime_backend = ComputeBackend.VULKAN
selected_gpu_device = selected_device["index"]
runtime_info = selected_device["name"]
logger.info(
"Requesting Vulkan GPU [%s]: %s (%s)",
selected_device["index"],
selected_device["name"],
selected_device["backend_name"],
)
else:
use_gpu = False
skipped_devices = [
device["name"]
for device in vulkan_devices
if device.get("device_type") != "software"
]
logger.warning(
"whisper.cpp found no usable Vulkan device and will use CPU. "
"Vulkan hardware devices skipped by ggml: %s",
", ".join(skipped_devices) if skipped_devices else "none found",
)
else:
logger.warning(
"pywhispercpp is Vulkan-backed but vulkaninfo found no devices. "
"Install vulkan-tools so Vocalinux can pick a discrete GPU. "
"whisper.cpp will default to GPU 0, which may be an iGPU on hybrid laptops."
)
gpu_device_index = self.whispercpp_gpu_device
if gpu_device_index is None or gpu_device_index < 0:
gpu_device_index = _prefer_discrete_vulkan_device()
devices = detect_vulkan_devices()
runtime_backend = ComputeBackend.VULKAN
if gpu_device_index is not None:
device_name = next(
(d["name"] for d in devices if d["index"] == gpu_device_index),
"unknown",
)
logger.info(f"Using Vulkan GPU [{gpu_device_index}]: {device_name}")
selected_gpu_device = gpu_device_index
runtime_info = device_name
elif devices:
runtime_info = devices[0]["name"]
logger.warning(
"pywhispercpp is Vulkan-backed but no preferred GPU was selected. "
"whisper.cpp will default to GPU 0, which may be an iGPU on hybrid laptops."
)
else:
logger.warning(
"pywhispercpp is Vulkan-backed but vulkaninfo found no devices. "
"Install vulkan-tools so Vocalinux can pick a discrete GPU. "
"whisper.cpp will default to GPU 0, which may be an iGPU on hybrid laptops."
)
elif host_backend != ComputeBackend.CPU:
logger.warning(
"Host reports %s (%s), but pywhispercpp has no GPU libraries. "
@@ -1649,7 +1734,11 @@ class SpeechRecognitionManager:
host_info,
)
if actual_gpu_backend in ("vulkan", "cuda") and host_backend != actual_gpu_backend:
if (
runtime_backend == actual_gpu_backend
and actual_gpu_backend in ("vulkan", "cuda")
and host_backend != actual_gpu_backend
):
logger.info(
"Host capability label is %s; bundled pywhispercpp libraries are %s. "
"Using the bundled %s backend.",
@@ -1659,7 +1748,9 @@ class SpeechRecognitionManager:
)
logger.info(
f"whisper.cpp using {get_backend_display_name(runtime_backend)} backend: {runtime_info}"
"whisper.cpp requesting %s backend: %s",
get_backend_display_name(runtime_backend),
runtime_info,
)
import psutil
@@ -1678,9 +1769,8 @@ class SpeechRecognitionManager:
logger.info(f"Loading whisper.cpp '{self.model_size}' model...")
self.model = None # Release previous model if re-initializing
has_gpu_libs = actual_gpu_backend in ("vulkan", "cuda")
if has_gpu_libs and not (
has_gpu_backend = runtime_backend in (ComputeBackend.VULKAN, ComputeBackend.CUDA)
if has_gpu_backend and not (
self.whispercpp_n_threads is not None and self.whispercpp_n_threads > 0
):
n_threads = max(1, multiprocessing.cpu_count() // 4)
@@ -1695,18 +1785,25 @@ class SpeechRecognitionManager:
try:
self.model = self._load_model_with_compatible_params(
model_path, model_kwargs, gpu_device=selected_gpu_device
model_path, model_kwargs, gpu_device=selected_gpu_device, use_gpu=use_gpu
)
except RuntimeError as model_error:
loaded_backend = self._handle_gpu_fallback(
model_error, model_path, model_kwargs, ComputeBackend.CPU
)
runtime_info = detect_cpu_info()
effective_threads = self._whispercpp_effective_threads
if effective_threads is None:
effective_threads = n_threads
load_duration = time.time() - load_start_time
logger.info(
"whisper.cpp loaded with effective %s backend: %s (n_threads=%d)",
get_backend_display_name(loaded_backend),
runtime_info,
effective_threads,
)
logger.info(
f"whisper.cpp configured with n_threads={effective_threads} "
f"({loaded_backend} backend)"
src/vocalinux/ui/settings_dialog.py
+20 -15
View File
@@ -2422,6 +2422,24 @@ class ModelDownloadDialog(Gtk.Dialog):
self.add_button("OK", Gtk.ResponseType.OK)
def _populate_gpu_device_choices(combo: Any, saved_device: Optional[int]) -> None:
combo.remove_all()
combo.append("-1", "Auto (discrete GPU)")
from ..utils.whispercpp_model_info import detect_ggml_gpu_devices
devices = detect_ggml_gpu_devices()
if devices is None:
devices = detect_vulkan_devices()
for device in devices:
type_label = device["device_type"].capitalize()
label = f"[{device['index']}] {device['name']} ({type_label})"
combo.append(str(device["index"]), label)
if saved_device is None or not combo.set_active_id(str(saved_device)):
combo.set_active_id("-1")
class SettingsDialog(AboutSectionMixin):
"""Modern GTK Dialog for configuring Vocalinux settings."""
@@ -9592,23 +9610,10 @@ For now, the engine has been reverted to VOSK."""
error_dialog.destroy()
return False
def _populate_gpu_devices(self):
def _populate_gpu_devices(self) -> None:
"""Populate the GPU device dropdown with available Vulkan devices."""
self.gpu_device_combo.remove_all()
self.gpu_device_combo.append("-1", "Auto (discrete GPU)")
devices = detect_vulkan_devices()
for device in devices:
type_label = device["device_type"].capitalize()
label = f"[{device['index']}] {device['name']} ({type_label})"
self.gpu_device_combo.append(str(device["index"]), label)
saved_device = self.config_manager.get("advanced", "whispercpp_gpu_device", None)
if saved_device is None:
self.gpu_device_combo.set_active_id("-1")
else:
if not self.gpu_device_combo.set_active_id(str(saved_device)):
self.gpu_device_combo.set_active_id("-1")
_populate_gpu_device_choices(self.gpu_device_combo, saved_device)
def _populate_audio_devices(self):
"""Populate the audio device dropdown with available input devices."""
src/vocalinux/utils/whispercpp_model_info.py
+59 -2
View File
@@ -5,13 +5,15 @@ This module provides model metadata and hardware acceleration detection
for whisper.cpp, supporting Vulkan, CUDA, and CPU backends.
"""
import ctypes
import logging
import math
import os
import re
import subprocess
from functools import lru_cache
from typing import Optional, cast
from functools import cache, lru_cache
from importlib.metadata import PackageNotFoundError, version
from typing import Any, Optional, cast
from .host_process import host_env
from .model_checksums import whispercpp_revision
@@ -347,6 +349,61 @@ def detect_vulkan_devices() -> list[dict]:
return devices
@cache
def detect_ggml_gpu_devices() -> Optional[list[dict[str, Any]]]:
"""Return whisper.cpp's usable GPU devices in its device-selection order."""
try:
raw_version = version("pywhispercpp")
except (ImportError, PackageNotFoundError):
return None
version_match = re.match(r"^(\d+)\.(\d+)(?:\.(\d+))?", raw_version)
if version_match is None:
return None
installed_version = tuple(int(part or 0) for part in version_match.groups())
if installed_version < (1, 5, 0):
return None
try:
from . import pywhispercpp_loader
pywhispercpp_loader.preload_shared_libraries()
import _pywhispercpp
library = ctypes.CDLL(_pywhispercpp.__file__)
library.ggml_backend_dev_count.argtypes = []
library.ggml_backend_dev_count.restype = ctypes.c_size_t
library.ggml_backend_dev_get.argtypes = [ctypes.c_size_t]
library.ggml_backend_dev_get.restype = ctypes.c_void_p
library.ggml_backend_dev_name.argtypes = [ctypes.c_void_p]
library.ggml_backend_dev_name.restype = ctypes.c_char_p
library.ggml_backend_dev_description.argtypes = [ctypes.c_void_p]
library.ggml_backend_dev_description.restype = ctypes.c_char_p
library.ggml_backend_dev_type.argtypes = [ctypes.c_void_p]
library.ggml_backend_dev_type.restype = ctypes.c_int
devices: list[dict[str, Any]] = []
for registry_index in range(library.ggml_backend_dev_count()):
device = library.ggml_backend_dev_get(registry_index)
device_type = library.ggml_backend_dev_type(device)
if device_type not in (1, 2):
continue
# whisper.cpp counts every GPU/IGPU device, so never skip one here.
backend_name = library.ggml_backend_dev_name(device) or b"unknown"
description = library.ggml_backend_dev_description(device) or backend_name
devices.append(
{
"index": len(devices),
"name": description.decode("utf-8", errors="replace"),
"backend_name": backend_name.decode("utf-8", errors="replace"),
"device_type": "discrete" if device_type == 1 else "integrated",
}
)
return devices
except (ImportError, OSError, AttributeError):
return None
def _prefer_discrete_vulkan_device() -> Optional[int]:
"""Return the index of the preferred hardware Vulkan GPU (discrete if available)."""
devices = _hardware_vulkan_devices()