SergeyandGitHub
f4e9fcb0ee
* installer: leave model downloads to first run when the release pins no checksums * installer: refuse an unpinned VOSK model before spending the download * installer: require a real device before reporting a Vulkan GPU * build: replace the deprecated license table and complete the packages list * installer: treat software Vulkan renderers as no GPU, not as one * installer: keep diagnostics off stdout so captured detection output stays clean
327 lines
12 KiB
Python
327 lines
12 KiB
Python
"""Behavioural tests for the installer's Vulkan detection.
|
|
|
|
`detect_vulkan` used to accept any non-empty `vulkaninfo --summary` as a GPU, and
|
|
read only the first 20 lines while `check_vulkan_gpu_compatibility` read all of
|
|
them -- so the wizard could report a GPU in Step 1 and none in Step 3.
|
|
"""
|
|
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import textwrap
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
INSTALL_SH = Path(__file__).resolve().parents[1] / "install.sh"
|
|
SOURCE = INSTALL_SH.read_text()
|
|
|
|
FUNCTIONS = (
|
|
"is_software_renderer",
|
|
"vulkan_device_names",
|
|
"detect_vulkan",
|
|
"check_vulkan_gpu_compatibility",
|
|
)
|
|
|
|
PRELUDE = """
|
|
set -uo pipefail
|
|
HAS_VULKAN="no"
|
|
VULKAN_DEVICE=""
|
|
VULKAN_SOFTWARE_DEVICE=""
|
|
print_info() { echo "INFO: $*"; }
|
|
print_warning() { echo "WARNING: $*"; }
|
|
"""
|
|
|
|
# The one list both detectors read; lifted from install.sh so it cannot drift.
|
|
PATTERNS = re.search(r"^VULKAN_SOFTWARE_PATTERNS=\([^)]*\)", SOURCE, re.M)
|
|
assert PATTERNS, "VULKAN_SOFTWARE_PATTERNS missing from install.sh"
|
|
|
|
# A loader that resolved no driver: header, error, no device.
|
|
NO_DEVICES = """\
|
|
ERROR: [Loader Message] Code 0 : setup_loader_term_phys_devs: Failed to detect any valid GPUs
|
|
==========
|
|
VULKANINFO
|
|
==========
|
|
|
|
Vulkan Instance Version: 1.3.239
|
|
|
|
Instance Extensions: count = 19
|
|
Layers: count = 0
|
|
|
|
Devices:
|
|
========
|
|
"""
|
|
|
|
# A real adapter listed past the old `head -20` cutoff.
|
|
DEVICE_PAST_LINE_20 = (
|
|
"".join(f"instance preamble line {n}\n" for n in range(1, 25))
|
|
+ "\tdeviceName = AMD Radeon RX 7900 XTX (RADV NAVI31)\n"
|
|
)
|
|
|
|
SOFTWARE_ONLY = """\
|
|
==========
|
|
VULKANINFO
|
|
==========
|
|
|
|
Devices:
|
|
========
|
|
GPU0:
|
|
\tdeviceName = llvmpipe (LLVM 15.0.6, 256 bits)
|
|
"""
|
|
|
|
# A real adapter the loader happens to list after the software one.
|
|
SOFTWARE_THEN_HARDWARE = (
|
|
SOFTWARE_ONLY + "\tdeviceName = AMD Radeon RX 7900 XTX (RADV NAVI31)\n"
|
|
)
|
|
|
|
DEVICE_EARLY = """\
|
|
==========
|
|
VULKANINFO
|
|
==========
|
|
|
|
Devices:
|
|
========
|
|
GPU0:
|
|
\tdeviceName = Intel(R) Iris(R) Xe Graphics (TGL GT2)
|
|
"""
|
|
|
|
|
|
def _functions() -> str:
|
|
chunks = [PATTERNS.group(0)]
|
|
for name in FUNCTIONS:
|
|
match = re.search(rf"^{name}\(\) \{{.*?^\}}", SOURCE, re.M | re.S)
|
|
assert match, f"{name} not found in install.sh"
|
|
chunks.append(match.group(0))
|
|
return "\n".join(chunks)
|
|
|
|
|
|
# PATH is restricted to these so a real vulkaninfo cannot answer for a stub.
|
|
NEEDED_TOOLS = ("awk", "head", "grep", "cut", "tr", "xargs", "sed", "cat")
|
|
|
|
|
|
def _run(tmp_path: Path, summary: str | None, script: str) -> subprocess.CompletedProcess:
|
|
"""Run `script` with a stubbed vulkaninfo, or with none at all."""
|
|
binary = tmp_path / "bin"
|
|
binary.mkdir(exist_ok=True)
|
|
for tool in NEEDED_TOOLS:
|
|
found = shutil.which(tool)
|
|
if found and not (binary / tool).exists():
|
|
(binary / tool).symlink_to(found)
|
|
if summary is not None:
|
|
stub = binary / "vulkaninfo"
|
|
stub.write_text(textwrap.dedent("""\
|
|
#!/bin/bash
|
|
case "$1" in
|
|
--summary) cat <<'SUMMARY'
|
|
{summary}
|
|
SUMMARY
|
|
;;
|
|
--features) echo "" ;;
|
|
esac
|
|
""").format(summary=summary.rstrip("\n")))
|
|
stub.chmod(0o755)
|
|
env_path = f'export PATH="{binary}"\n'
|
|
return subprocess.run(
|
|
["bash", "-c", PRELUDE + env_path + _functions() + "\n" + script],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
|
|
|
|
class TestDetectVulkan:
|
|
def test_a_loader_without_devices_is_not_a_gpu(self, tmp_path):
|
|
result = _run(
|
|
tmp_path, NO_DEVICES, 'detect_vulkan; echo "RC=$? HAS=$HAS_VULKAN DEV=[$VULKAN_DEVICE]"'
|
|
)
|
|
assert "RC=1 HAS=no DEV=[]" in result.stdout, result.stdout + result.stderr
|
|
|
|
def test_it_never_reports_the_fallback_string_as_a_device(self, tmp_path):
|
|
"""The old code printed this literal as though it were an adapter name."""
|
|
result = _run(tmp_path, NO_DEVICES, 'detect_vulkan; echo "DEV=[$VULKAN_DEVICE]"')
|
|
assert "Vulkan-compatible GPU" not in result.stdout
|
|
|
|
def test_a_device_listed_past_line_20_is_still_found(self, tmp_path):
|
|
result = _run(
|
|
tmp_path,
|
|
DEVICE_PAST_LINE_20,
|
|
'detect_vulkan; echo "HAS=$HAS_VULKAN DEV=[$VULKAN_DEVICE]"',
|
|
)
|
|
assert "HAS=yes" in result.stdout, result.stdout + result.stderr
|
|
assert "DEV=[AMD Radeon RX 7900 XTX (RADV NAVI31)]" in result.stdout
|
|
|
|
def test_an_ordinary_device_is_reported_by_name(self, tmp_path):
|
|
result = _run(
|
|
tmp_path, DEVICE_EARLY, 'detect_vulkan; echo "HAS=$HAS_VULKAN DEV=[$VULKAN_DEVICE]"'
|
|
)
|
|
assert "HAS=yes" in result.stdout
|
|
assert "DEV=[Intel(R) Iris(R) Xe Graphics (TGL GT2)]" in result.stdout
|
|
|
|
def test_absent_vulkaninfo_is_no(self, tmp_path):
|
|
result = _run(tmp_path, None, 'detect_vulkan; echo "RC=$? HAS=$HAS_VULKAN"')
|
|
assert "RC=1 HAS=no" in result.stdout, result.stdout + result.stderr
|
|
|
|
|
|
class TestTheTwoDetectorsAgree:
|
|
"""Both readers must see the same devices, or the wizard contradicts itself."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"summary", [NO_DEVICES, DEVICE_PAST_LINE_20, DEVICE_EARLY], ids=["none", "late", "early"]
|
|
)
|
|
def test_a_claimed_gpu_is_a_gpu_the_compat_check_can_see(self, tmp_path, summary):
|
|
result = _run(
|
|
tmp_path,
|
|
summary,
|
|
'detect_vulkan || true; echo "HAS=$HAS_VULKAN"; '
|
|
"check_vulkan_gpu_compatibility || true",
|
|
)
|
|
claimed = "HAS=yes" in result.stdout
|
|
saw_no_devices = "No hardware GPU found" in result.stdout
|
|
assert not (claimed and saw_no_devices), (
|
|
"detect_vulkan announced a GPU that check_vulkan_gpu_compatibility "
|
|
f"cannot find:\n{result.stdout}"
|
|
)
|
|
|
|
|
|
class TestSoftwareRenderers:
|
|
"""whisper.cpp on a CPU implementation of Vulkan is slower than its own CPU
|
|
backend, so a software renderer must not be reported as a GPU."""
|
|
|
|
def test_a_software_renderer_is_not_a_gpu(self, tmp_path):
|
|
result = _run(
|
|
tmp_path,
|
|
SOFTWARE_ONLY,
|
|
'detect_vulkan; echo "RC=$? HAS=$HAS_VULKAN DEV=[$VULKAN_DEVICE]"',
|
|
)
|
|
assert "RC=1 HAS=no DEV=[]" in result.stdout, result.stdout + result.stderr
|
|
|
|
def test_it_is_remembered_for_the_message(self, tmp_path):
|
|
"""Step 1 says "software Vulkan only", which beats "no GPU detected"."""
|
|
result = _run(
|
|
tmp_path, SOFTWARE_ONLY, 'detect_vulkan || true; echo "SOFT=[$VULKAN_SOFTWARE_DEVICE]"'
|
|
)
|
|
assert "SOFT=[llvmpipe (LLVM 15.0.6, 256 bits)]" in result.stdout
|
|
|
|
def test_hardware_still_wins_when_listed_after_software(self, tmp_path):
|
|
result = _run(
|
|
tmp_path,
|
|
SOFTWARE_THEN_HARDWARE,
|
|
'detect_vulkan; echo "HAS=$HAS_VULKAN DEV=[$VULKAN_DEVICE]"',
|
|
)
|
|
assert "HAS=yes" in result.stdout, result.stdout + result.stderr
|
|
assert "DEV=[AMD Radeon RX 7900 XTX (RADV NAVI31)]" in result.stdout
|
|
|
|
def test_both_detectors_agree_that_it_is_not_hardware(self, tmp_path):
|
|
result = _run(
|
|
tmp_path,
|
|
SOFTWARE_ONLY,
|
|
'detect_vulkan || true; echo "HAS=$HAS_VULKAN"; check_vulkan_gpu_compatibility || true',
|
|
)
|
|
assert "HAS=no" in result.stdout
|
|
assert "No hardware GPU found" in result.stdout
|
|
|
|
@pytest.mark.parametrize(
|
|
"name", ["llvmpipe", "swiftshader", "lavapipe", "zink", "virtio", "venus"]
|
|
)
|
|
def test_every_pattern_is_recognised(self, tmp_path, name):
|
|
result = _run(
|
|
tmp_path, NO_DEVICES, f'is_software_renderer "{name} (something)"; echo "RC=$?"'
|
|
)
|
|
assert "RC=0" in result.stdout, result.stdout + result.stderr
|
|
|
|
|
|
class TestDiagnosticsDoNotPolluteCapturedOutput:
|
|
"""install.sh runs under `set -Eeuo pipefail` with an ERR trap that calls
|
|
print_error. While print_error wrote to stdout, that message landed *inside*
|
|
`$(detect_whispercpp_backends)` -- so a machine with a working Vulkan GPU
|
|
parsed garbage, fell back to CPU, and never logged the error, because the
|
|
substitution had swallowed it. `vulkaninfo --features` exits non-zero on real
|
|
drivers, which is what fired the trap.
|
|
"""
|
|
|
|
# A working hardware GPU, and --features exiting non-zero as it does in practice.
|
|
STUB = """#!/bin/bash
|
|
case "$1" in
|
|
--summary) echo "\tdeviceName = Intel(R) Graphics (LNL)" ;;
|
|
--features) echo "VK_KHR_16bit_storage"; exit 1 ;;
|
|
esac
|
|
"""
|
|
|
|
FUNCTIONS = (
|
|
"is_software_renderer",
|
|
"vulkan_device_names",
|
|
"detect_nvidia_gpu",
|
|
"detect_vulkan",
|
|
"check_vulkan_gpu_compatibility",
|
|
"cuda_toolkit_root_has_runtime_library",
|
|
"find_valid_cuda_toolkit_root",
|
|
"detect_whispercpp_backends",
|
|
# the real ones, so the test fails if they go back to stdout
|
|
"print_error",
|
|
"print_warning",
|
|
)
|
|
|
|
def _script(self, tmp_path: Path, body: str) -> subprocess.CompletedProcess:
|
|
binary = tmp_path / "bin"
|
|
binary.mkdir(exist_ok=True)
|
|
for tool in NEEDED_TOOLS + ("find", "ls", "wc", "dirname", "basename", "readlink"):
|
|
found = shutil.which(tool)
|
|
if found and not (binary / tool).exists():
|
|
(binary / tool).symlink_to(found)
|
|
stub = binary / "vulkaninfo"
|
|
stub.write_text(self.STUB)
|
|
stub.chmod(0o755)
|
|
|
|
chunks = [PATTERNS.group(0)]
|
|
for name in self.FUNCTIONS:
|
|
match = re.search(rf"^{name}\(\) \{{.*?^\}}", SOURCE, re.M | re.S)
|
|
assert match, f"{name} not found in install.sh"
|
|
chunks.append(match.group(0))
|
|
|
|
prelude = (
|
|
"set -Eeuo pipefail\n"
|
|
f'export PATH="{binary}"\n'
|
|
'HAS_NVIDIA_GPU="no"\nHAS_VULKAN="no"\nVULKAN_DEVICE=""\n'
|
|
'VULKAN_SOFTWARE_DEVICE=""\nGPU_NAME=""\nGPU_MEMORY=""\n'
|
|
'INSTALL_LOG_FILE="/dev/null"\n'
|
|
'print_info() { echo "INFO: $*"; }\n'
|
|
'print_success() { echo "SUCCESS: $*"; }\n'
|
|
)
|
|
trap = (
|
|
"trap 'print_error \"Unexpected error near line $LINENO "
|
|
"(exit code $?); see $INSTALL_LOG_FILE\"' ERR\n"
|
|
)
|
|
return subprocess.run(
|
|
["bash", "-c", prelude + "\n".join(chunks) + "\n" + trap + body],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
|
|
def test_print_error_stays_off_stdout(self, tmp_path):
|
|
result = self._script(tmp_path, 'print_error "boom"')
|
|
assert "boom" not in result.stdout
|
|
assert "boom" in result.stderr
|
|
|
|
def test_print_warning_stays_off_stdout(self, tmp_path):
|
|
result = self._script(tmp_path, 'print_warning "careful"')
|
|
assert "careful" not in result.stdout
|
|
assert "careful" in result.stderr
|
|
|
|
def test_backend_info_is_a_single_clean_line(self, tmp_path):
|
|
result = self._script(
|
|
tmp_path, 'B=$(detect_whispercpp_backends); printf "%s" "$B" | wc -l; echo "[$B]"'
|
|
)
|
|
assert "ERROR" not in result.stdout, f"trap output captured:\n{result.stdout}"
|
|
lines = result.stdout.splitlines()
|
|
assert lines, result.stdout + result.stderr
|
|
assert lines[0] == "0", f"BACKEND_INFO spans several lines:\n{result.stdout}"
|
|
|
|
def test_a_working_gpu_is_still_offered(self, tmp_path):
|
|
"""The user-visible symptom: Step 3 refused GPU on hardware that supports it."""
|
|
result = self._script(
|
|
tmp_path,
|
|
'B=$(detect_whispercpp_backends); CAN=$(echo "$B" | cut -d: -f3); echo "CAN=[$CAN]"',
|
|
)
|
|
assert "CAN=[true]" in result.stdout, result.stdout + result.stderr
|