xavierk
a451e1d512
The niri example binds move to Mod+Shift+M (toggle), Mod+Shift+S (start) and Mod+Shift+X (stop), free in stock niri and in Dank Material Shell's binds. inject-probe reuses the injector's niri guard, so it no longer reports the portal on niri. Fixes #45 Fixes #46
227 lines
8.5 KiB
Python
227 lines
8.5 KiB
Python
"""Behaviour tests for the ticket-19 performance-baseline protocol script.
|
|
|
|
The script (scripts/measure_perf_baseline.py) is the locked measurement
|
|
protocol for the installed-build baseline: every number it reports comes
|
|
from an observed run on the target machine class, parsed by the pure
|
|
helpers tested here. Tests run offline with fixture text only and never
|
|
touch audio hardware, the GPU, or the network.
|
|
"""
|
|
|
|
import argparse
|
|
import importlib.util
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "measure_perf_baseline.py"
|
|
SPEC = importlib.util.spec_from_file_location("measure_perf_baseline", SCRIPT)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
PROTOCOL = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(PROTOCOL)
|
|
|
|
|
|
def test_parse_vmrss_kb_reads_kilobytes() -> None:
|
|
"""VmRSS lines parse to kilobytes; anything else yields None."""
|
|
status = "Name:\tvoculinux\nVmRSS:\t 412336 kB\nVmSwap:\t0 kB\n"
|
|
assert PROTOCOL.parse_vmrss_kb(status) == 412336
|
|
assert PROTOCOL.parse_vmrss_kb("Name:\tvoculinux\n") is None
|
|
|
|
|
|
def test_describe_reports_count_mean_min_max() -> None:
|
|
"""Spread statistics come from the samples, never fabricated."""
|
|
stats = PROTOCOL.describe([1.0, 2.0, 3.0])
|
|
assert stats["n"] == 3
|
|
assert stats["mean"] == pytest.approx(2.0)
|
|
assert stats["min"] == pytest.approx(1.0)
|
|
assert stats["max"] == pytest.approx(3.0)
|
|
|
|
|
|
def test_describe_empty_is_explicit() -> None:
|
|
"""No samples means n == 0, never a zero mean pretending to be data."""
|
|
stats = PROTOCOL.describe([])
|
|
assert stats["n"] == 0
|
|
assert stats["mean"] is None
|
|
|
|
|
|
WAKE_LOG = """\
|
|
2026-11-05 22:44:01,100 - vocalinux.ui.action_handler - DEBUG - Handling action: toggle
|
|
2026-11-05 22:44:01,350 - vocalinux.audio.capture - DEBUG - Opened capture stream: 1 channel(s) at 16000Hz
|
|
2026-11-05 22:44:05,100 - vocalinux.ui.action_handler - DEBUG - Handling action: toggle
|
|
2026-11-05 22:44:05,500 - vocalinux.audio.capture - DEBUG - Opened capture stream: 1 channel(s) at 16000Hz
|
|
"""
|
|
|
|
|
|
def test_parse_wake_latencies_pairs_toggle_to_capture() -> None:
|
|
"""Each toggle pairs with the next capture-stream line, in seconds."""
|
|
assert PROTOCOL.parse_wake_latencies(WAKE_LOG) == pytest.approx([0.25, 0.4])
|
|
|
|
|
|
def test_parse_wake_latencies_ignores_unpaired_toggle() -> None:
|
|
"""A toggle with no later capture start is dropped, not zero-filled."""
|
|
log = (
|
|
WAKE_LOG
|
|
+ "2026-11-05 22:44:09,100 - vocalinux.ui.action_handler - DEBUG - Handling action: start\n"
|
|
)
|
|
assert PROTOCOL.parse_wake_latencies(log) == pytest.approx([0.25, 0.4])
|
|
|
|
|
|
def test_realtime_factor_is_audio_over_wall() -> None:
|
|
"""RTF below one means faster than real time; zero audio is refused."""
|
|
assert PROTOCOL.realtime_factor(30.0, 15.0) == pytest.approx(0.5)
|
|
with pytest.raises(ValueError):
|
|
PROTOCOL.realtime_factor(0.0, 1.0)
|
|
|
|
|
|
def test_parse_nvidia_smi_csv_reads_first_gpu() -> None:
|
|
"""The locked nvidia-smi query shape parses to total/free MiB."""
|
|
out = (
|
|
"name, driver_version, memory.total [MiB], memory.free [MiB]\n"
|
|
"NVIDIA GeForce RTX 2050, 595.112.07, 4096 MiB, 3711 MiB\n"
|
|
)
|
|
assert PROTOCOL.parse_nvidia_smi_memory(out) == (4096, 3711)
|
|
assert PROTOCOL.parse_nvidia_smi_memory("no gpu here\n") is None
|
|
|
|
|
|
def test_recommend_thresholds_come_from_footprints() -> None:
|
|
"""Ladder thresholds derive from measured footprints plus headroom."""
|
|
footprints = {"q8_0": 1000.0, "q5_0": 700.0, "small": 500.0}
|
|
advice = PROTOCOL.recommend_thresholds(footprints, headroom_mib=512.0)
|
|
assert advice["prefer_q8_0_above_mib"] == pytest.approx(1512.0)
|
|
assert advice["prefer_q5_0_above_mib"] == pytest.approx(1212.0)
|
|
assert advice["floor_small_below_mib"] == pytest.approx(1212.0)
|
|
|
|
|
|
def test_render_report_names_protocol_and_host() -> None:
|
|
"""The evidence report carries protocol version and host facts."""
|
|
report = PROTOCOL.render_markdown_report(
|
|
{"os": "Void", "gpu": "NVIDIA GeForce RTX 2050"}, {"idle": {}}
|
|
)
|
|
assert PROTOCOL.PROTOCOL_VERSION in report
|
|
assert "Void" in report
|
|
assert "NVIDIA GeForce RTX 2050" in report
|
|
|
|
|
|
def test_sample_generator_is_deterministic(tmp_path) -> None:
|
|
"""The fixed sample regenerates byte-identical from its seed."""
|
|
first = tmp_path / "a.wav"
|
|
second = tmp_path / "b.wav"
|
|
PROTOCOL.make_sample_wav(first, seed=19)
|
|
PROTOCOL.make_sample_wav(second, seed=19)
|
|
assert first.read_bytes() == second.read_bytes()
|
|
|
|
|
|
def test_sample_shape_is_thirty_seconds_mono_16k(tmp_path) -> None:
|
|
"""The sample is 30 s, mono, 16 kHz, 16-bit for whisper.cpp."""
|
|
import wave
|
|
|
|
target = tmp_path / "sample.wav"
|
|
PROTOCOL.make_sample_wav(target, seed=19)
|
|
with wave.open(str(target), "rb") as handle:
|
|
assert handle.getnchannels() == 1
|
|
assert handle.getsampwidth() == 2
|
|
assert handle.getframerate() == 16000
|
|
assert handle.getnframes() / 16000 == pytest.approx(30.0)
|
|
|
|
|
|
DBUS_TOGGLE_LOG = """\
|
|
2026-11-05 23:01:11,400 - vocalinux.ui.tray_indicator - DEBUG - Toggle requested in state IDLE
|
|
2026-11-05 23:01:11,546 - vocalinux.audio.capture - DEBUG - Opened capture stream: 2 channel(s) at 44100Hz
|
|
2026-11-05 23:01:16,400 - vocalinux.ui.tray_indicator - DEBUG - Toggle requested in state LISTENING
|
|
2026-11-05 23:01:22,100 - vocalinux.ui.tray_indicator - DEBUG - Toggle requested in state IDLE
|
|
2026-11-05 23:01:22,749 - vocalinux.audio.capture - DEBUG - Opened capture stream: 2 channel(s) at 44100Hz
|
|
"""
|
|
|
|
|
|
def test_parse_wake_latencies_pairs_dbus_toggle_starts_only() -> None:
|
|
"""Only IDLE toggles open capture; a stop toggle never borrows the next run."""
|
|
assert PROTOCOL.parse_wake_latencies(DBUS_TOGGLE_LOG) == pytest.approx([0.146, 0.649])
|
|
|
|
|
|
#: Session variables the injector's niri guard reads, plus Wayland; cleared so
|
|
#: the host's own session (this machine runs niri) cannot leak into a test.
|
|
_SESSION_ENV = (
|
|
"NIRI_SOCKET",
|
|
"XDG_CURRENT_DESKTOP",
|
|
"XDG_SESSION_DESKTOP",
|
|
"DESKTOP_SESSION",
|
|
"WAYLAND_DISPLAY",
|
|
)
|
|
|
|
|
|
class _FakeInjector:
|
|
"""Stands in for TextInjector: a Wayland session that selected wtype."""
|
|
|
|
wayland_mode = True
|
|
wayland_tool = "wtype"
|
|
|
|
|
|
def _run_inject_probe(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
session_env: dict[str, str],
|
|
portal_answers: bool,
|
|
) -> tuple[dict, list[bool]]:
|
|
"""Run inject-probe under a fake session; return its report and portal probe calls."""
|
|
from vocalinux.text_injection import remote_desktop_portal, text_injector
|
|
|
|
probes: list[bool] = []
|
|
|
|
class _SpyPortal:
|
|
def probe(self) -> bool:
|
|
probes.append(True)
|
|
return portal_answers
|
|
|
|
for name in _SESSION_ENV:
|
|
monkeypatch.delenv(name, raising=False)
|
|
for name, value in session_env.items():
|
|
monkeypatch.setenv(name, value)
|
|
monkeypatch.setattr(text_injector, "TextInjector", _FakeInjector)
|
|
monkeypatch.setattr(remote_desktop_portal, "RemoteDesktopPortal", _SpyPortal)
|
|
monkeypatch.setattr(shutil, "which", lambda _name: None)
|
|
assert PROTOCOL.cmd_inject_probe(argparse.Namespace()) == 0
|
|
return json.loads(capsys.readouterr().out), probes
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"session_env",
|
|
[
|
|
{"XDG_CURRENT_DESKTOP": "niri"},
|
|
{"NIRI_SOCKET": "/run/user/1000/niri.wayland-1.1.sock"},
|
|
],
|
|
)
|
|
def test_inject_probe_never_probes_portal_on_niri(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
session_env: dict[str, str],
|
|
) -> None:
|
|
"""The injector never offers the portal on niri, so the probe must not claim it.
|
|
|
|
Both niri signals are covered: the probe reuses the injector's guard, so a
|
|
session the injector treats as niri is reported the same way here.
|
|
"""
|
|
report, probes = _run_inject_probe(
|
|
monkeypatch, capsys, session_env=session_env, portal_answers=True
|
|
)
|
|
assert probes == []
|
|
assert report["portal_available"] is False
|
|
assert report["portal_probe_seconds"] is None
|
|
assert report["portal_skip_reason"] == "niri"
|
|
|
|
|
|
def test_inject_probe_reports_portal_answer_off_niri(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
"""Off niri the portal is probed and its answer is reported unchanged."""
|
|
report, probes = _run_inject_probe(
|
|
monkeypatch,
|
|
capsys,
|
|
session_env={"XDG_CURRENT_DESKTOP": "GNOME"},
|
|
portal_answers=True,
|
|
)
|
|
assert probes == [True]
|
|
assert report["portal_available"] is True
|
|
assert report["portal_skip_reason"] is None
|