From 688d9e9bd2d3333813d197ee7ddc7a61e609622c Mon Sep 17 00:00:00 2001 From: Kacper Paczos Date: Mon, 7 Sep 2026 23:11:40 +0200 Subject: [PATCH] feat(language): start a first run from the language the system points at (#796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(language): start a first run from the language the system points at Nothing asked the system what language the user speaks, so whisper.cpp always began on auto-detect and VOSK on a hardcoded en-us. Both are wrong for anyone who does not speak English, and neither improves on its own because nothing ever suggests setting a language. Seed the language on first run only, so a saved choice — including a deliberate "auto" — is never touched. The keyboard layout is consulted before the locale: running the desktop in English while speaking something else is common among the people who install a Linux dictation tool, which makes the interface language a weak signal for the spoken one. A "us" layout is the default on many installs and says little, so the locale gets a say before it is accepted. localectl is used rather than setxkbmap, which reports nothing useful on Wayland. Detection failures leave the packaged default in place. Closes #777 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TKVa2qTLQYMCHffjp97yqT * fix(language): prefer LANGUAGE, keep cs/ca, map nb_NO LANG was consulted before LANGUAGE, so a gettext preference list lost to the fallback locale. C/POSIX detection used startswith("c"), which rejected Czech and Catalan. nb_NO normalised to nb_no and missed the nb→no catalogue map, so Norwegian fell through to en-us. Tests cover LANG+LANGUAGE, LANGUAGE=cs, and nb_NO. * fix(language): walk LANGUAGE prefs until one is supported split(":")[0] dropped later supported preferences, so first-run could stay on auto when LANGUAGE started with an unsupported tag. Try each preference through the catalogue map. Annotate _seed_language_from_system. * style(language): keep _language_from_locale_env on one line for black * fix(language): fall through unmapped LANGUAGE to LANG When every LANGUAGE preference missed the catalogue, the helper returned before LANG was examined, so first-run could stay on auto. Skip unmapped env vars and keep looking. Regression: LANGUAGE=xx:yy with LANG=pl_PL → pl. --------- Co-authored-by: Kacper Paczos <23406555+kacperpaczos@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Jatin K Malik --- src/vocalinux/ui/config_manager.py | 27 ++++ src/vocalinux/utils/system_language.py | 202 +++++++++++++++++++++++++ tests/test_config_manager.py | 10 ++ tests/test_system_language.py | 174 +++++++++++++++++++++ 4 files changed, 413 insertions(+) create mode 100644 src/vocalinux/utils/system_language.py create mode 100644 tests/test_system_language.py diff --git a/src/vocalinux/ui/config_manager.py b/src/vocalinux/ui/config_manager.py index a29b368..cea67d2 100644 --- a/src/vocalinux/ui/config_manager.py +++ b/src/vocalinux/ui/config_manager.py @@ -173,6 +173,32 @@ class ConfigManager: """Ensure the configuration directory exists.""" os.makedirs(CONFIG_DIR, exist_ok=True) + def _seed_language_from_system(self) -> None: + """Start a first run from the language the system points at (#777). + + Only applied when there is no config file yet, so a saved choice — including + a deliberate "auto" — is never overwritten. When nothing decisive is found + the packaged default stays in place. + """ + try: + from ..utils.system_language import detect_system_language + from ..utils.vosk_model_info import SUPPORTED_LANGUAGES + except ImportError as exc: # pragma: no cover - defensive + logger.debug(f"Language detection unavailable: {exc}") + return + + try: + detected = detect_system_language(SUPPORTED_LANGUAGES) + except Exception as exc: # pragma: no cover - detection must never block startup + logger.debug(f"Language detection failed: {exc}") + return + + if not detected: + return + + self.config.setdefault("speech_recognition", {})["language"] = detected + logger.info(f"First run: starting with language {detected}") + def load_config(self): """ Load configuration from the config file. @@ -181,6 +207,7 @@ class ConfigManager: """ if not os.path.exists(CONFIG_FILE): logger.info(f"Config file not found at {CONFIG_FILE}. Using defaults.") + self._seed_language_from_system() return try: diff --git a/src/vocalinux/utils/system_language.py b/src/vocalinux/utils/system_language.py new file mode 100644 index 0000000..2e3b3fe --- /dev/null +++ b/src/vocalinux/utils/system_language.py @@ -0,0 +1,202 @@ +"""Work out which language the user probably speaks (#777). + +Nothing used to ask the system this, so whisper.cpp always started on auto-detect +and VOSK on a hardcoded ``en-us``. Both are wrong for anyone who does not speak +English, and neither ever improves on its own. + +The keyboard layout is consulted before the locale on purpose. Plenty of people +run their desktop in English and speak something else — among the people who +install a Linux dictation tool that is closer to the rule than the exception — +so the interface language is a weak signal for *spoken* language. What someone +types in tracks what they say far more closely. +""" + +import logging +import os +import re +import subprocess +from typing import Optional + +from .host_process import host_env + +logger = logging.getLogger(__name__) + +# xkb layout names that do not match the language id we use. Layouts not listed +# here are looked up as-is, which covers pl, de, fr, it, es, ru and the rest. +_LAYOUT_TO_LANGUAGE = { + "us": "en-us", + "gb": "en-us", + "cz": "cs", + "se": "sv", + "dk": "da", + "gr": "el", + "ua": "uk", + "jp": "ja", + "kr": "ko", + "cn": "zh", + "ir": "fa", + "il": "he", + "br": "pt", + "nb": "no", + "nn": "no", + "vn": "vi", +} + +# Locale territory codes that pick a specific catalogue entry. +_LOCALE_TO_LANGUAGE = { + "en_in": "en-in", + "nb": "no", + "nn": "no", + "zh_cn": "zh", + "zh_tw": "zh", +} + +_LOCALE_ENV_VARS = ("LC_ALL", "LC_MESSAGES", "LANGUAGE", "LANG") + + +def _normalise(value: str) -> str: + return value.strip().lower().replace("-", "_") + + +def _language_for_layout(layout: str, supported: set[str] | dict) -> Optional[str]: + layout = _normalise(layout) + if not layout: + return None + mapped = _LAYOUT_TO_LANGUAGE.get(layout, layout) + return mapped if mapped in supported else None + + +def _language_for_locale(value: str, supported: set[str] | dict) -> Optional[str]: + """Map a locale string such as ``pl_PL.UTF-8`` onto a catalogue entry.""" + value = _normalise(value) + if not value: + return None + + # Strip the encoding and any modifier: pl_pl.utf_8@euro -> pl_pl + value = re.split(r"[.@]", value, maxsplit=1)[0] + # Exact C/POSIX only — startswith("c") wrongly rejects cs/ca/cy. + if value in ("c", "posix"): + return None + + if value in _LOCALE_TO_LANGUAGE: + candidate = _LOCALE_TO_LANGUAGE[value] + return candidate if candidate in supported else None + + if value in supported: + return value + + base = value.split("_", 1)[0] + if base in _LOCALE_TO_LANGUAGE: + candidate = _LOCALE_TO_LANGUAGE[base] + return candidate if candidate in supported else None + if base == "en": + return "en-us" if "en-us" in supported else None + return base if base in supported else None + + +def detect_keyboard_layout() -> Optional[str]: + """Return the primary xkb layout, or None when it cannot be read. + + ``localectl`` is used rather than ``setxkbmap`` because the latter reports + nothing useful on Wayland, where the compositor owns the keyboard. + """ + try: + result = subprocess.run( + ["localectl", "status"], + capture_output=True, + text=True, + timeout=2, + env=host_env(), + ) + except (FileNotFoundError, OSError, subprocess.SubprocessError) as exc: + logger.debug(f"Could not query localectl: {exc}") + return None + + if result.returncode != 0: + logger.debug(f"localectl exited with {result.returncode}") + return None + + layout = None + keymap = None + for line in result.stdout.splitlines(): + key, _, value = line.partition(":") + key = key.strip().lower() + value = value.strip() + if key == "x11 layout" and value: + layout = value + elif key == "vc keymap" and value: + keymap = value + + # A multi-layout setup lists the primary one first: "pl,us". + chosen = layout or keymap + if not chosen: + return None + return chosen.split(",")[0].strip() or None + + +def detect_locale_language(environ: Optional[dict] = None) -> Optional[str]: + """Return the first locale value the environment offers, or None. + + LANGUAGE is returned intact (a colon-separated preference list). Mapping + that list onto a supported catalogue entry happens in + ``_language_from_locale_env``. + """ + environ = os.environ if environ is None else environ + for name in _LOCALE_ENV_VARS: + value = environ.get(name) + if value: + return value + return None + + +def _language_from_locale_env(environ: dict, supported: set[str] | dict) -> Optional[str]: + """Map locale env vars onto a catalogue entry, first supported win. + + LANGUAGE is a colon-separated preference list: each entry is tried through + ``_language_for_locale`` until one is supported. An env var whose values + are all unmapped is skipped so a later var (typically LANG) can still + win. LC_ALL, LC_MESSAGES, and LANG are single values and are not walked + as lists. + """ + for name in _LOCALE_ENV_VARS: + value = environ.get(name) + if not value: + continue + candidates = value.split(":") if name == "LANGUAGE" else (value,) + for candidate in candidates: + mapped = _language_for_locale(candidate, supported) + if mapped: + return mapped + continue + return None + + +def detect_system_language( + supported: set[str] | dict, environ: Optional[dict] = None +) -> Optional[str]: + """Return the language id to start from, or None when nothing is decisive. + + A non-English keyboard layout is the strongest signal available and wins. + "us" is the fallback layout on a great many installs and therefore says very + little, so the locale gets a say before that layout is accepted. + """ + layout_language = None + layout = detect_keyboard_layout() + if layout: + layout_language = _language_for_layout(layout, supported) + if layout_language and not layout_language.startswith("en"): + logger.info(f"Language {layout_language} taken from keyboard layout {layout!r}") + return layout_language + + environ = os.environ if environ is None else environ + locale_language = _language_from_locale_env(environ, supported) + if locale_language: + logger.info(f"Language {locale_language} taken from locale environment") + return locale_language + + if layout_language: + logger.info(f"Language {layout_language} taken from keyboard layout {layout!r}") + return layout_language + + logger.info("Could not work out a language from the system") + return None diff --git a/tests/test_config_manager.py b/tests/test_config_manager.py index d88164c..92bfd59 100644 --- a/tests/test_config_manager.py +++ b/tests/test_config_manager.py @@ -52,6 +52,15 @@ class TestConfigManager(unittest.TestCase): # Recreate after patching so each test starts from a known config path. _ensure_test_config_dir(self.temp_config_dir) + # Every test here starts without a config file, which is a first run, so + # language detection would shell out to localectl and make the result depend + # on the machine's keyboard layout. The seeding itself is covered in + # tests/test_system_language.py. + self.detect_patcher = patch( + "vocalinux.utils.system_language.detect_system_language", return_value=None + ) + self.detect_patcher.start() + # Patch logging to avoid actual logging self.logger_patcher = patch("vocalinux.ui.config_manager.logger") self.mock_logger = self.logger_patcher.start() @@ -61,6 +70,7 @@ class TestConfigManager(unittest.TestCase): self.config_dir_patcher.stop() self.config_file_patcher.stop() self.makedirs_patcher.stop() + self.detect_patcher.stop() self.logger_patcher.stop() self.temp_dir.cleanup() diff --git a/tests/test_system_language.py b/tests/test_system_language.py new file mode 100644 index 0000000..a470deb --- /dev/null +++ b/tests/test_system_language.py @@ -0,0 +1,174 @@ +"""First run should start from the language the system points at (#777).""" + +from unittest.mock import patch + +import pytest + +from vocalinux.ui import config_manager as cm +from vocalinux.utils import system_language as sl +from vocalinux.utils.vosk_model_info import SUPPORTED_LANGUAGES + + +@pytest.fixture +def isolated_config(tmp_path, monkeypatch): + monkeypatch.setattr(cm, "CONFIG_DIR", str(tmp_path)) + monkeypatch.setattr(cm, "CONFIG_FILE", str(tmp_path / "config.json")) + monkeypatch.setattr(cm, "_shared_instance", None) + return tmp_path + + +@pytest.mark.parametrize( + "layout,expected", + [ + ("pl", "pl"), + ("de", "de"), + ("cz", "cs"), + ("se", "sv"), + ("us", "en-us"), + ("gb", "en-us"), + ("xyz", None), + ], +) +def test_layouts_map_onto_catalogue_entries(layout, expected): + assert sl._language_for_layout(layout, SUPPORTED_LANGUAGES) == expected + + +@pytest.mark.parametrize( + "value,expected", + [ + ("pl_PL.UTF-8", "pl"), + ("de_DE@euro", "de"), + ("en_US.UTF-8", "en-us"), + ("en_GB", "en-us"), + ("en_IN", "en-in"), + ("cs", "cs"), + ("cs_CZ.UTF-8", "cs"), + ("ca", "ca"), + ("ca_ES.UTF-8", "ca"), + ("nb_NO", "no"), + ("nb_NO.UTF-8", "no"), + ("nn_NO", "no"), + ("C", None), + ("POSIX", None), + ("C.UTF-8", None), + ("xx_YY", None), + ], +) +def test_locales_map_onto_catalogue_entries(value, expected): + assert sl._language_for_locale(value, SUPPORTED_LANGUAGES) == expected + + +def test_a_non_english_layout_beats_an_english_locale(): + """The reported case: desktop in English, keyboard in Polish, speaks Polish.""" + with patch.object(sl, "detect_keyboard_layout", return_value="pl"): + detected = sl.detect_system_language(SUPPORTED_LANGUAGES, {"LANG": "en_US.UTF-8"}) + + assert detected == "pl" + + +def test_a_us_layout_defers_to_the_locale(): + """ "us" is the default layout on many installs, so it carries little signal.""" + with patch.object(sl, "detect_keyboard_layout", return_value="us"): + detected = sl.detect_system_language(SUPPORTED_LANGUAGES, {"LANG": "pl_PL.UTF-8"}) + + assert detected == "pl" + + +def test_a_us_layout_is_used_when_the_locale_says_nothing(): + with patch.object(sl, "detect_keyboard_layout", return_value="us"): + detected = sl.detect_system_language(SUPPORTED_LANGUAGES, {"LANG": "C"}) + + assert detected == "en-us" + + +def test_nothing_decisive_returns_none(): + with patch.object(sl, "detect_keyboard_layout", return_value=None): + assert sl.detect_system_language(SUPPORTED_LANGUAGES, {}) is None + + +def test_a_missing_localectl_does_not_raise(): + with patch.object(sl.subprocess, "run", side_effect=FileNotFoundError): + assert sl.detect_keyboard_layout() is None + + +def test_first_run_starts_from_the_detected_language(isolated_config): + """Without this the fresh config keeps the packaged "auto".""" + with patch.object(sl, "detect_keyboard_layout", return_value="pl"): + with patch.dict("os.environ", {"LANG": "en_US.UTF-8"}, clear=False): + manager = cm.ConfigManager() + + assert manager.get("speech_recognition", "language") == "pl" + + +def test_first_run_keeps_the_default_when_detection_finds_nothing(isolated_config): + with patch.object(sl, "detect_keyboard_layout", return_value=None): + with patch.dict("os.environ", {"LANG": "C", "LC_ALL": "C"}, clear=True): + manager = cm.ConfigManager() + + assert manager.get("speech_recognition", "language") == "auto" + + +def test_a_saved_language_is_never_overwritten(isolated_config): + """A returning user who chose auto-detect keeps it.""" + import json + + with open(cm.CONFIG_FILE, "w") as handle: + json.dump({"speech_recognition": {"language": "auto"}}, handle) + + with patch.object(sl, "detect_keyboard_layout", return_value="pl"): + manager = cm.ConfigManager() + + assert manager.get("speech_recognition", "language") == "auto" + + +def test_language_env_beats_lang_when_both_set(): + """LANGUAGE is the gettext preference list; LANG must not mask it (#796).""" + with patch.object(sl, "detect_keyboard_layout", return_value="us"): + detected = sl.detect_system_language( + SUPPORTED_LANGUAGES, + {"LANG": "en_US.UTF-8", "LANGUAGE": "pl:en"}, + ) + + assert detected == "pl" + + +def test_language_env_skips_unsupported_preferences(): + """An unsupported first LANGUAGE pref must not hide a later supported one.""" + with patch.object(sl, "detect_keyboard_layout", return_value="us"): + detected = sl.detect_system_language( + SUPPORTED_LANGUAGES, + {"LANG": "en_US.UTF-8", "LANGUAGE": "xx:pl:en"}, + ) + + assert detected == "pl" + + +def test_unmapped_language_env_falls_through_to_lang(): + """An unsupported LANGUAGE list must not hide LANG (#796).""" + with patch.object(sl, "detect_keyboard_layout", return_value="us"): + detected = sl.detect_system_language( + SUPPORTED_LANGUAGES, + {"LANGUAGE": "xx:yy", "LANG": "pl_PL.UTF-8"}, + ) + + assert detected == "pl" + + +def test_language_env_czech_is_not_treated_as_c(): + with patch.object(sl, "detect_keyboard_layout", return_value="us"): + detected = sl.detect_system_language( + SUPPORTED_LANGUAGES, + {"LANG": "en_US.UTF-8", "LANGUAGE": "cs"}, + ) + + assert detected == "cs" + + +def test_nb_no_locale_maps_to_norwegian(): + with patch.object(sl, "detect_keyboard_layout", return_value="us"): + detected = sl.detect_system_language( + SUPPORTED_LANGUAGES, + {"LANG": "nb_NO.UTF-8"}, + ) + + assert detected == "no"