feat(asr): FunASR/SenseVoice support for the Remote API engine (#468)

* feat(asr): add chat audio remote benchmarking

* style(asr): apply black formatting

* test(main): include remote API model kwarg

* test(asr): add coverage for chat-completions error handling and parsing

Add 10 new tests to cover uncovered lines in PR #468:
- ConnectionError handler in _try_chat_completions_audio_api (lines 1601-1602)
- _parse_chat_completion_transcription edge cases: None, list content,
  empty/whitespace strings, JSON without text field, plain text, and
  single-line language metadata (lines 1610, 1613-1621, 1625, 1629-1640)

Coverage for recognition_manager.py improved from 85% to 86%.

* feat(asr): support FunASR remote responses

* refactor(asr): scope remote FunASR support to the #469 path

Drop chat-completions/Qwen client, evaluation notes, and the remote ASR
benchmark harness. Keep model field, tolerant OpenAI response parsing,
SenseVoice label stripping, FunASR docs, and tests.
This commit is contained in:
Jatin K Malik
2026-07-13 15:03:19 -07:00
committed by GitHub
parent 9d9638e854
commit 85927be4b4
11 changed files with 236 additions and 25 deletions
+1
View File
@@ -392,6 +392,7 @@ def main():
remote_api_url=saved_settings.get("remote_api_url", ""),
remote_api_key=saved_settings.get("remote_api_key", ""),
remote_api_endpoint=saved_settings.get("remote_api_endpoint", "/inference"),
remote_api_model=saved_settings.get("remote_api_model", "whisper-1"),
)
# Initialize text injection system
@@ -11,6 +11,7 @@ import json
import logging
import os
import queue
import re
import sys
import threading
import time
@@ -844,6 +845,7 @@ class SpeechRecognitionManager:
self.remote_api_url = kwargs.get("remote_api_url", "")
self.remote_api_key = kwargs.get("remote_api_key", "")
self.remote_api_endpoint = kwargs.get("remote_api_endpoint", "/inference")
self.remote_api_model = kwargs.get("remote_api_model", "whisper-1")
self._http_session = None
# Audio diagnostics tracking
@@ -1576,7 +1578,7 @@ class SpeechRecognitionManager:
url = f"{self.remote_api_url}{self.remote_api_endpoint}"
files = {"file": ("audio.wav", wav_bytes, "audio/wav")}
data = {"model": "whisper-1"}
data = {"model": self.remote_api_model or "whisper-1"}
if lang:
data["language"] = lang
@@ -1590,8 +1592,7 @@ class SpeechRecognitionManager:
response.raise_for_status()
result = response.json()
# OpenAI format returns {"text": "..."}
return result.get("text", "")
return self._extract_remote_transcription_text(result)
except requests.exceptions.ConnectionError as e:
logger.error(f"Cannot connect to remote server {url}: {e}")
@@ -1600,6 +1601,43 @@ class SpeechRecognitionManager:
logger.debug(f"OpenAI API format attempt failed: {e}")
return None
def _extract_remote_transcription_text(self, payload: object) -> str:
"""Extract transcript text from OpenAI-compatible ASR responses.
FunASR/SenseVoice wrappers may include extra metadata or segment arrays.
Vocalinux only injects the transcript text and ignores the metadata.
"""
if payload is None:
return ""
if isinstance(payload, str):
return self._strip_remote_transcription_metadata(payload)
if isinstance(payload, list):
parts = [self._extract_remote_transcription_text(item).strip() for item in payload]
return "\n".join(part for part in parts if part)
if not isinstance(payload, dict):
return self._strip_remote_transcription_metadata(str(payload))
for key in ("text", "transcript", "transcription"):
value = payload.get(key)
if isinstance(value, str):
return self._strip_remote_transcription_metadata(value)
segments = payload.get("segments")
if isinstance(segments, list):
return self._extract_remote_transcription_text(segments)
return ""
def _strip_remote_transcription_metadata(self, text: str) -> str:
"""Remove inline ASR metadata tags while preserving ordinary text."""
if not text.strip():
return ""
return re.sub(r"^\s*(?:<\|[^|>\n]+\|>)+\s*", "", text)
def _try_whispercpp_server_api(self, wav_bytes: bytes, lang, headers: dict, session):
"""Try to transcribe using whisper.cpp server API format.
@@ -2825,6 +2863,8 @@ class SpeechRecognitionManager:
self.remote_api_key = kwargs.get("remote_api_key", "")
if "remote_api_endpoint" in kwargs:
self.remote_api_endpoint = kwargs.get("remote_api_endpoint", "/inference")
if "remote_api_model" in kwargs:
self.remote_api_model = kwargs.get("remote_api_model", "whisper-1")
self._voice_commands_enabled = self._resolve_voice_commands_enabled()
+2 -1
View File
@@ -31,7 +31,8 @@ DEFAULT_CONFIG = {
"voice_commands_enabled": None, # None = auto (enabled for VOSK, disabled for Whisper)
"remote_api_url": "", # Remote speech recognition server URL (e.g. http://192.168.1.100:8080)
"remote_api_key": "", # Remote server API key (optional)
"remote_api_endpoint": "/inference", # Remote server API endpoint format (/v1/audio/transcriptions or /inference)
"remote_api_endpoint": "/inference", # Remote server API endpoint format
"remote_api_model": "whisper-1", # Model name sent to compatible remote APIs
},
"audio": {
"device_index": None, # Audio input device index (None for system default)
+24 -2
View File
@@ -1437,17 +1437,19 @@ class SettingsDialog(Gtk.Dialog):
self.language_combo.connect("changed", self._on_language_changed)
def _on_remote_api_settings_changed(self, widget):
"""Handle remote API URL/Key/endpoint changes (Advanced tab)."""
"""Handle remote API URL/Key/endpoint changes."""
if self._initializing or self._applying_settings:
return
url = self.remote_api_url_entry.get_text().strip()
key = self.remote_api_key_entry.get_text().strip()
endpoint = self.remote_api_endpoint_combo.get_active_id() or "/inference"
model = self.remote_api_model_entry.get_text().strip() or "whisper-1"
self.config_manager.set("speech_recognition", "remote_api_url", url)
self.config_manager.set("speech_recognition", "remote_api_key", key)
self.config_manager.set("speech_recognition", "remote_api_endpoint", endpoint)
self.config_manager.set("speech_recognition", "remote_api_model", model)
self.config_manager.save_settings()
# Auto apply only if remote engine is currently active
@@ -2252,7 +2254,7 @@ class SettingsDialog(Gtk.Dialog):
"Select the API format of the remote server (API Endpoint Format)"
)
self.remote_api_endpoint_combo.append(
"/v1/audio/transcriptions", "OpenAI (/v1/audio/transcriptions)"
"/v1/audio/transcriptions", "OpenAI/FunASR (/v1/audio/transcriptions)"
)
self.remote_api_endpoint_combo.append("/inference", "Whisper.cpp (/inference)")
_prevent_scroll_on_hover(self.remote_api_endpoint_combo)
@@ -2263,6 +2265,20 @@ class SettingsDialog(Gtk.Dialog):
)
self.remote_server_group.add_row(remote_endpoint_row)
# Model name
self.remote_api_model_entry = Gtk.Entry()
self.remote_api_model_entry.set_placeholder_text("whisper-1")
self.remote_api_model_entry.set_tooltip_text(
"Model identifier sent to OpenAI-compatible and FunASR servers"
)
self.remote_api_model_entry.set_size_request(280, -1)
remote_model_row = PreferenceRow(
title="Model",
subtitle="Remote model name, for example whisper-1 or sensevoice",
widget=self.remote_api_model_entry,
)
self.remote_server_group.add_row(remote_model_row)
# Connection test
self.remote_test_btn = Gtk.Button(label="Test Connection")
self.remote_test_btn.set_tooltip_text("Test connection to remote server")
@@ -2289,15 +2305,18 @@ class SettingsDialog(Gtk.Dialog):
saved_endpoint = self.config_manager.get(
"speech_recognition", "remote_api_endpoint", "/inference"
)
saved_model = self.config_manager.get("speech_recognition", "remote_api_model", "whisper-1")
if saved_url:
self.remote_api_url_entry.set_text(saved_url)
if saved_key:
self.remote_api_key_entry.set_text(saved_key)
self.remote_api_endpoint_combo.set_active_id(saved_endpoint)
self.remote_api_model_entry.set_text(saved_model or "whisper-1")
self.remote_api_url_entry.connect("changed", self._on_remote_api_settings_changed)
self.remote_api_key_entry.connect("changed", self._on_remote_api_settings_changed)
self.remote_api_endpoint_combo.connect("changed", self._on_remote_api_settings_changed)
self.remote_api_model_entry.connect("changed", self._on_remote_api_settings_changed)
self.remote_server_group.hide()
self.remote_status_label.hide()
@@ -3217,6 +3236,9 @@ class SettingsDialog(Gtk.Dialog):
settings["remote_api_endpoint"] = (
self.remote_api_endpoint_combo.get_active_id() or "/inference"
)
settings["remote_api_model"] = (
self.remote_api_model_entry.get_text().strip() or "whisper-1"
)
return settings