diff --git a/src/vocalinux/speech_recognition/recognition_manager.py b/src/vocalinux/speech_recognition/recognition_manager.py index 48b8934..a7763ea 100644 --- a/src/vocalinux/speech_recognition/recognition_manager.py +++ b/src/vocalinux/speech_recognition/recognition_manager.py @@ -17,7 +17,10 @@ import sys import threading import time from pathlib import Path -from typing import Callable, Optional +from typing import TYPE_CHECKING, Callable, Optional + +if TYPE_CHECKING: + import numpy as np from ..common_types import RecognitionState from ..ui.audio_feedback import play_error_sound, play_start_sound, play_stop_sound @@ -518,10 +521,10 @@ def _open_capture_stream(audio, device_index: Optional[int] = None) -> tuple[int heap corruption (see GitHub issue #567). This function opens PortAudio exactly once per capture session: candidate - formats are tried in order (device default rate first, mono before stereo, - stereo skipped entirely for mono-only devices) and the FIRST successfully - opened stream is returned to the caller for actual capture — never closed - and reopened. + formats are tried in order (device default rate first, native channel count + first for 2–8ch devices, stereo skipped entirely for mono-only devices) + and the FIRST successfully opened stream is returned to the caller for + actual capture — never closed and reopened. Args: audio: PyAudio instance @@ -550,18 +553,25 @@ def _open_capture_stream(audio, device_index: Optional[int] = None) -> tuple[int if rate not in rates_to_try: rates_to_try.append(rate) - # Never probe stereo on a device that reports a single input channel - # (opening with more channels than supported is itself a known - # PortAudio/ALSA corruption trigger). Stereo-capable devices must be - # opened at their native layout first: Intel SOF DMICs (Raptor Lake - # "Digital Microphone", etc.) often only support 2ch at the PCM, and - # PortAudio can abort with heap corruption if open() accepts a converted - # 1ch stream and the callback then overruns (#666). + # Never probe extra channels on a device that reports a single input + # channel (opening with more than supported is itself a known + # PortAudio/ALSA corruption trigger). Devices must be opened at their + # native layout first: Intel SOF DMICs often only support 2ch at the + # PCM (#666), and HDA analog mics commonly expose 4 capture channels + # even when only the first pair is a mic (#813). Opening below native + # channel count can succeed then abort in PortAudio CleanUpStream with + # ``free(): corrupted unsorted chunks``. Pulse virtual devices often + # report 32 or 128 channels; those are not native PCM layouts, so we + # still try 2ch then 1ch. reported_channels = int(device_info.get("maxInputChannels", 0) or 0) if reported_channels == 1: channel_options = [1] - elif reported_channels >= 2: + elif reported_channels == 2: channel_options = [2, 1] + elif 2 < reported_channels <= 8: + channel_options = [reported_channels, 2, 1] # HDA 4ch (#813) + elif reported_channels > 8: + channel_options = [2, 1] # Pulse 32/128 else: channel_options = [1, 2] @@ -600,6 +610,79 @@ def _open_capture_stream(audio, device_index: Optional[int] = None) -> tuple[int return 1, 16000, None +# Mean-square energy floor on int16 PCM before locking the N>=3 sticky channel. +# Capture samples are raw int16 (see _record_audio np.frombuffer(..., int16)); +# RMS ≈ 100 ≈ -50 dBFS — above idle dither/ambient, well below speech. +_STICKY_LOCK_MIN_MEAN_SQUARE = 10_000.0 + + +def _downmix_to_mono( + audio_array: "np.ndarray", + channels: int, + sticky_channel: Optional[int] = None, +) -> tuple["np.ndarray", Optional[int]]: + """Downmix interleaved int16 PCM to mono. + + Speech recognition engines expect mono audio. + + Stereo (2ch) still averages both channels. For 3+ channels, energy-aware + selection is required: the microphone may not be on ch0/ch1 (HDA analog + capture often puts it on ch2/ch3), so keeping only the first stereo pair + can yield silence, while averaging all N attenuates speech when only some + channels are live. + + While sticky is unset, each buffer returns the loudest channel by + mean-square energy, but sticky is only *set* when that channel's + mean-square exceeds :data:`_STICKY_LOCK_MIN_MEAN_SQUARE` (speech-gated + lock). That avoids pinning an ambient/noise channel from a silent first + buffer before the mic speaks. Once sticky is set, later buffers reuse it + until cleared on open/reconnect/cleanup so a noise burst on another input + cannot switch the mic mid-utterance. + + Args: + audio_array: 1-D int16 samples with interleaved channels. + channels: Number of interleaved channels in *audio_array*. + sticky_channel: Previously selected channel for N>=3 streams, or + ``None`` to (re)select by loudest mean-square energy. + + Returns: + ``(mono, sticky)`` where *mono* is 1-D int16 samples and *sticky* is + the channel index to reuse on the next N>=3 buffer (``None`` for + N<=2, or when N>=3 and no speech-gated lock yet). ``channels <= 1`` + is a passthrough. If ``len(audio_array)`` is not divisible by + *channels*, leftover samples are truncated using the full N-channel + frame width before reshape; an empty array after truncation is + returned as-is (sticky unchanged for N>=3 when already set, else + ``None``). Stereo averages both channels; N>=3 uses the sticky index + when in range, otherwise the loudest channel by per-buffer + mean-square (ties keep the first index), locking sticky only when + that channel clears the non-silence energy floor. + """ + if channels <= 1: + return audio_array, None + leftover = len(audio_array) % channels + if leftover: + audio_array = audio_array[: len(audio_array) - leftover] + if len(audio_array) == 0: + if channels >= 3 and sticky_channel is not None and 0 <= sticky_channel < channels: + return audio_array, sticky_channel + return audio_array, None + frames = audio_array.reshape(-1, channels) + if channels == 2: + return frames.mean(axis=1).astype(audio_array.dtype), None + if sticky_channel is not None and 0 <= sticky_channel < channels: + selected = int(sticky_channel) + return frames[:, selected].astype(audio_array.dtype), selected + # Cast before squaring so int16 does not overflow. + energy = (frames.astype("float64") ** 2).mean(axis=0) + selected = int(energy.argmax()) + mono = frames[:, selected].astype(audio_array.dtype) + # Speech-gate: return loudest mono now, but only pin sticky on real energy. + if float(energy[selected]) >= _STICKY_LOCK_MIN_MEAN_SQUARE: + return mono, selected + return mono, None + + def _get_supported_channels(audio, device_index: Optional[int] = None) -> int: """ Detect the supported number of channels for the audio device. @@ -613,7 +696,8 @@ def _get_supported_channels(audio, device_index: Optional[int] = None) -> int: device_index: The device index to test (None for default) Returns: - int: Number of channels supported (1 or 2), defaults to 1 + int: Negotiated channel count (1, 2, or native 3–8 for HDA), + defaults to 1 """ channels, _rate, stream = _open_capture_stream(audio, device_index) _safe_close_stream(stream) @@ -1087,6 +1171,8 @@ class SpeechRecognitionManager: self._audio_stream = None self._pyaudio_instance = None self._capture_sample_rate = 16000 # Default, updated when device is opened + self._capture_channels = 1 # Default, updated when device is opened + self._capture_downmix_channel = None # Speech-gated sticky N>=3 channel for open stream # Create models directory if it doesn't exist os.makedirs(MODELS_DIR, exist_ok=True) @@ -3218,6 +3304,8 @@ class SpeechRecognitionManager: CHANNELS, RATE, negotiated_stream = _open_capture_stream(audio, resolved_device_index) logger.info(f"Using {CHANNELS} channel(s) for recording") self._capture_sample_rate = RATE + self._capture_channels = CHANNELS + self._capture_downmix_channel = None # New stream: speech-gated sticky unset logger.info(f"Using sample rate: {RATE}Hz") try: @@ -3266,6 +3354,7 @@ class SpeechRecognitionManager: # Attempt reconnection if self._attempt_audio_reconnection(audio): stream = self._audio_stream + CHANNELS = self._capture_channels else: play_error_sound() audio.terminate() @@ -3307,14 +3396,15 @@ class SpeechRecognitionManager: data = stream.read(CHUNK, exception_on_overflow=False) - # Convert stereo to mono if necessary + # Convert multi-channel capture to mono if necessary # Speech recognition engines expect mono (1 channel) audio - if CHANNELS == 2: + if CHANNELS > 1: audio_array = np.frombuffer(data, dtype=np.int16) - # Reshape to (n_samples, 2) and average channels - stereo_samples = audio_array.reshape(-1, 2) - mono_samples = stereo_samples.mean(axis=1).astype(np.int16) - data = mono_samples.tobytes() + mono, selected = _downmix_to_mono( + audio_array, CHANNELS, self._capture_downmix_channel + ) + self._capture_downmix_channel = selected + data = mono.tobytes() # Resample to 16kHz if capturing at non-16kHz for Vosk/Whisper compatibility if self._capture_sample_rate != 16000: @@ -3441,6 +3531,7 @@ class SpeechRecognitionManager: if self._attempt_audio_reconnection(audio): logger.info("Audio reconnection successful, continuing recording") stream = self._audio_stream # Update stream reference + CHANNELS = self._capture_channels continue # Continue recording with new stream else: logger.error("Audio reconnection failed, stopping recording") @@ -3466,6 +3557,7 @@ class SpeechRecognitionManager: # Reset audio stream reference and reconnection state self._audio_stream = None self._pyaudio_instance = None + self._capture_downmix_channel = None self._reconnection_attempts = 0 self._last_audio_error_time = 0 @@ -3896,6 +3988,8 @@ class SpeechRecognitionManager: CHANNELS, RATE, new_stream = _open_capture_stream(audio_instance, resolved_device_index) logger.debug(f"Reconnecting with {CHANNELS} channel(s)") self._capture_sample_rate = RATE + self._capture_channels = CHANNELS + self._capture_downmix_channel = None # Reopened stream: clear speech-gated sticky logger.debug(f"Reconnecting with sample rate: {RATE}Hz") if new_stream is None: diff --git a/tests/test_recognition_manager_device_config.py b/tests/test_recognition_manager_device_config.py index f4f40e5..ad0e6df 100644 --- a/tests/test_recognition_manager_device_config.py +++ b/tests/test_recognition_manager_device_config.py @@ -27,7 +27,9 @@ import pytest from vocalinux.common_types import RecognitionState from vocalinux.speech_recognition.recognition_manager import ( + _STICKY_LOCK_MIN_MEAN_SQUARE, SpeechRecognitionManager, + _downmix_to_mono, _filter_non_speech, _get_supported_channels, _get_supported_sample_rate, @@ -387,6 +389,80 @@ class TestAudioDeviceDetection(unittest.TestCase): assert mock_audio.open.call_args.kwargs.get("channels") == 1 assert any(call.kwargs.get("channels") == 2 for call in mock_audio.open.call_args_list) + def test_open_capture_stream_hda_4ch_prefers_native_channels(self): + """HDA mics reporting 4 capture channels must open 4ch-first (#813). + + Built-in analog HDA devices often expose 4ch (front/rear). Opening + at 2ch can succeed then abort in PortAudio CleanUpStream with + ``free(): corrupted unsorted chunks``. + """ + mock_audio = MagicMock() + mock_stream = MagicMock() + mock_audio.open.return_value = mock_stream + mock_audio.get_device_info_by_index.return_value = { + "name": "HDA Intel PCH Analog", + "defaultSampleRate": 48000, + "maxInputChannels": 4, + } + mock_pyaudio = MagicMock(paInt16=8) + + with patch.dict("sys.modules", {"pyaudio": mock_pyaudio}): + channels, rate, stream = _open_capture_stream(mock_audio, 0) + + assert channels == 4 + assert rate == 48000 + assert stream is mock_stream + assert mock_audio.open.call_count == 1 + assert mock_audio.open.call_args.kwargs.get("channels") == 4 + + def test_open_capture_stream_pulse_32ch_opens_stereo_never_native(self): + """Pulse devices reporting 32 channels must try 2ch first, never 32.""" + mock_audio = MagicMock() + mock_stream = MagicMock() + mock_audio.open.return_value = mock_stream + mock_audio.get_device_info_by_index.return_value = { + "name": "pulse", + "defaultSampleRate": 48000, + "maxInputChannels": 32, + } + mock_pyaudio = MagicMock(paInt16=8) + + with patch.dict("sys.modules", {"pyaudio": mock_pyaudio}): + channels, rate, stream = _open_capture_stream(mock_audio, 0) + + assert channels == 2 + assert rate == 48000 + assert stream is mock_stream + assert mock_audio.open.call_args.kwargs.get("channels") == 2 + assert all(call.kwargs.get("channels") != 32 for call in mock_audio.open.call_args_list) + + def test_open_capture_stream_hda_4ch_falls_back_to_stereo(self): + """If native 4ch open fails, still try 2ch on a 4ch-reporting device.""" + mock_audio = MagicMock() + mock_stream = MagicMock() + + def open_side_effect(**kwargs): + if kwargs.get("channels") == 4: + raise IOError("[Errno -9998] Invalid number of channels") + return mock_stream + + mock_audio.open.side_effect = open_side_effect + mock_audio.get_device_info_by_index.return_value = { + "name": "HDA Intel PCH Analog", + "defaultSampleRate": 48000, + "maxInputChannels": 4, + } + mock_pyaudio = MagicMock(paInt16=8) + + with patch.dict("sys.modules", {"pyaudio": mock_pyaudio}): + channels, rate, stream = _open_capture_stream(mock_audio, 0) + + assert channels == 2 + assert rate == 48000 + assert stream is mock_stream + assert mock_audio.open.call_args.kwargs.get("channels") == 2 + assert any(call.kwargs.get("channels") == 4 for call in mock_audio.open.call_args_list) + def test_open_capture_stream_mono_device_never_probes_stereo(self): """Mono-only devices must never be opened with 2 channels. @@ -727,6 +803,142 @@ class TestAudioDeviceDetection(unittest.TestCase): assert rate == 16000 # Default fallback +class TestDownmixToMono(unittest.TestCase): + """Energy-aware loudest-channel downmix policy for HDA capture (#813 / PR #829).""" + + @classmethod + def setUpClass(cls): + # Other test modules may have left sys.modules["numpy"] as MagicMock. + if isinstance(sys.modules.get("numpy"), MagicMock): + real = getattr(sys, "_vocalinux_real_numpy", None) + if real is not None: + sys.modules["numpy"] = real + else: + del sys.modules["numpy"] + import numpy as np + + cls.np = np + + def test_mono_passthrough_unchanged(self): + np = self.np + audio = np.array([100, 200, 300], dtype=np.int16) + out, sticky = _downmix_to_mono(audio, 1) + np.testing.assert_array_equal(out, audio) + assert out is audio + assert sticky is None + + def test_stereo_mean_of_both_channels(self): + np = self.np + audio = np.array([100, 200], dtype=np.int16) + out, sticky = _downmix_to_mono(audio, 2) + np.testing.assert_array_equal(out, np.array([150], dtype=np.int16)) + assert sticky is None + + def test_quad_loudest_channel_does_not_halve_speech_level(self): + """4ch (speech, speech, silence, silence) keeps the loudest channel at full level.""" + np = self.np + audio = np.array([1000, 1000, 0, 0], dtype=np.int16) + out, sticky = _downmix_to_mono(audio, 4) + np.testing.assert_array_equal(out, np.array([1000], dtype=np.int16)) + # Averaging all 4 channels would have produced 500. + assert out[0] != 500 + assert sticky == 0 + + def test_quad_recovers_speech_on_channel_2(self): + """4ch (silence, silence, speech, silence) selects the loud rear channel.""" + np = self.np + audio = np.array([0, 0, 1000, 0], dtype=np.int16) + out, sticky = _downmix_to_mono(audio, 4) + np.testing.assert_array_equal(out, np.array([1000], dtype=np.int16)) + assert sticky == 2 + + def test_quad_recovers_speech_on_channel_3(self): + """4ch (silence, silence, silence, speech) selects the loud rear channel.""" + np = self.np + audio = np.array([0, 0, 0, 1000], dtype=np.int16) + out, sticky = _downmix_to_mono(audio, 4) + np.testing.assert_array_equal(out, np.array([1000], dtype=np.int16)) + assert sticky == 3 + + def test_quad_leftover_truncation(self): + """Incomplete trailing frame is dropped using full N-channel width, then loudest channel.""" + np = self.np + audio = np.array([1000, 1000, 0, 0, 2000, 2000, 0, 0, 99], dtype=np.int16) + out, sticky = _downmix_to_mono(audio, 4) + np.testing.assert_array_equal(out, np.array([1000, 2000], dtype=np.int16)) + assert sticky == 0 + + def test_quad_leftover_truncation_then_loudest_rear_channel(self): + """Truncate leftover at full N-channel width, then pick speech on ch2 (not first pair).""" + np = self.np + audio = np.array([0, 0, 1000, 0, 0, 0, 2000, 0, 99], dtype=np.int16) + out, sticky = _downmix_to_mono(audio, 4) + np.testing.assert_array_equal(out, np.array([1000, 2000], dtype=np.int16)) + assert sticky == 2 + + def test_quad_sticky_channel_ignores_later_noise_burst(self): + """First buffer speech on ch2 sticks; later louder noise on ch0 still returns ch2.""" + np = self.np + first = np.array([0, 0, 1000, 0], dtype=np.int16) + out1, sticky = _downmix_to_mono(first, 4) + np.testing.assert_array_equal(out1, np.array([1000], dtype=np.int16)) + assert sticky == 2 + # Noise burst on ch0 is louder than sticky mic speech on ch2. + second = np.array([5000, 0, 800, 0], dtype=np.int16) + out2, sticky2 = _downmix_to_mono(second, 4, sticky) + np.testing.assert_array_equal(out2, np.array([800], dtype=np.int16)) + assert sticky2 == 2 + + def test_quad_front_pair_full_level_with_sticky(self): + """Front-pair speech still comes through at full level once sticky is set.""" + np = self.np + first = np.array([1200, 1100, 0, 0], dtype=np.int16) + out1, sticky = _downmix_to_mono(first, 4) + np.testing.assert_array_equal(out1, np.array([1200], dtype=np.int16)) + assert sticky == 0 + second = np.array([900, 850, 0, 0], dtype=np.int16) + out2, sticky2 = _downmix_to_mono(second, 4, sticky) + np.testing.assert_array_equal(out2, np.array([900], dtype=np.int16)) + assert sticky2 == 0 + + def test_quad_silent_first_buffer_does_not_lock_sticky(self): + """Idle/ambient first buffer must not permanently pin ch0 before speech.""" + np = self.np + # ch0 has tiny ambient (mean-square << lock floor); others silent. + ambient = 20 # MS = 400 << _STICKY_LOCK_MIN_MEAN_SQUARE (10000) + assert ambient * ambient < _STICKY_LOCK_MIN_MEAN_SQUARE + first = np.array([ambient, 0, 0, 0], dtype=np.int16) + out1, sticky = _downmix_to_mono(first, 4) + np.testing.assert_array_equal(out1, np.array([ambient], dtype=np.int16)) + assert sticky is None + # Second buffer: real speech on ch2 — lock there, not ch0. + second = np.array([0, 0, 1000, 0], dtype=np.int16) + out2, sticky2 = _downmix_to_mono(second, 4, sticky) + np.testing.assert_array_equal(out2, np.array([1000], dtype=np.int16)) + assert sticky2 == 2 + + def test_quad_near_silence_noise_then_speech_locks_speech_channel(self): + """Noise-floor ch0 then speech on ch2: return speech and pin ch2.""" + np = self.np + # Multi-frame idle: faint noise on ch0 only. + idle = np.array( + [30, 0, 0, 0, 40, 0, 0, 0, 25, 0, 0, 0], + dtype=np.int16, + ) + out1, sticky = _downmix_to_mono(idle, 4) + assert sticky is None + assert out1.tolist() == [30, 40, 25] + speech = np.array([10, 0, 2000, 0, 5, 0, 1800, 0], dtype=np.int16) + out2, sticky2 = _downmix_to_mono(speech, 4, sticky) + np.testing.assert_array_equal(out2, np.array([2000, 1800], dtype=np.int16)) + assert sticky2 == 2 + # Once locked, later louder noise on another channel stays ignored. + burst = np.array([8000, 0, 900, 0], dtype=np.int16) + out3, sticky3 = _downmix_to_mono(burst, 4, sticky2) + np.testing.assert_array_equal(out3, np.array([900], dtype=np.int16)) + assert sticky3 == 2 + + class TestRecordAudioNegotiationFallback(unittest.TestCase): """Cover _record_audio fallback when stream negotiation returns None.""" diff --git a/tests/test_recognition_manager_silero.py b/tests/test_recognition_manager_silero.py index 9c3dc8c..1ba63d9 100644 --- a/tests/test_recognition_manager_silero.py +++ b/tests/test_recognition_manager_silero.py @@ -201,12 +201,8 @@ class TestRecordAudioSileroPath(unittest.TestCase): with ( patch.dict(sys.modules, {"pyaudio": pyaudio_mod, "numpy": np}), patch( - "vocalinux.speech_recognition.recognition_manager._get_supported_channels", - return_value=channels, - ), - patch( - "vocalinux.speech_recognition.recognition_manager._get_supported_sample_rate", - return_value=rate, + "vocalinux.speech_recognition.recognition_manager._open_capture_stream", + return_value=(channels, rate, stream), ), ): self.mgr._record_audio() @@ -271,6 +267,10 @@ class TestRecordAudioSileroPath(unittest.TestCase): self._drive(probs=[0.05] * 12, vad_sensitivity=3, channels=2, chunk_bytes=1024 * 2 * 2) # No assertion on enqueue count -- just verify the path runs cleanly. + def test_quad_capture_is_downmixed_without_error(self): + """CHANNELS=4 -> the N-channel downmix branch runs and the loop completes.""" + self._drive(probs=[0.05] * 12, vad_sensitivity=3, channels=4, chunk_bytes=1024 * 4 * 2) + def test_resample_branch_runs_at_48khz(self): """rate=48000 -> the resample branch in _record_audio runs.""" self._drive(probs=[0.05] * 12, vad_sensitivity=3, rate=48000)