fix(audio): open 3-8ch HDA mics at native layout (#813) (#829)

* fix(audio): open 3-8ch HDA mics at native layout (#813)

HDA analog capture often reports 4 channels. Opening below that native
count can abort PortAudio after read with free(): corrupted unsorted
chunks (same class of bug as #666/#673).

Try the reported channel count first when it is between 3 and 8, then
2ch and 1ch. Pulse devices that advertise 32/128 channels still open
stereo-first. Downmix N channels to mono so engines keep seeing 1ch.

Virtual-device filtering for pipewire/default (#629) is unchanged.

* fix(audio): downmix HDA capture from first channel pair

Averaging all N HDA channels halves speech when only the front pair is live.
Keep stereo mean-of-both, and for 3+ channels average channels 0 and 1.

* fix(audio): pick loudest channel for N-ch downmix

First-pair-only downmix silenced mics on ch2+. For 3+ channels, select
the loudest channel by per-buffer mean-square so speech stays at full
level whether it lands on the front or rear pair. Stereo still averages
both channels.

* fix(audio): sticky N-ch downmix channel for open stream

Pin the loudest-channel pick for the life of a capture stream so a
noise burst on another input cannot switch the mic mid-utterance.

* fix(audio): gate sticky downmix lock on speech energy

Don't pin the N-ch sticky channel from a silent/ambient first buffer;
only lock once the loudest channel clears a documented mean-square floor.

* style: isort sticky-lock import in device config tests
This commit is contained in:
Jatin K Malik
2026-09-14 00:21:17 -07:00
committed by GitHub
parent 5e8a08fbb8
commit 092dde105e
3 changed files with 332 additions and 26 deletions
@@ -17,7 +17,10 @@ import sys
import threading
import time
from pathlib import Path
from typing import Callable, Optional
from typing import TYPE_CHECKING, Callable, Optional
if TYPE_CHECKING:
import numpy as np
from ..common_types import RecognitionState
from ..ui.audio_feedback import play_error_sound, play_start_sound, play_stop_sound
@@ -518,10 +521,10 @@ def _open_capture_stream(audio, device_index: Optional[int] = None) -> tuple[int
heap corruption (see GitHub issue #567).
This function opens PortAudio exactly once per capture session: candidate
formats are tried in order (device default rate first, mono before stereo,
stereo skipped entirely for mono-only devices) and the FIRST successfully
opened stream is returned to the caller for actual capture — never closed
and reopened.
formats are tried in order (device default rate first, native channel count
first for 2–8ch devices, stereo skipped entirely for mono-only devices)
and the FIRST successfully opened stream is returned to the caller for
actual capture — never closed and reopened.
Args:
audio: PyAudio instance
@@ -550,18 +553,25 @@ def _open_capture_stream(audio, device_index: Optional[int] = None) -> tuple[int
if rate not in rates_to_try:
rates_to_try.append(rate)
# Never probe stereo on a device that reports a single input channel
# (opening with more channels than supported is itself a known
# PortAudio/ALSA corruption trigger). Stereo-capable devices must be
# opened at their native layout first: Intel SOF DMICs (Raptor Lake
# "Digital Microphone", etc.) often only support 2ch at the PCM, and
# PortAudio can abort with heap corruption if open() accepts a converted
# 1ch stream and the callback then overruns (#666).
# Never probe extra channels on a device that reports a single input
# channel (opening with more than supported is itself a known
# PortAudio/ALSA corruption trigger). Devices must be opened at their
# native layout first: Intel SOF DMICs often only support 2ch at the
# PCM (#666), and HDA analog mics commonly expose 4 capture channels
# even when only the first pair is a mic (#813). Opening below native
# channel count can succeed then abort in PortAudio CleanUpStream with
# ``free(): corrupted unsorted chunks``. Pulse virtual devices often
# report 32 or 128 channels; those are not native PCM layouts, so we
# still try 2ch then 1ch.
reported_channels = int(device_info.get("maxInputChannels", 0) or 0)
if reported_channels == 1:
channel_options = [1]
elif reported_channels >= 2:
elif reported_channels == 2:
channel_options = [2, 1]
elif 2 < reported_channels <= 8:
channel_options = [reported_channels, 2, 1] # HDA 4ch (#813)
elif reported_channels > 8:
channel_options = [2, 1] # Pulse 32/128
else:
channel_options = [1, 2]
@@ -600,6 +610,79 @@ def _open_capture_stream(audio, device_index: Optional[int] = None) -> tuple[int
return 1, 16000, None
# Mean-square energy floor on int16 PCM before locking the N>=3 sticky channel.
# Capture samples are raw int16 (see _record_audio np.frombuffer(..., int16));
# RMS ≈ 100 ≈ -50 dBFS — above idle dither/ambient, well below speech.
_STICKY_LOCK_MIN_MEAN_SQUARE = 10_000.0
def _downmix_to_mono(
audio_array: "np.ndarray",
channels: int,
sticky_channel: Optional[int] = None,
) -> tuple["np.ndarray", Optional[int]]:
"""Downmix interleaved int16 PCM to mono.
Speech recognition engines expect mono audio.
Stereo (2ch) still averages both channels. For 3+ channels, energy-aware
selection is required: the microphone may not be on ch0/ch1 (HDA analog
capture often puts it on ch2/ch3), so keeping only the first stereo pair
can yield silence, while averaging all N attenuates speech when only some
channels are live.
While sticky is unset, each buffer returns the loudest channel by
mean-square energy, but sticky is only *set* when that channel's
mean-square exceeds :data:`_STICKY_LOCK_MIN_MEAN_SQUARE` (speech-gated
lock). That avoids pinning an ambient/noise channel from a silent first
buffer before the mic speaks. Once sticky is set, later buffers reuse it
until cleared on open/reconnect/cleanup so a noise burst on another input
cannot switch the mic mid-utterance.
Args:
audio_array: 1-D int16 samples with interleaved channels.
channels: Number of interleaved channels in *audio_array*.
sticky_channel: Previously selected channel for N>=3 streams, or
``None`` to (re)select by loudest mean-square energy.
Returns:
``(mono, sticky)`` where *mono* is 1-D int16 samples and *sticky* is
the channel index to reuse on the next N>=3 buffer (``None`` for
N<=2, or when N>=3 and no speech-gated lock yet). ``channels <= 1``
is a passthrough. If ``len(audio_array)`` is not divisible by
*channels*, leftover samples are truncated using the full N-channel
frame width before reshape; an empty array after truncation is
returned as-is (sticky unchanged for N>=3 when already set, else
``None``). Stereo averages both channels; N>=3 uses the sticky index
when in range, otherwise the loudest channel by per-buffer
mean-square (ties keep the first index), locking sticky only when
that channel clears the non-silence energy floor.
"""
if channels <= 1:
return audio_array, None
leftover = len(audio_array) % channels
if leftover:
audio_array = audio_array[: len(audio_array) - leftover]
if len(audio_array) == 0:
if channels >= 3 and sticky_channel is not None and 0 <= sticky_channel < channels:
return audio_array, sticky_channel
return audio_array, None
frames = audio_array.reshape(-1, channels)
if channels == 2:
return frames.mean(axis=1).astype(audio_array.dtype), None
if sticky_channel is not None and 0 <= sticky_channel < channels:
selected = int(sticky_channel)
return frames[:, selected].astype(audio_array.dtype), selected
# Cast before squaring so int16 does not overflow.
energy = (frames.astype("float64") ** 2).mean(axis=0)
selected = int(energy.argmax())
mono = frames[:, selected].astype(audio_array.dtype)
# Speech-gate: return loudest mono now, but only pin sticky on real energy.
if float(energy[selected]) >= _STICKY_LOCK_MIN_MEAN_SQUARE:
return mono, selected
return mono, None
def _get_supported_channels(audio, device_index: Optional[int] = None) -> int:
"""
Detect the supported number of channels for the audio device.
@@ -613,7 +696,8 @@ def _get_supported_channels(audio, device_index: Optional[int] = None) -> int:
device_index: The device index to test (None for default)
Returns:
int: Number of channels supported (1 or 2), defaults to 1
int: Negotiated channel count (1, 2, or native 3–8 for HDA),
defaults to 1
"""
channels, _rate, stream = _open_capture_stream(audio, device_index)
_safe_close_stream(stream)
@@ -1087,6 +1171,8 @@ class SpeechRecognitionManager:
self._audio_stream = None
self._pyaudio_instance = None
self._capture_sample_rate = 16000 # Default, updated when device is opened
self._capture_channels = 1 # Default, updated when device is opened
self._capture_downmix_channel = None # Speech-gated sticky N>=3 channel for open stream
# Create models directory if it doesn't exist
os.makedirs(MODELS_DIR, exist_ok=True)
@@ -3218,6 +3304,8 @@ class SpeechRecognitionManager:
CHANNELS, RATE, negotiated_stream = _open_capture_stream(audio, resolved_device_index)
logger.info(f"Using {CHANNELS} channel(s) for recording")
self._capture_sample_rate = RATE
self._capture_channels = CHANNELS
self._capture_downmix_channel = None # New stream: speech-gated sticky unset
logger.info(f"Using sample rate: {RATE}Hz")
try:
@@ -3266,6 +3354,7 @@ class SpeechRecognitionManager:
# Attempt reconnection
if self._attempt_audio_reconnection(audio):
stream = self._audio_stream
CHANNELS = self._capture_channels
else:
play_error_sound()
audio.terminate()
@@ -3307,14 +3396,15 @@ class SpeechRecognitionManager:
data = stream.read(CHUNK, exception_on_overflow=False)
# Convert stereo to mono if necessary
# Convert multi-channel capture to mono if necessary
# Speech recognition engines expect mono (1 channel) audio
if CHANNELS == 2:
if CHANNELS > 1:
audio_array = np.frombuffer(data, dtype=np.int16)
# Reshape to (n_samples, 2) and average channels
stereo_samples = audio_array.reshape(-1, 2)
mono_samples = stereo_samples.mean(axis=1).astype(np.int16)
data = mono_samples.tobytes()
mono, selected = _downmix_to_mono(
audio_array, CHANNELS, self._capture_downmix_channel
)
self._capture_downmix_channel = selected
data = mono.tobytes()
# Resample to 16kHz if capturing at non-16kHz for Vosk/Whisper compatibility
if self._capture_sample_rate != 16000:
@@ -3441,6 +3531,7 @@ class SpeechRecognitionManager:
if self._attempt_audio_reconnection(audio):
logger.info("Audio reconnection successful, continuing recording")
stream = self._audio_stream # Update stream reference
CHANNELS = self._capture_channels
continue # Continue recording with new stream
else:
logger.error("Audio reconnection failed, stopping recording")
@@ -3466,6 +3557,7 @@ class SpeechRecognitionManager:
# Reset audio stream reference and reconnection state
self._audio_stream = None
self._pyaudio_instance = None
self._capture_downmix_channel = None
self._reconnection_attempts = 0
self._last_audio_error_time = 0
@@ -3896,6 +3988,8 @@ class SpeechRecognitionManager:
CHANNELS, RATE, new_stream = _open_capture_stream(audio_instance, resolved_device_index)
logger.debug(f"Reconnecting with {CHANNELS} channel(s)")
self._capture_sample_rate = RATE
self._capture_channels = CHANNELS
self._capture_downmix_channel = None # Reopened stream: clear speech-gated sticky
logger.debug(f"Reconnecting with sample rate: {RATE}Hz")
if new_stream is None:
@@ -27,7 +27,9 @@ import pytest
from vocalinux.common_types import RecognitionState
from vocalinux.speech_recognition.recognition_manager import (
_STICKY_LOCK_MIN_MEAN_SQUARE,
SpeechRecognitionManager,
_downmix_to_mono,
_filter_non_speech,
_get_supported_channels,
_get_supported_sample_rate,
@@ -387,6 +389,80 @@ class TestAudioDeviceDetection(unittest.TestCase):
assert mock_audio.open.call_args.kwargs.get("channels") == 1
assert any(call.kwargs.get("channels") == 2 for call in mock_audio.open.call_args_list)
def test_open_capture_stream_hda_4ch_prefers_native_channels(self):
"""HDA mics reporting 4 capture channels must open 4ch-first (#813).
Built-in analog HDA devices often expose 4ch (front/rear). Opening
at 2ch can succeed then abort in PortAudio CleanUpStream with
``free(): corrupted unsorted chunks``.
"""
mock_audio = MagicMock()
mock_stream = MagicMock()
mock_audio.open.return_value = mock_stream
mock_audio.get_device_info_by_index.return_value = {
"name": "HDA Intel PCH Analog",
"defaultSampleRate": 48000,
"maxInputChannels": 4,
}
mock_pyaudio = MagicMock(paInt16=8)
with patch.dict("sys.modules", {"pyaudio": mock_pyaudio}):
channels, rate, stream = _open_capture_stream(mock_audio, 0)
assert channels == 4
assert rate == 48000
assert stream is mock_stream
assert mock_audio.open.call_count == 1
assert mock_audio.open.call_args.kwargs.get("channels") == 4
def test_open_capture_stream_pulse_32ch_opens_stereo_never_native(self):
"""Pulse devices reporting 32 channels must try 2ch first, never 32."""
mock_audio = MagicMock()
mock_stream = MagicMock()
mock_audio.open.return_value = mock_stream
mock_audio.get_device_info_by_index.return_value = {
"name": "pulse",
"defaultSampleRate": 48000,
"maxInputChannels": 32,
}
mock_pyaudio = MagicMock(paInt16=8)
with patch.dict("sys.modules", {"pyaudio": mock_pyaudio}):
channels, rate, stream = _open_capture_stream(mock_audio, 0)
assert channels == 2
assert rate == 48000
assert stream is mock_stream
assert mock_audio.open.call_args.kwargs.get("channels") == 2
assert all(call.kwargs.get("channels") != 32 for call in mock_audio.open.call_args_list)
def test_open_capture_stream_hda_4ch_falls_back_to_stereo(self):
"""If native 4ch open fails, still try 2ch on a 4ch-reporting device."""
mock_audio = MagicMock()
mock_stream = MagicMock()
def open_side_effect(**kwargs):
if kwargs.get("channels") == 4:
raise IOError("[Errno -9998] Invalid number of channels")
return mock_stream
mock_audio.open.side_effect = open_side_effect
mock_audio.get_device_info_by_index.return_value = {
"name": "HDA Intel PCH Analog",
"defaultSampleRate": 48000,
"maxInputChannels": 4,
}
mock_pyaudio = MagicMock(paInt16=8)
with patch.dict("sys.modules", {"pyaudio": mock_pyaudio}):
channels, rate, stream = _open_capture_stream(mock_audio, 0)
assert channels == 2
assert rate == 48000
assert stream is mock_stream
assert mock_audio.open.call_args.kwargs.get("channels") == 2
assert any(call.kwargs.get("channels") == 4 for call in mock_audio.open.call_args_list)
def test_open_capture_stream_mono_device_never_probes_stereo(self):
"""Mono-only devices must never be opened with 2 channels.
@@ -727,6 +803,142 @@ class TestAudioDeviceDetection(unittest.TestCase):
assert rate == 16000 # Default fallback
class TestDownmixToMono(unittest.TestCase):
"""Energy-aware loudest-channel downmix policy for HDA capture (#813 / PR #829)."""
@classmethod
def setUpClass(cls):
# Other test modules may have left sys.modules["numpy"] as MagicMock.
if isinstance(sys.modules.get("numpy"), MagicMock):
real = getattr(sys, "_vocalinux_real_numpy", None)
if real is not None:
sys.modules["numpy"] = real
else:
del sys.modules["numpy"]
import numpy as np
cls.np = np
def test_mono_passthrough_unchanged(self):
np = self.np
audio = np.array([100, 200, 300], dtype=np.int16)
out, sticky = _downmix_to_mono(audio, 1)
np.testing.assert_array_equal(out, audio)
assert out is audio
assert sticky is None
def test_stereo_mean_of_both_channels(self):
np = self.np
audio = np.array([100, 200], dtype=np.int16)
out, sticky = _downmix_to_mono(audio, 2)
np.testing.assert_array_equal(out, np.array([150], dtype=np.int16))
assert sticky is None
def test_quad_loudest_channel_does_not_halve_speech_level(self):
"""4ch (speech, speech, silence, silence) keeps the loudest channel at full level."""
np = self.np
audio = np.array([1000, 1000, 0, 0], dtype=np.int16)
out, sticky = _downmix_to_mono(audio, 4)
np.testing.assert_array_equal(out, np.array([1000], dtype=np.int16))
# Averaging all 4 channels would have produced 500.
assert out[0] != 500
assert sticky == 0
def test_quad_recovers_speech_on_channel_2(self):
"""4ch (silence, silence, speech, silence) selects the loud rear channel."""
np = self.np
audio = np.array([0, 0, 1000, 0], dtype=np.int16)
out, sticky = _downmix_to_mono(audio, 4)
np.testing.assert_array_equal(out, np.array([1000], dtype=np.int16))
assert sticky == 2
def test_quad_recovers_speech_on_channel_3(self):
"""4ch (silence, silence, silence, speech) selects the loud rear channel."""
np = self.np
audio = np.array([0, 0, 0, 1000], dtype=np.int16)
out, sticky = _downmix_to_mono(audio, 4)
np.testing.assert_array_equal(out, np.array([1000], dtype=np.int16))
assert sticky == 3
def test_quad_leftover_truncation(self):
"""Incomplete trailing frame is dropped using full N-channel width, then loudest channel."""
np = self.np
audio = np.array([1000, 1000, 0, 0, 2000, 2000, 0, 0, 99], dtype=np.int16)
out, sticky = _downmix_to_mono(audio, 4)
np.testing.assert_array_equal(out, np.array([1000, 2000], dtype=np.int16))
assert sticky == 0
def test_quad_leftover_truncation_then_loudest_rear_channel(self):
"""Truncate leftover at full N-channel width, then pick speech on ch2 (not first pair)."""
np = self.np
audio = np.array([0, 0, 1000, 0, 0, 0, 2000, 0, 99], dtype=np.int16)
out, sticky = _downmix_to_mono(audio, 4)
np.testing.assert_array_equal(out, np.array([1000, 2000], dtype=np.int16))
assert sticky == 2
def test_quad_sticky_channel_ignores_later_noise_burst(self):
"""First buffer speech on ch2 sticks; later louder noise on ch0 still returns ch2."""
np = self.np
first = np.array([0, 0, 1000, 0], dtype=np.int16)
out1, sticky = _downmix_to_mono(first, 4)
np.testing.assert_array_equal(out1, np.array([1000], dtype=np.int16))
assert sticky == 2
# Noise burst on ch0 is louder than sticky mic speech on ch2.
second = np.array([5000, 0, 800, 0], dtype=np.int16)
out2, sticky2 = _downmix_to_mono(second, 4, sticky)
np.testing.assert_array_equal(out2, np.array([800], dtype=np.int16))
assert sticky2 == 2
def test_quad_front_pair_full_level_with_sticky(self):
"""Front-pair speech still comes through at full level once sticky is set."""
np = self.np
first = np.array([1200, 1100, 0, 0], dtype=np.int16)
out1, sticky = _downmix_to_mono(first, 4)
np.testing.assert_array_equal(out1, np.array([1200], dtype=np.int16))
assert sticky == 0
second = np.array([900, 850, 0, 0], dtype=np.int16)
out2, sticky2 = _downmix_to_mono(second, 4, sticky)
np.testing.assert_array_equal(out2, np.array([900], dtype=np.int16))
assert sticky2 == 0
def test_quad_silent_first_buffer_does_not_lock_sticky(self):
"""Idle/ambient first buffer must not permanently pin ch0 before speech."""
np = self.np
# ch0 has tiny ambient (mean-square << lock floor); others silent.
ambient = 20 # MS = 400 << _STICKY_LOCK_MIN_MEAN_SQUARE (10000)
assert ambient * ambient < _STICKY_LOCK_MIN_MEAN_SQUARE
first = np.array([ambient, 0, 0, 0], dtype=np.int16)
out1, sticky = _downmix_to_mono(first, 4)
np.testing.assert_array_equal(out1, np.array([ambient], dtype=np.int16))
assert sticky is None
# Second buffer: real speech on ch2 — lock there, not ch0.
second = np.array([0, 0, 1000, 0], dtype=np.int16)
out2, sticky2 = _downmix_to_mono(second, 4, sticky)
np.testing.assert_array_equal(out2, np.array([1000], dtype=np.int16))
assert sticky2 == 2
def test_quad_near_silence_noise_then_speech_locks_speech_channel(self):
"""Noise-floor ch0 then speech on ch2: return speech and pin ch2."""
np = self.np
# Multi-frame idle: faint noise on ch0 only.
idle = np.array(
[30, 0, 0, 0, 40, 0, 0, 0, 25, 0, 0, 0],
dtype=np.int16,
)
out1, sticky = _downmix_to_mono(idle, 4)
assert sticky is None
assert out1.tolist() == [30, 40, 25]
speech = np.array([10, 0, 2000, 0, 5, 0, 1800, 0], dtype=np.int16)
out2, sticky2 = _downmix_to_mono(speech, 4, sticky)
np.testing.assert_array_equal(out2, np.array([2000, 1800], dtype=np.int16))
assert sticky2 == 2
# Once locked, later louder noise on another channel stays ignored.
burst = np.array([8000, 0, 900, 0], dtype=np.int16)
out3, sticky3 = _downmix_to_mono(burst, 4, sticky2)
np.testing.assert_array_equal(out3, np.array([900], dtype=np.int16))
assert sticky3 == 2
class TestRecordAudioNegotiationFallback(unittest.TestCase):
"""Cover _record_audio fallback when stream negotiation returns None."""
+6 -6
View File
@@ -201,12 +201,8 @@ class TestRecordAudioSileroPath(unittest.TestCase):
with (
patch.dict(sys.modules, {"pyaudio": pyaudio_mod, "numpy": np}),
patch(
"vocalinux.speech_recognition.recognition_manager._get_supported_channels",
return_value=channels,
),
patch(
"vocalinux.speech_recognition.recognition_manager._get_supported_sample_rate",
return_value=rate,
"vocalinux.speech_recognition.recognition_manager._open_capture_stream",
return_value=(channels, rate, stream),
),
):
self.mgr._record_audio()
@@ -271,6 +267,10 @@ class TestRecordAudioSileroPath(unittest.TestCase):
self._drive(probs=[0.05] * 12, vad_sensitivity=3, channels=2, chunk_bytes=1024 * 2 * 2)
# No assertion on enqueue count -- just verify the path runs cleanly.
def test_quad_capture_is_downmixed_without_error(self):
"""CHANNELS=4 -> the N-channel downmix branch runs and the loop completes."""
self._drive(probs=[0.05] * 12, vad_sensitivity=3, channels=4, chunk_bytes=1024 * 4 * 2)
def test_resample_branch_runs_at_48khz(self):
"""rate=48000 -> the resample branch in _record_audio runs."""
self._drive(probs=[0.05] * 12, vad_sensitivity=3, rate=48000)