diff --git a/README.md b/README.md index ec4cef1..acc93f6 100644 --- a/README.md +++ b/README.md @@ -446,3 +446,18 @@ release manifest against those same pins. The manifest download participates in Hugging Face's normal model-download accounting. Cached recognition works offline and does not send audio or usage events. The model occupies about 641 MiB; its CC BY-SA 4.0 weights license and NVIDIA attribution are included. + +### Optional Underdog Bark 3.0 models + +Settings → Speech Model → Parakeet also offers two builds of +[Underdog Bark 3.0](https://huggingface.co/ConwayResearch/Underdog-Bark-3.0), an +English fine-tune of Parakeet TDT 0.6B v3 (CC BY 4.0): `bark-3.0` (int8, 653 MiB) +and `bark-3.0-int4` (4-bit, 379 MiB, about half the memory, slower). Both come +from a pinned revision of +[velddev/Underdog-Bark-3.0-sherpa-onnx](https://huggingface.co/velddev/Underdog-Bark-3.0-sherpa-onnx), +an unofficial sherpa-onnx conversion not made or endorsed by Conway Research. +They run on the existing sherpa-onnx recognizer, every file is checked against +its bundled SHA-256 pin, and the downloads include the license and attribution +notice. +Choose one for English dictation of conversational speech, `bark-3.0-int4` when +memory is tight; for read or multilingual speech keep `v3-european`. diff --git a/src/vocalinux/speech_recognition/recognition_manager.py b/src/vocalinux/speech_recognition/recognition_manager.py index 626c4a3..f4eed71 100644 --- a/src/vocalinux/speech_recognition/recognition_manager.py +++ b/src/vocalinux/speech_recognition/recognition_manager.py @@ -2336,10 +2336,11 @@ class SpeechRecognitionManager: # reconfigure()/reinitialize_after_resume() already hold it when they # call this, so acquiring it would deadlock the caller. self.model = None + precision = parakeet.PARAKEET_MODEL_INFO[self.model_size].get("precision", "int8") self.model = sherpa_onnx.OfflineRecognizer.from_transducer( - encoder=os.path.join(model_dir, "encoder.int8.onnx"), - decoder=os.path.join(model_dir, "decoder.int8.onnx"), - joiner=os.path.join(model_dir, "joiner.int8.onnx"), + encoder=os.path.join(model_dir, f"encoder.{precision}.onnx"), + decoder=os.path.join(model_dir, f"decoder.{precision}.onnx"), + joiner=os.path.join(model_dir, f"joiner.{precision}.onnx"), tokens=os.path.join(model_dir, "tokens.txt"), model_type="nemo_transducer", feature_dim=128, @@ -2377,7 +2378,7 @@ class SpeechRecognitionManager: stream = self.model.create_stream() stream.accept_waveform(16000, audio_float) self.model.decode_stream(stream) - return cast(str, stream.result.text.strip()) + return " ".join(stream.result.text.replace("", " ").split()) except (RuntimeError, OSError, ValueError) as e: audio_info = ( diff --git a/src/vocalinux/utils/model_checksums.txt b/src/vocalinux/utils/model_checksums.txt index 9e2ac99..459e4d2 100644 --- a/src/vocalinux/utils/model_checksums.txt +++ b/src/vocalinux/utils/model_checksums.txt @@ -106,6 +106,18 @@ parakeet-v2-english-encoder.int8.onnx sha256 a32b12d17bbbc309d0686f parakeet-v2-english-decoder.int8.onnx sha256 b6bb64963457237b900e496ee9994b59294526439fbcc1fecf705b31a15c6b4e 7257753 parakeet-v2-english-joiner.int8.onnx sha256 7946164367946e7f9f29a122407c3252b680dbae9a51343eb2488d057c3c43d2 1739080 parakeet-v2-english-tokens.txt sha256 ec182b70dd42113aff6c5372c75cac58c952443eb22322f57bbd7f53977d497d 9384 +parakeet-bark-3.0-encoder.int8.onnx sha256 d7e8f38d29aa486777eb7d505f2c7f35a0d5914760d622ed328e009cf18b9429 630748293 +parakeet-bark-3.0-decoder.int8.onnx sha256 89a081f0cf1500ec64e8a45f24f0bc10d1224aa334191e7f7e6f3226365b06b8 47658545 +parakeet-bark-3.0-joiner.int8.onnx sha256 b8fe34824c5d2ec077489e4b4f66907cb47a971e1d34fe96915e9e5427df3c89 6123465 +parakeet-bark-3.0-tokens.txt sha256 d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d 93939 +parakeet-bark-3.0-LICENSE sha256 9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411 18657 +parakeet-bark-3.0-NOTICE sha256 7760825e72004ccf97f7a8b63a75e2890105a7cae8ca875b70d4df78cff9abcd 603 +parakeet-bark-3.0-int4-encoder.int4.onnx sha256 622e9383cc74d13a1142c29f5f9c4098a7cc1f2f9939e5426ed9c6ac4a71c61f 345679872 +parakeet-bark-3.0-int4-decoder.int4.onnx sha256 91cf512d5874383c94ed3c6ff2667f50553737d2cffeb995546f838e2ca059cd 47466590 +parakeet-bark-3.0-int4-joiner.int4.onnx sha256 48f9314b85f48f79d2576cd3ce7ed49e5807b518af2511f2cd56ef1e8dad8e32 3356957 +parakeet-bark-3.0-int4-tokens.txt sha256 d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d 93939 +parakeet-bark-3.0-int4-LICENSE sha256 9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411 18657 +parakeet-bark-3.0-int4-NOTICE sha256 7760825e72004ccf97f7a8b63a75e2890105a7cae8ca875b70d4df78cff9abcd 603 faster-whisper-tiny-config.json sha256 a73a28cdfe1c43ccc7202fa333d1f89c202477271407ae9a7f19afa52039cac8 2249 faster-whisper-tiny-model.bin sha256 dcb76c6586fc06cbdac6dd21f14cfd129cc4cdd9dce19bf4ffa62e59cbe6e6d1 75538270 faster-whisper-tiny-tokenizer.json sha256 fb7b63191e9bb045082c79fd742a3106a12c99513ab30df4a0d47fa6cb6fd0ab 2203239 diff --git a/src/vocalinux/utils/parakeet_model_info.py b/src/vocalinux/utils/parakeet_model_info.py index 617bedd..40e0035 100644 --- a/src/vocalinux/utils/parakeet_model_info.py +++ b/src/vocalinux/utils/parakeet_model_info.py @@ -53,9 +53,38 @@ PARAKEET_MODEL_INFO: Dict[str, Dict[str, Any]] = { "size_mb": 630, "desc": "Parakeet TDT 0.6B v2 (int8), English", }, + "bark-3.0": { + "repo": "velddev/Underdog-Bark-3.0-sherpa-onnx", + "revision": "3ae3af1a7ae2cb3a47f23498f83bb67ecb30a198", + "files": [ + "encoder.int8.onnx", + "decoder.int8.onnx", + "joiner.int8.onnx", + "tokens.txt", + "LICENSE", + "NOTICE", + ], + "size_mb": 653, + "desc": "Underdog Bark 3.0 (int8), English; CC BY 4.0", + }, + "bark-3.0-int4": { + "repo": "velddev/Underdog-Bark-3.0-sherpa-onnx", + "revision": "3ae3af1a7ae2cb3a47f23498f83bb67ecb30a198", + "precision": "int4", + "files": [ + "encoder.int4.onnx", + "decoder.int4.onnx", + "joiner.int4.onnx", + "tokens.txt", + "LICENSE", + "NOTICE", + ], + "size_mb": 379, + "desc": "Underdog Bark 3.0 (4-bit), English, half the memory; CC BY 4.0", + }, } -MODEL_SIZES = ["v3-european", "v2-english", "orukeet-v0.1.0"] +MODEL_SIZES = ["v3-european", "v2-english", "orukeet-v0.1.0", "bark-3.0", "bark-3.0-int4"] # Model offered when the engine has nothing downloaded yet. RECOMMENDED_MODEL = "v3-european" diff --git a/tests/test_bark.py b/tests/test_bark.py new file mode 100644 index 0000000..dcacaf5 --- /dev/null +++ b/tests/test_bark.py @@ -0,0 +1,55 @@ +"""Optional Underdog Bark 3.0 catalog entries, their file names and output.""" + +import sys +import threading +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from vocalinux.speech_recognition.recognition_manager import SpeechRecognitionManager +from vocalinux.utils import parakeet_model_info as models +from vocalinux.utils.model_checksums import expected_for + +REPO = "velddev/Underdog-Bark-3.0-sherpa-onnx" + + +@pytest.mark.parametrize("model, precision", [("bark-3.0", "int8"), ("bark-3.0-int4", "int4")]) +def test_bark_is_optional_pinned_and_keeps_its_license_files(model: str, precision: str) -> None: + """Both Bark builds are offered but not recommended, and every file they download is pinned.""" + assert models.RECOMMENDED_MODEL == "v3-european" + assert model in models.MODEL_SIZES + assert {"LICENSE", "NOTICE", f"encoder.{precision}.onnx"} <= set(models.model_files(model)) + for name in models.model_files(model): + assert expected_for(models.manifest_key(model, name)) is not None + assert models.get_model_file_url(model, "NOTICE").startswith(f"https://huggingface.co/{REPO}/") + + +def test_int4_build_loads_its_own_file_names( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The 4-bit entry's precision key makes the loader open the *.int4.onnx files.""" + manager = SpeechRecognitionManager.__new__(SpeechRecognitionManager) + manager.model_size = "bark-3.0-int4" + manager._defer_download = False + sherpa = MagicMock() + monkeypatch.setitem(sys.modules, "sherpa_onnx", sherpa) + monkeypatch.setattr(models, "get_model_path", lambda _: str(tmp_path)) + monkeypatch.setattr(models, "is_model_downloaded", lambda _: True) + monkeypatch.setattr(manager, "_parakeet_model_is_verified", lambda *_: True) + manager._init_parakeet() + kwargs = sherpa.OfflineRecognizer.from_transducer.call_args.kwargs + assert [kwargs[part].rsplit("/", 1)[1] for part in ("encoder", "decoder", "joiner")] == [ + "encoder.int4.onnx", + "decoder.int4.onnx", + "joiner.int4.onnx", + ] + + +def test_unknown_word_token_is_dropped_from_the_transcript() -> None: + """Parakeet transcripts drop and collapse the spaces it leaves.""" + manager = SpeechRecognitionManager.__new__(SpeechRecognitionManager) + manager._model_lock = threading.Lock() + manager.model = MagicMock() + manager.model.create_stream.return_value.result.text = " like a shampooer,Next man." + assert manager._transcribe_with_parakeet([b"\x00\x00" * 160]) == "like a shampooer, Next man."