feat(parakeet): add optional Underdog Bark 3.0 models, int8 and 4-bit (#1031)
* feat(parakeet): add optional Underdog Bark 3.0 int8 model Adds bark-3.0 to the Parakeet model picker: Underdog Bark 3.0, an English fine-tune of Parakeet TDT 0.6B v3, as the int8 sherpa-onnx conversion published at velddev/Underdog-Bark-3.0-sherpa-onnx. It runs on the existing sherpa-onnx recognizer, pinned to a revision with every file checked against model_checksums.txt, like Orukeet (#840). The conversion writes the unknown-word token where a quote mark was, so the Parakeet transcript drops <unk>; the other models never emit it. * docs(parakeet): say when to pick Bark 3.0; loosen its catalog test The README now says Bark suits English dictation of conversational speech and v3-european stays the choice for read or multilingual speech. The test checks that bark-3.0 is offered rather than that it is the last catalog entry, so a later model does not break it. * test(parakeet): add docstrings to the Bark tests * feat(parakeet): add optional 4-bit Underdog Bark 3.0 build Adds bark-3.0-int4: the same Bark 3.0 weights quantized to 4-bit with onnxruntime's HQQ quantizer, for machines short on memory. It needs about half the RAM of the int8 build (0.64 GB against 1.28 GB peak) and a 379 MiB download, at two to four times the CPU time (about 2x on Intel, 3.7x on AMD by the publisher's measurement). The files are published in velddev/Underdog-Bark-3.0-sherpa-onnx (Hugging Face PR #1), and both Bark entries now pin its revision 3ae3af1, which holds both builds; the int8 files are unchanged there. Catalog entries gain an optional "precision" key, so the loader opens encoder.int4.onnx and friends instead of assuming int8; every existing model keeps the int8 default. The README section now covers both builds in a few lines. * test(parakeet): type the Bark test signatures * docs(parakeet): name the Bark sherpa-onnx source and its pins The README now says where both Bark builds download from (a pinned revision of velddev/Underdog-Bark-3.0-sherpa-onnx), that it is an unofficial conversion, and that each file is checked against its bundled SHA-256 pin, as the Orukeet section does. --------- Co-authored-by: Jatin <jatinkrmalik@gmail.com>
This commit is contained in:
AlexThunder1989
@@ -446,3 +446,18 @@ release manifest against those same pins. The manifest download participates
|
||||
in Hugging Face's normal model-download accounting. Cached recognition works
|
||||
offline and does not send audio or usage events. The model occupies about
|
||||
641 MiB; its CC BY-SA 4.0 weights license and NVIDIA attribution are included.
|
||||
|
||||
### Optional Underdog Bark 3.0 models
|
||||
|
||||
Settings → Speech Model → Parakeet also offers two builds of
|
||||
[Underdog Bark 3.0](https://huggingface.co/ConwayResearch/Underdog-Bark-3.0), an
|
||||
English fine-tune of Parakeet TDT 0.6B v3 (CC BY 4.0): `bark-3.0` (int8, 653 MiB)
|
||||
and `bark-3.0-int4` (4-bit, 379 MiB, about half the memory, slower). Both come
|
||||
from a pinned revision of
|
||||
[velddev/Underdog-Bark-3.0-sherpa-onnx](https://huggingface.co/velddev/Underdog-Bark-3.0-sherpa-onnx),
|
||||
an unofficial sherpa-onnx conversion not made or endorsed by Conway Research.
|
||||
They run on the existing sherpa-onnx recognizer, every file is checked against
|
||||
its bundled SHA-256 pin, and the downloads include the license and attribution
|
||||
notice.
|
||||
Choose one for English dictation of conversational speech, `bark-3.0-int4` when
|
||||
memory is tight; for read or multilingual speech keep `v3-european`.
|
||||
|
||||
@@ -2336,10 +2336,11 @@ class SpeechRecognitionManager:
|
||||
# reconfigure()/reinitialize_after_resume() already hold it when they
|
||||
# call this, so acquiring it would deadlock the caller.
|
||||
self.model = None
|
||||
precision = parakeet.PARAKEET_MODEL_INFO[self.model_size].get("precision", "int8")
|
||||
self.model = sherpa_onnx.OfflineRecognizer.from_transducer(
|
||||
encoder=os.path.join(model_dir, "encoder.int8.onnx"),
|
||||
decoder=os.path.join(model_dir, "decoder.int8.onnx"),
|
||||
joiner=os.path.join(model_dir, "joiner.int8.onnx"),
|
||||
encoder=os.path.join(model_dir, f"encoder.{precision}.onnx"),
|
||||
decoder=os.path.join(model_dir, f"decoder.{precision}.onnx"),
|
||||
joiner=os.path.join(model_dir, f"joiner.{precision}.onnx"),
|
||||
tokens=os.path.join(model_dir, "tokens.txt"),
|
||||
model_type="nemo_transducer",
|
||||
feature_dim=128,
|
||||
@@ -2377,7 +2378,7 @@ class SpeechRecognitionManager:
|
||||
stream = self.model.create_stream()
|
||||
stream.accept_waveform(16000, audio_float)
|
||||
self.model.decode_stream(stream)
|
||||
return cast(str, stream.result.text.strip())
|
||||
return " ".join(stream.result.text.replace("<unk>", " ").split())
|
||||
|
||||
except (RuntimeError, OSError, ValueError) as e:
|
||||
audio_info = (
|
||||
|
||||
@@ -106,6 +106,18 @@ parakeet-v2-english-encoder.int8.onnx sha256 a32b12d17bbbc309d0686f
|
||||
parakeet-v2-english-decoder.int8.onnx sha256 b6bb64963457237b900e496ee9994b59294526439fbcc1fecf705b31a15c6b4e 7257753
|
||||
parakeet-v2-english-joiner.int8.onnx sha256 7946164367946e7f9f29a122407c3252b680dbae9a51343eb2488d057c3c43d2 1739080
|
||||
parakeet-v2-english-tokens.txt sha256 ec182b70dd42113aff6c5372c75cac58c952443eb22322f57bbd7f53977d497d 9384
|
||||
parakeet-bark-3.0-encoder.int8.onnx sha256 d7e8f38d29aa486777eb7d505f2c7f35a0d5914760d622ed328e009cf18b9429 630748293
|
||||
parakeet-bark-3.0-decoder.int8.onnx sha256 89a081f0cf1500ec64e8a45f24f0bc10d1224aa334191e7f7e6f3226365b06b8 47658545
|
||||
parakeet-bark-3.0-joiner.int8.onnx sha256 b8fe34824c5d2ec077489e4b4f66907cb47a971e1d34fe96915e9e5427df3c89 6123465
|
||||
parakeet-bark-3.0-tokens.txt sha256 d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d 93939
|
||||
parakeet-bark-3.0-LICENSE sha256 9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411 18657
|
||||
parakeet-bark-3.0-NOTICE sha256 7760825e72004ccf97f7a8b63a75e2890105a7cae8ca875b70d4df78cff9abcd 603
|
||||
parakeet-bark-3.0-int4-encoder.int4.onnx sha256 622e9383cc74d13a1142c29f5f9c4098a7cc1f2f9939e5426ed9c6ac4a71c61f 345679872
|
||||
parakeet-bark-3.0-int4-decoder.int4.onnx sha256 91cf512d5874383c94ed3c6ff2667f50553737d2cffeb995546f838e2ca059cd 47466590
|
||||
parakeet-bark-3.0-int4-joiner.int4.onnx sha256 48f9314b85f48f79d2576cd3ce7ed49e5807b518af2511f2cd56ef1e8dad8e32 3356957
|
||||
parakeet-bark-3.0-int4-tokens.txt sha256 d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d 93939
|
||||
parakeet-bark-3.0-int4-LICENSE sha256 9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411 18657
|
||||
parakeet-bark-3.0-int4-NOTICE sha256 7760825e72004ccf97f7a8b63a75e2890105a7cae8ca875b70d4df78cff9abcd 603
|
||||
faster-whisper-tiny-config.json sha256 a73a28cdfe1c43ccc7202fa333d1f89c202477271407ae9a7f19afa52039cac8 2249
|
||||
faster-whisper-tiny-model.bin sha256 dcb76c6586fc06cbdac6dd21f14cfd129cc4cdd9dce19bf4ffa62e59cbe6e6d1 75538270
|
||||
faster-whisper-tiny-tokenizer.json sha256 fb7b63191e9bb045082c79fd742a3106a12c99513ab30df4a0d47fa6cb6fd0ab 2203239
|
||||
|
||||
@@ -53,9 +53,38 @@ PARAKEET_MODEL_INFO: Dict[str, Dict[str, Any]] = {
|
||||
"size_mb": 630,
|
||||
"desc": "Parakeet TDT 0.6B v2 (int8), English",
|
||||
},
|
||||
"bark-3.0": {
|
||||
"repo": "velddev/Underdog-Bark-3.0-sherpa-onnx",
|
||||
"revision": "3ae3af1a7ae2cb3a47f23498f83bb67ecb30a198",
|
||||
"files": [
|
||||
"encoder.int8.onnx",
|
||||
"decoder.int8.onnx",
|
||||
"joiner.int8.onnx",
|
||||
"tokens.txt",
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
],
|
||||
"size_mb": 653,
|
||||
"desc": "Underdog Bark 3.0 (int8), English; CC BY 4.0",
|
||||
},
|
||||
"bark-3.0-int4": {
|
||||
"repo": "velddev/Underdog-Bark-3.0-sherpa-onnx",
|
||||
"revision": "3ae3af1a7ae2cb3a47f23498f83bb67ecb30a198",
|
||||
"precision": "int4",
|
||||
"files": [
|
||||
"encoder.int4.onnx",
|
||||
"decoder.int4.onnx",
|
||||
"joiner.int4.onnx",
|
||||
"tokens.txt",
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
],
|
||||
"size_mb": 379,
|
||||
"desc": "Underdog Bark 3.0 (4-bit), English, half the memory; CC BY 4.0",
|
||||
},
|
||||
}
|
||||
|
||||
MODEL_SIZES = ["v3-european", "v2-english", "orukeet-v0.1.0"]
|
||||
MODEL_SIZES = ["v3-european", "v2-english", "orukeet-v0.1.0", "bark-3.0", "bark-3.0-int4"]
|
||||
|
||||
# Model offered when the engine has nothing downloaded yet.
|
||||
RECOMMENDED_MODEL = "v3-european"
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Optional Underdog Bark 3.0 catalog entries, their file names and <unk> output."""
|
||||
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vocalinux.speech_recognition.recognition_manager import SpeechRecognitionManager
|
||||
from vocalinux.utils import parakeet_model_info as models
|
||||
from vocalinux.utils.model_checksums import expected_for
|
||||
|
||||
REPO = "velddev/Underdog-Bark-3.0-sherpa-onnx"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model, precision", [("bark-3.0", "int8"), ("bark-3.0-int4", "int4")])
|
||||
def test_bark_is_optional_pinned_and_keeps_its_license_files(model: str, precision: str) -> None:
|
||||
"""Both Bark builds are offered but not recommended, and every file they download is pinned."""
|
||||
assert models.RECOMMENDED_MODEL == "v3-european"
|
||||
assert model in models.MODEL_SIZES
|
||||
assert {"LICENSE", "NOTICE", f"encoder.{precision}.onnx"} <= set(models.model_files(model))
|
||||
for name in models.model_files(model):
|
||||
assert expected_for(models.manifest_key(model, name)) is not None
|
||||
assert models.get_model_file_url(model, "NOTICE").startswith(f"https://huggingface.co/{REPO}/")
|
||||
|
||||
|
||||
def test_int4_build_loads_its_own_file_names(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""The 4-bit entry's precision key makes the loader open the *.int4.onnx files."""
|
||||
manager = SpeechRecognitionManager.__new__(SpeechRecognitionManager)
|
||||
manager.model_size = "bark-3.0-int4"
|
||||
manager._defer_download = False
|
||||
sherpa = MagicMock()
|
||||
monkeypatch.setitem(sys.modules, "sherpa_onnx", sherpa)
|
||||
monkeypatch.setattr(models, "get_model_path", lambda _: str(tmp_path))
|
||||
monkeypatch.setattr(models, "is_model_downloaded", lambda _: True)
|
||||
monkeypatch.setattr(manager, "_parakeet_model_is_verified", lambda *_: True)
|
||||
manager._init_parakeet()
|
||||
kwargs = sherpa.OfflineRecognizer.from_transducer.call_args.kwargs
|
||||
assert [kwargs[part].rsplit("/", 1)[1] for part in ("encoder", "decoder", "joiner")] == [
|
||||
"encoder.int4.onnx",
|
||||
"decoder.int4.onnx",
|
||||
"joiner.int4.onnx",
|
||||
]
|
||||
|
||||
|
||||
def test_unknown_word_token_is_dropped_from_the_transcript() -> None:
|
||||
"""Parakeet transcripts drop <unk> and collapse the spaces it leaves."""
|
||||
manager = SpeechRecognitionManager.__new__(SpeechRecognitionManager)
|
||||
manager._model_lock = threading.Lock()
|
||||
manager.model = MagicMock()
|
||||
manager.model.create_stream.return_value.result.text = " like a shampooer,<unk>Next man.<unk>"
|
||||
assert manager._transcribe_with_parakeet([b"\x00\x00" * 160]) == "like a shampooer, Next man."
|
||||
Reference in New Issue
Block a user