feat(parakeet): add optional Underdog Bark 3.0 models, int8 and 4-bit (#1031)
* feat(parakeet): add optional Underdog Bark 3.0 int8 model Adds bark-3.0 to the Parakeet model picker: Underdog Bark 3.0, an English fine-tune of Parakeet TDT 0.6B v3, as the int8 sherpa-onnx conversion published at velddev/Underdog-Bark-3.0-sherpa-onnx. It runs on the existing sherpa-onnx recognizer, pinned to a revision with every file checked against model_checksums.txt, like Orukeet (#840). The conversion writes the unknown-word token where a quote mark was, so the Parakeet transcript drops <unk>; the other models never emit it. * docs(parakeet): say when to pick Bark 3.0; loosen its catalog test The README now says Bark suits English dictation of conversational speech and v3-european stays the choice for read or multilingual speech. The test checks that bark-3.0 is offered rather than that it is the last catalog entry, so a later model does not break it. * test(parakeet): add docstrings to the Bark tests * feat(parakeet): add optional 4-bit Underdog Bark 3.0 build Adds bark-3.0-int4: the same Bark 3.0 weights quantized to 4-bit with onnxruntime's HQQ quantizer, for machines short on memory. It needs about half the RAM of the int8 build (0.64 GB against 1.28 GB peak) and a 379 MiB download, at two to four times the CPU time (about 2x on Intel, 3.7x on AMD by the publisher's measurement). The files are published in velddev/Underdog-Bark-3.0-sherpa-onnx (Hugging Face PR #1), and both Bark entries now pin its revision 3ae3af1, which holds both builds; the int8 files are unchanged there. Catalog entries gain an optional "precision" key, so the loader opens encoder.int4.onnx and friends instead of assuming int8; every existing model keeps the int8 default. The README section now covers both builds in a few lines. * test(parakeet): type the Bark test signatures * docs(parakeet): name the Bark sherpa-onnx source and its pins The README now says where both Bark builds download from (a pinned revision of velddev/Underdog-Bark-3.0-sherpa-onnx), that it is an unofficial conversion, and that each file is checked against its bundled SHA-256 pin, as the Orukeet section does. --------- Co-authored-by: Jatin <jatinkrmalik@gmail.com>
This commit is contained in:
AlexThunder1989
@@ -2336,10 +2336,11 @@ class SpeechRecognitionManager:
|
||||
# reconfigure()/reinitialize_after_resume() already hold it when they
|
||||
# call this, so acquiring it would deadlock the caller.
|
||||
self.model = None
|
||||
precision = parakeet.PARAKEET_MODEL_INFO[self.model_size].get("precision", "int8")
|
||||
self.model = sherpa_onnx.OfflineRecognizer.from_transducer(
|
||||
encoder=os.path.join(model_dir, "encoder.int8.onnx"),
|
||||
decoder=os.path.join(model_dir, "decoder.int8.onnx"),
|
||||
joiner=os.path.join(model_dir, "joiner.int8.onnx"),
|
||||
encoder=os.path.join(model_dir, f"encoder.{precision}.onnx"),
|
||||
decoder=os.path.join(model_dir, f"decoder.{precision}.onnx"),
|
||||
joiner=os.path.join(model_dir, f"joiner.{precision}.onnx"),
|
||||
tokens=os.path.join(model_dir, "tokens.txt"),
|
||||
model_type="nemo_transducer",
|
||||
feature_dim=128,
|
||||
@@ -2377,7 +2378,7 @@ class SpeechRecognitionManager:
|
||||
stream = self.model.create_stream()
|
||||
stream.accept_waveform(16000, audio_float)
|
||||
self.model.decode_stream(stream)
|
||||
return cast(str, stream.result.text.strip())
|
||||
return " ".join(stream.result.text.replace("<unk>", " ").split())
|
||||
|
||||
except (RuntimeError, OSError, ValueError) as e:
|
||||
audio_info = (
|
||||
|
||||
@@ -106,6 +106,18 @@ parakeet-v2-english-encoder.int8.onnx sha256 a32b12d17bbbc309d0686f
|
||||
parakeet-v2-english-decoder.int8.onnx sha256 b6bb64963457237b900e496ee9994b59294526439fbcc1fecf705b31a15c6b4e 7257753
|
||||
parakeet-v2-english-joiner.int8.onnx sha256 7946164367946e7f9f29a122407c3252b680dbae9a51343eb2488d057c3c43d2 1739080
|
||||
parakeet-v2-english-tokens.txt sha256 ec182b70dd42113aff6c5372c75cac58c952443eb22322f57bbd7f53977d497d 9384
|
||||
parakeet-bark-3.0-encoder.int8.onnx sha256 d7e8f38d29aa486777eb7d505f2c7f35a0d5914760d622ed328e009cf18b9429 630748293
|
||||
parakeet-bark-3.0-decoder.int8.onnx sha256 89a081f0cf1500ec64e8a45f24f0bc10d1224aa334191e7f7e6f3226365b06b8 47658545
|
||||
parakeet-bark-3.0-joiner.int8.onnx sha256 b8fe34824c5d2ec077489e4b4f66907cb47a971e1d34fe96915e9e5427df3c89 6123465
|
||||
parakeet-bark-3.0-tokens.txt sha256 d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d 93939
|
||||
parakeet-bark-3.0-LICENSE sha256 9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411 18657
|
||||
parakeet-bark-3.0-NOTICE sha256 7760825e72004ccf97f7a8b63a75e2890105a7cae8ca875b70d4df78cff9abcd 603
|
||||
parakeet-bark-3.0-int4-encoder.int4.onnx sha256 622e9383cc74d13a1142c29f5f9c4098a7cc1f2f9939e5426ed9c6ac4a71c61f 345679872
|
||||
parakeet-bark-3.0-int4-decoder.int4.onnx sha256 91cf512d5874383c94ed3c6ff2667f50553737d2cffeb995546f838e2ca059cd 47466590
|
||||
parakeet-bark-3.0-int4-joiner.int4.onnx sha256 48f9314b85f48f79d2576cd3ce7ed49e5807b518af2511f2cd56ef1e8dad8e32 3356957
|
||||
parakeet-bark-3.0-int4-tokens.txt sha256 d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d 93939
|
||||
parakeet-bark-3.0-int4-LICENSE sha256 9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411 18657
|
||||
parakeet-bark-3.0-int4-NOTICE sha256 7760825e72004ccf97f7a8b63a75e2890105a7cae8ca875b70d4df78cff9abcd 603
|
||||
faster-whisper-tiny-config.json sha256 a73a28cdfe1c43ccc7202fa333d1f89c202477271407ae9a7f19afa52039cac8 2249
|
||||
faster-whisper-tiny-model.bin sha256 dcb76c6586fc06cbdac6dd21f14cfd129cc4cdd9dce19bf4ffa62e59cbe6e6d1 75538270
|
||||
faster-whisper-tiny-tokenizer.json sha256 fb7b63191e9bb045082c79fd742a3106a12c99513ab30df4a0d47fa6cb6fd0ab 2203239
|
||||
|
||||
@@ -53,9 +53,38 @@ PARAKEET_MODEL_INFO: Dict[str, Dict[str, Any]] = {
|
||||
"size_mb": 630,
|
||||
"desc": "Parakeet TDT 0.6B v2 (int8), English",
|
||||
},
|
||||
"bark-3.0": {
|
||||
"repo": "velddev/Underdog-Bark-3.0-sherpa-onnx",
|
||||
"revision": "3ae3af1a7ae2cb3a47f23498f83bb67ecb30a198",
|
||||
"files": [
|
||||
"encoder.int8.onnx",
|
||||
"decoder.int8.onnx",
|
||||
"joiner.int8.onnx",
|
||||
"tokens.txt",
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
],
|
||||
"size_mb": 653,
|
||||
"desc": "Underdog Bark 3.0 (int8), English; CC BY 4.0",
|
||||
},
|
||||
"bark-3.0-int4": {
|
||||
"repo": "velddev/Underdog-Bark-3.0-sherpa-onnx",
|
||||
"revision": "3ae3af1a7ae2cb3a47f23498f83bb67ecb30a198",
|
||||
"precision": "int4",
|
||||
"files": [
|
||||
"encoder.int4.onnx",
|
||||
"decoder.int4.onnx",
|
||||
"joiner.int4.onnx",
|
||||
"tokens.txt",
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
],
|
||||
"size_mb": 379,
|
||||
"desc": "Underdog Bark 3.0 (4-bit), English, half the memory; CC BY 4.0",
|
||||
},
|
||||
}
|
||||
|
||||
MODEL_SIZES = ["v3-european", "v2-english", "orukeet-v0.1.0"]
|
||||
MODEL_SIZES = ["v3-european", "v2-english", "orukeet-v0.1.0", "bark-3.0", "bark-3.0-int4"]
|
||||
|
||||
# Model offered when the engine has nothing downloaded yet.
|
||||
RECOMMENDED_MODEL = "v3-european"
|
||||
|
||||
Reference in New Issue
Block a user