From eabd1304cce0e053cd32ec910d2f0ea429e8af14 Mon Sep 17 00:00:00 2001 From: soryu Date: Wed, 28 Jan 2026 02:54:17 +0000 Subject: Add Qwen3-TTS streaming endpoint for voice synthesis (#40) * Task completion checkpoint * Task completion checkpoint * Task completion checkpoint * Add Qwen3-TTS research document for live TTS replacement Research findings for replacing Chatterbox TTS with Qwen3-TTS-12Hz-0.6B-Base: - Current TTS: Chatterbox-Turbo-ONNX with batch-only generation, no streaming - Qwen3-TTS: 97ms end-to-end latency, streaming support, 3-second voice cloning - Voice cloning: Requires 3s reference audio + transcript (Makima voice planned) - Integration: Python service with WebSocket bridge (no ONNX export available) - Languages: 10 supported including English and Japanese Document includes: - Current architecture analysis (makima/src/tts.rs) - Qwen3-TTS capabilities and requirements - Feasibility assessment for live/streaming TTS - Audio clip requirements for voice cloning - Preliminary technical approach with architecture diagrams Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 03:11:15 UTC * Add Qwen3-TTS research documentation Comprehensive research on replacing Chatterbox TTS with Qwen3-TTS-12Hz-0.6B-Base: - Current TTS implementation analysis (Chatterbox-Turbo-ONNX in makima/src/tts.rs) - Qwen3-TTS capabilities: 97ms streaming latency, voice cloning with 3s reference - Cross-lingual support: Japanese voice (Makima/Tomori Kusunoki) speaking English - Python microservice architecture recommendation (FastAPI + WebSocket) - Implementation phases and technical approach - Hardware requirements and dependencies Key findings: - Live/streaming TTS is highly feasible with 97ms latency - Voice cloning fully supported with 0.95 speaker similarity - Recommended: Python microservice with WebSocket streaming Co-Authored-By: Claude Opus 4.5 * Add comprehensive Qwen3-TTS integration specification This specification document defines the complete integration of Qwen3-TTS-12Hz-0.6B-Base as a replacement for the existing Chatterbox-Turbo TTS implementation. The document covers: ## Functional Requirements - WebSocket endpoint /api/v1/speak for streaming TTS - Voice cloning with default Makima voice (Japanese VA speaking English) - Support for custom voice references - Detailed client-to-server and server-to-client message protocols - Integration with Listen page for bidirectional speech ## Non-Functional Requirements - Latency targets: < 200ms first audio byte - Audio quality: 24kHz, mono, PCM16/PCM32f - Hardware requirements: CUDA GPU with 4-8GB VRAM - Scalability: 10 concurrent sessions per GPU ## Architecture Specification - Python TTS microservice with FastAPI/WebSocket - Rust proxy endpoint in makima server - Voice prompt caching mechanism (LRU cache) - Error handling and recovery strategies ## API Contract - Complete WebSocket message format definitions (TypeScript) - Error codes and responses (TTS_UNAVAILABLE, SYNTHESIS_ERROR, etc.) - Session state machine and lifecycle management ## Voice Asset Requirements - Makima voice clip specifications (5-10s WAV, transcript required) - Storage location: models/voices/makima/ - Metadata format for voice management ## Testing Strategy - Unit tests for Python TTS service and Rust proxy - Integration tests for WebSocket flow - Latency benchmarks with performance targets - Test data fixtures for various text lengths Co-Authored-By: Claude Opus 4.5 * Add Qwen3-TTS implementation plan Comprehensive implementation plan for replacing Chatterbox-TTS with Qwen3-TTS streaming TTS service, including: - Task breakdown with estimated hours for each phase - Phase 1: Python TTS microservice (FastAPI, WebSocket) - Phase 2: Rust proxy integration (speak.rs, tts_client.rs) - Detailed file changes and new module structure - Testing plan with unit, integration, and latency benchmarks - Risk assessment with mitigation strategies - Success criteria for each phase Based on specification in docs/specs/qwen3-tts-spec.md Co-Authored-By: Claude Opus 4.5 * Add author and research references to TTS implementation plan Add links to research documentation and author attribution. Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 03:25:06 UTC * Add Python TTS service project structure (Phase 1.1-1.3) Create the initial makima-tts Python service directory structure with: - pyproject.toml with FastAPI, Qwen-TTS, and torch dependencies - config.py with pydantic-settings TTSConfig class - models.py with Pydantic message models (Start, Speak, Stop, Ready, etc.) This implements tasks P1.1, P1.2, and P1.3 from the Qwen3-TTS implementation plan. Co-Authored-By: Claude Opus 4.5 * Add TTS engine and voice manager for Qwen3-TTS (Phase 1.4-1.5) Implement core TTS functionality: - tts_engine.py: Qwen3-TTS wrapper with streaming audio chunk generation - voice_manager.py: Voice prompt caching with LRU eviction and TTL support Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 03:30:06 UTC * Add TTS proxy client and message types (Phase 2.1, 2.2, 2.4) - Add tts_client.rs with TtsConfig, TtsCircuitBreaker, TtsError, TtsProxyClient, and TtsConnection structs for WebSocket proxying - Add TTS message types to messages.rs (TtsAudioEncoding, TtsPriority, TtsStartMessage, TtsSpeakMessage, TtsStopMessage, TtsClientMessage, TtsReadyMessage, TtsAudioChunkMessage, TtsCompleteMessage, TtsErrorMessage, TtsStoppedMessage, TtsServerMessage) - Export tts_client module from server mod.rs - tokio-tungstenite already present in Cargo.toml Co-Authored-By: Claude Opus 4.5 * Add TTS WebSocket handler and route (Phase 2.3, 2.5, 2.6) - Create speak.rs WebSocket handler that proxies to Python TTS service - Add TtsState fields (tts_client, tts_config) to AppState - Add with_tts() builder and is_tts_healthy() methods to AppState - Register /api/v1/speak route in the router - Add speak module export in handlers/mod.rs The handler forwards WebSocket messages bidirectionally between the client and the Python TTS microservice with proper error handling. Co-Authored-By: Claude Opus 4.5 * Add Makima voice profile assets for TTS voice cloning Creates the voice assets directory structure with: - manifest.json containing voice configuration (voice_id, speaker, language, reference audio path, and Japanese transcript placeholder) - README.md with instructions for obtaining voice reference audio Co-Authored-By: Claude Opus 4.5 * Add Rust-native Qwen3-TTS integration research document Research findings for integrating Qwen3-TTS-12Hz-0.6B-Base directly into the makima Rust codebase without Python. Key conclusions: - ONNX export is not viable (unsupported architecture) - Candle (HF Rust ML framework) is the recommended approach - Model weights available in safetensors format (2.52GB total) - Three components needed: LM backbone, code predictor, speech tokenizer - Crane project has Qwen3-TTS as highest priority (potential upstream) Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 11:21:43 UTC * [WIP] Heartbeat checkpoint - 2026-01-27 11:24:19 UTC * [WIP] Heartbeat checkpoint - 2026-01-27 11:26:43 UTC * feat: implement Rust-native Qwen3-TTS using candle framework Replace monolithic tts.rs with modular tts/ directory structure: - tts/mod.rs: TtsEngine trait, TtsEngineFactory, shared types (AudioChunk, TtsError), and utility functions (save_wav, resample, argmax) - tts/chatterbox.rs: existing ONNX-based ChatterboxTTS adapted to implement TtsEngine trait with Mutex-wrapped sessions for Send+Sync - tts/qwen3/mod.rs: Qwen3Tts entry point with HuggingFace model loading - tts/qwen3/config.rs: Qwen3TtsConfig parsing from HF config.json - tts/qwen3/model.rs: 28-layer Qwen3 transformer with RoPE, GQA (16 heads, 8 KV heads), SiLU MLP, RMS norm, and KV cache - tts/qwen3/code_predictor.rs: 5-layer MTP module predicting 16 codebooks - tts/qwen3/speech_tokenizer.rs: ConvNet encoder/decoder with 16-layer RVQ - tts/qwen3/generate.rs: autoregressive generation loop with streaming support Add candle-core, candle-nn, candle-transformers, safetensors to Cargo.toml. Co-Authored-By: Claude Opus 4.5 * feat: integrate TTS engine into speak WebSocket handler - Update speak.rs handler to use TTS engine directly from SharedState instead of returning a stub "not implemented" error - Add TtsEngine (OnceCell lazy-loaded) to AppState in state.rs with get_tts_engine() method for lazy initialization on first connection - Implement full WebSocket protocol: client sends JSON speak/cancel/stop messages, server streams binary PCM audio chunks and audio_end signals - Create voices/makima/manifest.json for Makima voice profile configuration - All files compile successfully with zero errors Co-Authored-By: Claude Opus 4.5 * feat: add /speak TTS page with WebSocket audio playback Add a new /speak frontend page for text-to-speech via WebSocket. The page accepts text input and streams synthesized PCM audio through the Web Audio API. Includes model loading indicator, cancel support, and connection status. Also adds a loading bar to the listen page ControlPanel during WebSocket connection. Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- Cargo.lock | 688 ++++++++++++++- TTS_RESEARCH.md | 351 ++++++++ docs/plans/qwen3-tts-implementation-plan.md | 514 ++++++++++++ docs/research/TTS_RESEARCH_NOTES.md | 405 +++++++++ docs/research/rust-native-tts-research.md | 297 +++++++ docs/research/tts-qwen3-research.md | 548 ++++++++++++ docs/specs/qwen3-tts-spec.md | 928 +++++++++++++++++++++ makima/Cargo.toml | 6 + .../src/components/listen/ControlPanel.tsx | 36 +- makima/frontend/src/hooks/useSpeakWebSocket.ts | 329 ++++++++ makima/frontend/src/index.css | 6 + makima/frontend/src/lib/api.ts | 1 + makima/frontend/src/main.tsx | 9 + makima/frontend/src/routes/listen.tsx | 1 + makima/frontend/src/routes/speak.tsx | 159 ++++ makima/src/main.rs | 14 +- makima/src/server/handlers/mod.rs | 1 + makima/src/server/handlers/speak.rs | 274 ++++++ makima/src/server/messages.rs | 161 ++++ makima/src/server/mod.rs | 3 +- makima/src/server/state.rs | 22 + makima/src/tts.rs | 580 ------------- makima/src/tts/chatterbox.rs | 485 +++++++++++ makima/src/tts/mod.rs | 281 +++++++ makima/src/tts/qwen3/code_predictor.rs | 261 ++++++ makima/src/tts/qwen3/config.rs | 271 ++++++ makima/src/tts/qwen3/generate.rs | 426 ++++++++++ makima/src/tts/qwen3/mod.rs | 287 +++++++ makima/src/tts/qwen3/model.rs | 581 +++++++++++++ makima/src/tts/qwen3/speech_tokenizer.rs | 612 ++++++++++++++ voices/makima/manifest.json | 12 + 31 files changed, 7939 insertions(+), 610 deletions(-) create mode 100644 TTS_RESEARCH.md create mode 100644 docs/plans/qwen3-tts-implementation-plan.md create mode 100644 docs/research/TTS_RESEARCH_NOTES.md create mode 100644 docs/research/rust-native-tts-research.md create mode 100644 docs/research/tts-qwen3-research.md create mode 100644 docs/specs/qwen3-tts-spec.md create mode 100644 makima/frontend/src/hooks/useSpeakWebSocket.ts create mode 100644 makima/frontend/src/routes/speak.tsx create mode 100644 makima/src/server/handlers/speak.rs delete mode 100644 makima/src/tts.rs create mode 100644 makima/src/tts/chatterbox.rs create mode 100644 makima/src/tts/mod.rs create mode 100644 makima/src/tts/qwen3/code_predictor.rs create mode 100644 makima/src/tts/qwen3/config.rs create mode 100644 makima/src/tts/qwen3/generate.rs create mode 100644 makima/src/tts/qwen3/mod.rs create mode 100644 makima/src/tts/qwen3/model.rs create mode 100644 makima/src/tts/qwen3/speech_tokenizer.rs create mode 100644 voices/makima/manifest.json diff --git a/Cargo.lock b/Cargo.lock index 1aeb184..30e65ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,6 +249,21 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -284,6 +299,20 @@ name = "bytemuck" version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "byteorder" @@ -297,6 +326,62 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +[[package]] +name = "candle-core" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ccf5ee3532e66868516d9b315f73aec9f34ea1a37ae98514534d458915dbf1" +dependencies = [ + "byteorder", + "gemm 0.17.1", + "half", + "memmap2", + "num-traits", + "num_cpus", + "rand 0.9.2", + "rand_distr", + "rayon", + "safetensors", + "thiserror 1.0.69", + "ug", + "yoke 0.7.5", + "zip 1.1.4", +] + +[[package]] +name = "candle-nn" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1160c3b63f47d40d91110a3e1e1e566ae38edddbbf492a60b40ffc3bc1ff38" +dependencies = [ + "candle-core", + "half", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "candle-transformers" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a0900d49f8605e0e7e6693a1f560e6271279de98e5fa369e7abf3aac245020" +dependencies = [ + "byteorder", + "candle-core", + "candle-nn", + "fancy-regex", + "num-traits", + "rand 0.9.2", + "rayon", + "serde", + "serde_json", + "serde_plain", + "tracing", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -884,6 +969,32 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + +[[package]] +name = "dyn-stack" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" +dependencies = [ + "bytemuck", + "dyn-stack-macros", +] + +[[package]] +name = "dyn-stack-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" + [[package]] name = "either" version = "1.15.0" @@ -908,6 +1019,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -983,6 +1106,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1185,6 +1319,243 @@ dependencies = [ "thread_local", ] +[[package]] +name = "gemm" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-c32 0.17.1", + "gemm-c64 0.17.1", + "gemm-common 0.17.1", + "gemm-f16 0.17.1", + "gemm-f32 0.17.1", + "gemm-f64 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" +dependencies = [ + "dyn-stack 0.13.2", + "gemm-c32 0.18.2", + "gemm-c64 0.18.2", + "gemm-common 0.18.2", + "gemm-f16 0.18.2", + "gemm-f32 0.18.2", + "gemm-f64 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" +dependencies = [ + "dyn-stack 0.13.2", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" +dependencies = [ + "dyn-stack 0.13.2", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" +dependencies = [ + "bytemuck", + "dyn-stack 0.10.0", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp 0.18.22", + "raw-cpuid 10.7.0", + "rayon", + "seq-macro", + "sysctl 0.5.5", +] + +[[package]] +name = "gemm-common" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" +dependencies = [ + "bytemuck", + "dyn-stack 0.13.2", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp 0.21.5", + "raw-cpuid 11.6.0", + "rayon", + "seq-macro", + "sysctl 0.6.0", +] + +[[package]] +name = "gemm-f16" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "gemm-f32 0.17.1", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f16" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" +dependencies = [ + "dyn-stack 0.13.2", + "gemm-common 0.18.2", + "gemm-f32 0.18.2", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" +dependencies = [ + "dyn-stack 0.13.2", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" +dependencies = [ + "dyn-stack 0.13.2", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.6.0", + "seq-macro", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1245,6 +1616,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "num-traits", + "rand 0.9.2", + "rand_distr", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1549,7 +1935,7 @@ checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", - "yoke", + "yoke 0.8.1", "zerofrom", "zerovec", ] @@ -1616,7 +2002,7 @@ dependencies = [ "displaydoc", "icu_locale_core", "writeable", - "yoke", + "yoke 0.8.1", "zerofrom", "zerotrie", "zerovec", @@ -1895,6 +2281,16 @@ version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.15" @@ -2001,6 +2397,9 @@ dependencies = [ "backoff", "base64 0.22.1", "bytes", + "candle-core", + "candle-nn", + "candle-transformers", "chrono", "clap", "config", @@ -2030,6 +2429,7 @@ dependencies = [ "regex", "reqwest", "rusqlite", + "safetensors", "serde", "serde_json", "sha2", @@ -2091,6 +2491,16 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", + "stable_deref_trait", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -2260,6 +2670,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2292,6 +2716,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", ] @@ -2321,6 +2746,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2341,6 +2777,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -2698,6 +3156,15 @@ dependencies = [ "num-integer", ] +[[package]] +name = "proc-macro-crate" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +dependencies = [ + "toml_edit 0.23.10+spec-1.0.0", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -2707,6 +3174,32 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + +[[package]] +name = "pulp" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b86df24f0a7ddd5e4b95c94fc9ed8a98f1ca94d3b01bdce2824097e7835907" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "reborrow", + "version_check", +] + [[package]] name = "quote" version = "1.0.42" @@ -2781,6 +3274,16 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.2", +] + [[package]] name = "ratatui" version = "0.29.0" @@ -2802,6 +3305,24 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -2850,6 +3371,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3153,6 +3680,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safetensors" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "same-file" version = "1.0.6" @@ -3200,6 +3737,12 @@ dependencies = [ "libc", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -3254,6 +3797,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -3970,6 +4522,34 @@ dependencies = [ "syn", ] +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.10.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags 2.10.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -4310,8 +4890,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -4323,6 +4903,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -4332,11 +4921,32 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_write", "winnow", ] +[[package]] +name = "toml_edit" +version = "0.23.10+spec-1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +dependencies = [ + "indexmap", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +dependencies = [ + "winnow", +] + [[package]] name = "toml_write" version = "0.1.2" @@ -4529,6 +5139,27 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "ug" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03719c61a91b51541f076dfdba45caacf750b230cefaa4b32d6f5411c3f7f437" +dependencies = [ + "gemm 0.18.2", + "half", + "libloading", + "memmap2", + "num", + "num-traits", + "num_cpus", + "rayon", + "safetensors", + "serde", + "thiserror 1.0.69", + "tracing", + "yoke 0.7.5", +] + [[package]] name = "unicase" version = "2.8.1" @@ -4738,7 +5369,7 @@ dependencies = [ "serde_json", "url", "utoipa", - "zip", + "zip 3.0.0", ] [[package]] @@ -5322,6 +5953,18 @@ dependencies = [ "hashlink 0.8.4", ] +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive 0.7.5", + "zerofrom", +] + [[package]] name = "yoke" version = "0.8.1" @@ -5329,10 +5972,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ "stable_deref_trait", - "yoke-derive", + "yoke-derive 0.8.1", "zerofrom", ] +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "yoke-derive" version = "0.8.1" @@ -5399,7 +6054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", - "yoke", + "yoke 0.8.1", "zerofrom", ] @@ -5409,7 +6064,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ - "yoke", + "yoke 0.8.1", "zerofrom", "zerovec-derive", ] @@ -5425,6 +6080,21 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "indexmap", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "zip" version = "3.0.0" diff --git a/TTS_RESEARCH.md b/TTS_RESEARCH.md new file mode 100644 index 0000000..da7b8b8 --- /dev/null +++ b/TTS_RESEARCH.md @@ -0,0 +1,351 @@ +# TTS Research Document: Qwen3-TTS Integration for Makima + +## Executive Summary + +This document summarizes research on replacing the existing ChatterboxTTS implementation with Qwen3-TTS for live/streaming TTS with Makima's Japanese voice speaking English. + +--- + +## 1. Current TTS Implementation Analysis + +### 1.1 Architecture Overview + +The current makima codebase uses **ChatterboxTTS** (ResembleAI/chatterbox-turbo-ONNX) with the following components: + +| Component | File | Purpose | +|-----------|------|---------| +| TTS Module | `makima/src/tts.rs` | Core TTS inference using ONNX Runtime | +| Audio Processing | `makima/src/audio.rs` | Audio decoding, resampling (Symphonia) | +| Library Export | `makima/src/lib.rs` | Exposes `pub mod tts` | + +### 1.2 ChatterboxTTS Technical Details + +```rust +// Key constants from tts.rs +pub const SAMPLE_RATE: u32 = 24_000; +const MODEL_ID: &str = "ResembleAI/chatterbox-turbo-ONNX"; +const DEFAULT_MODEL_DIR: &str = "models/chatterbox-turbo"; +``` + +**ONNX Model Files:** +- `speech_encoder.onnx` - Encodes reference voice audio +- `embed_tokens.onnx` - Text token embedding +- `language_model.onnx` - Autoregressive token generation (24 layers, 16 KV heads) +- `conditional_decoder.onnx` - Decodes speech tokens to waveform +- `tokenizer.json` - Text tokenization + +### 1.3 Current API Surface + +```rust +pub struct ChatterboxTTS { + pub fn from_pretrained(model_dir: Option<&str>) -> Result + pub fn generate_tts(&mut self, text: &str) -> Result, TtsError> // Returns VoiceRequired error + pub fn generate_tts_with_voice(text: &str, sample_audio_path: &Path) -> Result, TtsError> + pub fn generate_tts_with_samples(text: &str, samples: &[f32], sample_rate: u32) -> Result, TtsError> +} +pub fn save_wav(samples: &[f32], path: &Path) -> Result<(), TtsError> +``` + +### 1.4 Voice Cloning Capabilities (Current) + +- **Requires** voice reference audio (returns `VoiceRequired` error without it) +- Accepts reference audio via file path or raw samples +- Resamples reference to 24kHz internally +- Uses speaker embeddings + speaker features for voice cloning + +### 1.5 Streaming/Live TTS (Current) + +**NOT SUPPORTED** - The current implementation: +- Generates entire audio in one pass +- Uses autoregressive token generation with max 1024 tokens +- No chunked/streaming output capability +- Full pipeline must complete before audio is available + +### 1.6 Server Integration Status + +The TTS module is **not currently exposed via HTTP endpoints**. The server (`makima/src/server/mod.rs`) has: +- `/api/v1/listen` - WebSocket for Speech-to-Text (STT) only +- No TTS endpoints exist + +--- + +## 2. Qwen3-TTS Model Analysis + +### 2.1 Model Specifications + +| Attribute | Value | +|-----------|-------| +| **Model** | Qwen/Qwen3-TTS-12Hz-0.6B-Base | +| **Parameters** | 0.6B (also available: 1.7B version) | +| **Architecture** | Discrete multi-codebook LM (16 codebooks, 2048 size) | +| **Tokenizer** | Qwen3-TTS-Tokenizer-12Hz | +| **Sample Rate** | 12 Hz tokenizer (reconstructs to standard rates) | +| **License** | Apache 2.0 | + +### 2.2 Key Capabilities + +#### Voice Cloning +- **3-second rapid voice clone** - Minimal reference audio needed +- **Flexible input formats**: local files, URLs, base64, (numpy_array, sample_rate) tuples +- **Reusable voice prompts**: Create once, use for multiple generations + +```python +# Voice cloning example +model = Qwen3TTSModel.from_pretrained("Qwen/Qwen3-TTS-12Hz-0.6B-Base") +wavs, sr = model.generate_voice_clone( + text="Target text", + language="English", + ref_audio="reference.wav", + ref_text="Reference transcript" +) +``` + +#### Streaming/Live TTS +- **97ms end-to-end latency** - Ultra-low latency streaming +- **Dual-track hybrid streaming architecture** - Supports both streaming and non-streaming +- **First packet after single character** - Immediate response capability + +#### Multilingual Support +10 languages: Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian + +#### Quality Metrics +| Metric | Value | +|--------|-------| +| WER (English) | 1.32 | +| Speaker Similarity (English) | 0.829 | +| PESQ_WB | 3.21 | +| STOI | 0.96 | +| UTMOS | 4.16 | + +### 2.3 Requirements + +```bash +# Environment +Python 3.12 (recommended) +CUDA-compatible GPU with 8GB+ VRAM + +# Installation +pip install -U qwen-tts +pip install -U flash-attn --no-build-isolation # Optional, reduces GPU memory +``` + +### 2.4 Current Deployment Options + +| Option | Status | Notes | +|--------|--------|-------| +| **Python (qwen-tts)** | Stable | Official package | +| **vLLM-Omni** | Offline only | Online serving coming | +| **ONNX Export** | Not available | No official support | +| **Rust Implementation** | Draft PR #8 | Early development | +| **DashScope API** | Available | Alibaba Cloud hosted | + +--- + +## 3. Makima Voice Audio Clips + +### 3.1 Voice Actress Information + +| Attribute | Value | +|-----------|-------| +| **Character** | Makima (Chainsaw Man) | +| **Japanese VA** | Tomori Kusunoki (楠木ともり) | +| **English VA** | Suzie Yeung | +| **Agency** | Sony Music Artists | + +### 3.2 Audio Clip Sources + +1. **Chainsaw Man Anime Episodes** - Primary source for Japanese voice +2. **Behind The Voice Actors** - Character voice samples +3. **YouTube Clips** - Interview compilations, scene clips +4. **Official Media** - Promotional videos, trailers + +### 3.3 Audio Requirements for Voice Cloning + +#### Qwen3-TTS Requirements +| Parameter | Requirement | +|-----------|-------------| +| **Minimum Duration** | 3 seconds (basic quality) | +| **Recommended Duration** | 10-30 seconds (professional quality) | +| **Format** | WAV, FLAC, MP3, OGG, AIFF, AAC | +| **Sample Rate** | 24kHz or above recommended | +| **Channels** | Mono preferred | + +#### Best Practices +- **Clean audio**: No background music/noise +- **Single speaker**: Makima's voice only +- **Consistent tone**: Avoid dramatic variations +- **Include transcript**: Reference text improves quality +- **Varied content**: Mix of sentence types for flexibility + +#### Recommended Clip Types +1. Calm, composed dialogue (Makima's signature tone) +2. Commands/instructions (authoritative delivery) +3. Questions (natural intonation) +4. Longer monologues (for voice consistency) + +--- + +## 4. Feasibility Assessment for Live/Streaming TTS + +### 4.1 Technical Challenges + +| Challenge | Severity | Notes | +|-----------|----------|-------| +| No ONNX export | **High** | Current codebase uses ONNX Runtime | +| Rust implementation | **High** | Only draft PR available | +| Python dependency | Medium | Would require sidecar service | +| GPU memory | Medium | 8GB+ VRAM required | +| Streaming API | Low | Supported in Qwen3-TTS | + +### 4.2 Integration Approaches + +#### Option A: Python Sidecar Service (Recommended) +**Architecture**: Rust server + Python TTS service via HTTP/gRPC + +**Pros:** +- Uses official Qwen3-TTS Python package +- Full streaming support (97ms latency) +- Simpler maintenance + +**Cons:** +- Additional deployment complexity +- Inter-process communication overhead + +``` +┌─────────────────┐ HTTP/gRPC ┌─────────────────┐ +│ Makima Server │ ◄──────────────► │ Qwen3-TTS │ +│ (Rust/Axum) │ │ (Python/FastAPI)│ +└─────────────────┘ └─────────────────┘ +``` + +**Available Implementations:** +- [ValyrianTech/Qwen3-TTS_server](https://github.com/ValyrianTech/Qwen3-TTS_server) - FastAPI server +- [Qwen3-TTS-Openai-Fastapi](https://github.com/twolven/Qwen3-TTS-Openai-Fastapi) - OpenAI-compatible API + +#### Option B: Wait for Rust Implementation +**Status**: Draft PR #8 in early development + +**Pros:** +- Native Rust integration +- No Python dependency +- Matches current architecture + +**Cons:** +- Unknown timeline +- May require significant adaptation + +#### Option C: Hybrid (ChatterboxTTS + Qwen3-TTS) +Keep ChatterboxTTS for ONNX compatibility, add Qwen3-TTS for streaming + +**Pros:** +- Gradual migration +- Fallback capability + +**Cons:** +- Dual model maintenance +- Increased complexity + +### 4.3 Recommendation + +**Short-term (1-2 weeks)**: Implement **Option A** with Python sidecar +- Deploy ValyrianTech/Qwen3-TTS_server or similar +- Add HTTP client in Rust to call TTS service +- Implement WebSocket endpoint for streaming audio + +**Long-term (3-6 months)**: Monitor Rust implementation progress +- Evaluate draft PR #8 stability +- Consider contributing to Rust port +- Migrate to native Rust when mature + +--- + +## 5. Preliminary Technical Approach + +### 5.1 Phase 1: Voice Preparation + +1. **Collect Makima Audio Clips** + - Extract 3-5 clean clips from anime (10-30 seconds each) + - Ensure Japanese voice, clear audio, no BGM + - Prepare transcripts for each clip + +2. **Test Voice Cloning Quality** + - Use Qwen3-TTS demo to validate clips + - Iterate on clip selection for best results + +### 5.2 Phase 2: TTS Service Setup + +1. **Deploy Qwen3-TTS Server** + ```bash + # Using ValyrianTech server + docker run --gpus all -p 7860:7860 qwen3-tts-server + ``` + +2. **Configure Voice Clone Profile** + - Upload Makima reference audio + - Store voice clone prompt for reuse + +### 5.3 Phase 3: Makima Integration + +1. **Add TTS Client Module** + ```rust + // New module: makima/src/tts_client.rs + pub struct QwenTTSClient { + base_url: String, + voice_profile: String, + } + + impl QwenTTSClient { + pub async fn generate_speech(&self, text: &str) -> Result, Error> + pub async fn generate_speech_streaming(&self, text: &str) -> impl Stream> + } + ``` + +2. **Add TTS Endpoint** + ```rust + // In makima/src/server/mod.rs + .route("/api/v1/tts", post(tts_handler)) + .route("/api/v1/tts/stream", get(tts_streaming_handler)) + ``` + +3. **WebSocket Integration for Listen Page** + - Bidirectional audio: STT input, TTS output + - Low-latency streaming for conversational flow + +### 5.4 Phase 4: Listen Page Integration + +1. **Update Frontend** + - Add TTS playback capability + - Handle streaming audio chunks + - UI for voice response indicators + +2. **Orchestration Logic** + - STT → LLM → TTS pipeline + - Interrupt handling for user speech + +--- + +## 6. Open Questions + +1. **Voice Rights**: Are there legal considerations for cloning Tomori Kusunoki's voice? +2. **GPU Allocation**: Shared GPU for STT + TTS, or separate? +3. **Latency Budget**: What's acceptable end-to-end latency for Listen page? +4. **Fallback Strategy**: What happens if TTS service is unavailable? +5. **Multi-user**: How to handle concurrent TTS requests? + +--- + +## 7. References + +- [Qwen3-TTS HuggingFace](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-Base) +- [Qwen3-TTS GitHub](https://github.com/QwenLM/Qwen3-TTS) +- [Qwen3-TTS Technical Report](https://arxiv.org/abs/2601.15621) +- [ValyrianTech Qwen3-TTS Server](https://github.com/ValyrianTech/Qwen3-TTS_server) +- [Qwen3-TTS OpenAI-Compatible FastAPI](https://github.com/twolven/Qwen3-TTS-Openai-Fastapi) +- [Makima Voice Actors](https://www.behindthevoiceactors.com/tv-shows/Chainsaw-Man/Makima/) +- [ChatterboxTTS Audio Guidelines](https://github.com/resemble-ai/chatterbox/issues/39) +- [Voice Cloning Best Practices - Resemble AI](https://www.resemble.ai/script-to-read-for-voice-cloning-guidelines/) +- [Qwen3-rs (Rust LLM implementation)](https://github.com/reinterpretcat/qwen3-rs) + +--- + +*Document created: Research phase for Makima TTS replacement contract* diff --git a/docs/plans/qwen3-tts-implementation-plan.md b/docs/plans/qwen3-tts-implementation-plan.md new file mode 100644 index 0000000..76ecb33 --- /dev/null +++ b/docs/plans/qwen3-tts-implementation-plan.md @@ -0,0 +1,514 @@ +# Qwen3-TTS Implementation Plan — Pure Rust (Candle) + +**Version:** 2.0 +**Created:** 2026-01-27 +**Status:** Final +**Authors:** makima development team +**Spec Reference:** [docs/specs/qwen3-tts-spec.md](../specs/qwen3-tts-spec.md) +**Research:** [docs/research/rust-native-tts-research.md](../research/rust-native-tts-research.md) + +--- + +## Table of Contents + +1. [Overview](#1-overview) +2. [Task Breakdown](#2-task-breakdown) +3. [File Changes](#3-file-changes) +4. [Phase 1: Candle-Based TTS Module](#4-phase-1-candle-based-tts-module) +5. [Phase 2: WebSocket Handler + Voice Assets](#5-phase-2-websocket-handler--voice-assets) +6. [Phase 3: Optimization + Integration](#6-phase-3-optimization--integration) +7. [Testing Plan](#7-testing-plan) +8. [Risk Assessment](#8-risk-assessment) +9. [Dependencies & Prerequisites](#9-dependencies--prerequisites) +10. [Success Criteria](#10-success-criteria) + +--- + +## 1. Overview + +This plan details the implementation of Qwen3-TTS integration for the makima system as a **pure Rust** solution using the **candle** ML framework. There is no Python microservice and no proxy pattern — the TTS model runs directly inside the main makima process, loading safetensors weights via candle. + +### Key Objectives + +1. Implement Qwen3-TTS model inference natively in Rust using candle +2. Create a `makima/src/tts/` module with TTS trait, Chatterbox adapter, and Qwen3 submodule +3. Update the `/api/v1/speak` WebSocket handler to call the TTS engine directly +4. Enable streaming audio delivery with <200ms time-to-first-audio (TTFA) +5. Support voice cloning with default Makima voice + +### Architecture Summary + +``` +Client Browser + │ + │ WebSocket: /api/v1/speak + ▼ +Makima Server (Rust/Axum) + │ + │ speak.rs handler → TTS Engine (in-process) + │ + │ candle-based Qwen3-TTS inference + │ (safetensors weights loaded directly) + ▼ +Audio Stream back to client +``` + +**Key architectural decisions:** +- **No Python.** All inference runs in Rust via candle. +- **No microservice.** TTS runs in-process, no separate service to deploy. +- **No proxy.** The speak handler calls the TTS engine directly. +- **Lazy loading.** Models loaded on first TTS request (like listen.rs pattern). +- **SafeTensors.** Weights loaded directly — no ONNX conversion needed. + +--- + +## 2. Task Breakdown + +### Phase 1: Candle-Based TTS Module (Priority: Critical) + +| ID | Task | Depends On | Estimated Hours | +|----|------|------------|-----------------| +| P1.1 | Create `makima/src/tts/mod.rs` — TTS trait + factory + types | - | 3 | +| P1.2 | Move existing `tts.rs` to `makima/src/tts/chatterbox.rs` | P1.1 | 2 | +| P1.3 | Create `makima/src/tts/qwen3/config.rs` — Model config parsing | P1.1 | 2 | +| P1.4 | Implement `makima/src/tts/qwen3/model.rs` — 28-layer LM backbone | P1.3 | 12 | +| P1.5 | Implement `makima/src/tts/qwen3/code_predictor.rs` — MTP module | P1.4 | 8 | +| P1.6 | Implement `makima/src/tts/qwen3/speech_tokenizer.rs` — ConvNet codec | P1.3 | 10 | +| P1.7 | Implement `makima/src/tts/qwen3/generate.rs` — Autoregressive generation | P1.4, P1.5, P1.6 | 8 | +| P1.8 | Create `makima/src/tts/qwen3/mod.rs` — Public API | P1.7 | 3 | +| P1.9 | Add candle dependencies to `Cargo.toml` | - | 1 | +| P1.10 | Unit tests for config, model layers, tokenizer | P1.4-P1.6 | 6 | + +**Phase 1 Total: ~55 hours** + +### Phase 2: WebSocket Handler + Voice Assets (Priority: High) + +| ID | Task | Depends On | Estimated Hours | +|----|------|------------|-----------------| +| P2.1 | Rewrite `speak.rs` — Direct TTS handler (remove proxy) | P1.8 | 6 | +| P2.2 | Add TTS models to `SharedState` (lazy loading via `OnceCell`) | P1.8 | 3 | +| P2.3 | Implement voice prompt caching (LRU) | P1.8 | 3 | +| P2.4 | Remove `tts_client.rs` (no longer needed) | P2.1 | 1 | +| P2.5 | Update `state.rs` — Remove TTS proxy fields, add TTS model fields | P2.2 | 2 | +| P2.6 | Update `mod.rs` — Remove `tts_client` module | P2.4 | 0.5 | +| P2.7 | Create voice manifest structure (`models/voices/makima/`) | - | 1 | +| P2.8 | Acquire Makima voice reference audio | - | 2 | +| P2.9 | Test voice cloning quality | P1.8, P2.8 | 2 | + +**Phase 2 Total: ~20.5 hours** + +### Phase 3: Optimization + Integration (Priority: Medium) + +| ID | Task | Depends On | Estimated Hours | +|----|------|------------|-----------------| +| P3.1 | Implement streaming generation (token-by-token waveform decode) | P2.1 | 6 | +| P3.2 | GPU memory optimization (bf16, cache management) | P3.1 | 4 | +| P3.3 | Listen page integration for bidirectional speech | P2.1 | 4 | +| P3.4 | Latency benchmarks | P3.1 | 3 | +| P3.5 | Integration tests (WebSocket end-to-end) | P2.1 | 4 | +| P3.6 | Documentation | P3.5 | 2 | + +**Phase 3 Total: ~23 hours** + +--- + +## 3. File Changes + +### New Files + +``` +makima/src/tts/ +├── mod.rs // TTS trait, factory, shared types +├── chatterbox.rs // Existing ONNX-based Chatterbox (moved from tts.rs) +└── qwen3/ + ├── mod.rs // Qwen3TTS public API + ├── model.rs // Qwen3 LM transformer (28 layers) + ├── code_predictor.rs // MTP module (5 layers, 16 codebooks) + ├── speech_tokenizer.rs // Encoder + Decoder (causal ConvNet + RVQ) + ├── config.rs // Model config from config.json / safetensors + └── generate.rs // Autoregressive generation loop with KV cache +``` + +### Modified Files + +| File | Change Description | +|------|-------------------| +| `makima/src/server/handlers/speak.rs` | Rewrite: direct TTS engine call instead of proxy | +| `makima/src/server/state.rs` | Remove `tts_client`/`tts_config` fields, add `tts_models: OnceCell` | +| `makima/src/server/mod.rs` | Remove `pub mod tts_client;` | +| `makima/src/server/handlers/mod.rs` | No change (speak already exported) | +| `makima/Cargo.toml` | Add candle-core, candle-nn, candle-transformers; remove tokio-tungstenite if unused | +| `makima/src/lib.rs` or `main.rs` | Add `pub mod tts;` | + +### Deleted Files + +| File | Reason | +|------|--------| +| `makima/src/server/tts_client.rs` | No longer needed — no proxy pattern | +| `tts-service/` (entire directory) | Python service rejected; pure Rust solution | + +--- + +## 4. Phase 1: Candle-Based TTS Module + +### 4.1 TTS Trait and Factory (`tts/mod.rs`) + +```rust +use async_trait::async_trait; + +/// Audio chunk for streaming output. +pub struct AudioChunk { + pub samples: Vec, + pub sample_rate: u32, + pub is_final: bool, +} + +/// TTS engine trait — implemented by Chatterbox and Qwen3. +#[async_trait] +pub trait TtsEngine: Send + Sync { + /// Generate audio from text. + async fn generate( + &self, + text: &str, + voice_id: &str, + language: &str, + ) -> Result, TtsError>; + + /// Pre-load a voice prompt. + async fn load_voice(&self, voice_id: &str) -> Result<(), TtsError>; + + /// Check if the engine is ready. + fn is_ready(&self) -> bool; +} +``` + +### 4.2 Qwen3 LM Backbone (`tts/qwen3/model.rs`) + +Extend candle-transformers' Qwen2 model implementation: + +- **28 transformer layers** with RoPE, GQA (16 heads, 8 KV heads), head dim 128 +- **Hidden size:** 1024, **intermediate size:** 3072 +- **Input:** text tokens + reference audio codes (concatenated) +- **Output:** zeroth codebook token logits + +**Key implementation detail:** The existing `candle_transformers::models::qwen2` module provides the base attention and MLP layers. We extend this with: +- TTS-specific input embedding (text + audio token embeddings) +- Speaker encoder concatenation +- Code predictor output head (instead of standard LM head) + +### 4.3 Code Predictor (`tts/qwen3/code_predictor.rs`) + +- **5-layer** transformer module +- **Input:** hidden states from the main LM +- **Output:** 16 codebook predictions (vocab size 2048 each) +- After the main LM predicts the zeroth codebook token, this module predicts the remaining 15 codebook layers in parallel + +### 4.4 Speech Tokenizer (`tts/qwen3/speech_tokenizer.rs`) + +Two sub-components: + +**Encoder** (used for voice cloning): +- Causal 1D ConvNet converting reference audio waveform → discrete multi-codebook tokens +- 16-layer RVQ (Residual Vector Quantization) +- First codebook = semantic (WavLM-guided), remaining 15 = acoustic + +**Decoder** (used for audio output): +- Causal 1D ConvNet reconstructing waveforms from discrete codes +- Input: 16 codebook indices → lookup embeddings → ConvNet → waveform +- Output: 24kHz mono audio + +**candle implementation notes:** +- `candle_nn::Conv1d` for all convolution layers +- `candle_nn::Embedding` for codebook lookups +- Weight normalization handled manually + +### 4.5 Autoregressive Generation (`tts/qwen3/generate.rs`) + +```rust +pub async fn generate( + model: &Qwen3Model, + code_predictor: &CodePredictor, + speech_tokenizer: &SpeechTokenizer, + text_tokens: &[u32], + voice_prompt: &VoicePrompt, +) -> Result, TtsError> { + // 1. Encode reference audio → speaker embedding + audio codes + let speaker_emb = speech_tokenizer.encode(&voice_prompt.audio)?; + + // 2. Prepare input: [text_tokens, audio_codes] + let input = prepare_input(text_tokens, &speaker_emb)?; + + // 3. Autoregressive loop with KV cache + let mut kv_cache = KvCache::new(model.num_layers()); + let mut generated_codes = Vec::new(); + + loop { + let logits = model.forward(&input, &mut kv_cache)?; + let next_token = sample_token(&logits); + + if next_token == EOS_TOKEN { break; } + generated_codes.push(next_token); + + // 4. Code predictor: predict remaining 15 codebooks + let all_codes = code_predictor.predict(&model.last_hidden_state(), next_token)?; + + // 5. Decode to audio (can be done incrementally for streaming) + let chunk = speech_tokenizer.decode(&all_codes)?; + // yield chunk for streaming + } +} +``` + +--- + +## 5. Phase 2: WebSocket Handler + Voice Assets + +### 5.1 Speak Handler (Rewritten) + +```rust +// makima/src/server/handlers/speak.rs +// +// Direct TTS handler — no proxy, no external service. + +pub async fn websocket_handler( + ws: WebSocketUpgrade, + State(state): State, +) -> Response { + ws.on_upgrade(|socket| handle_speak_socket(socket, state)) +} + +async fn handle_speak_socket(socket: WebSocket, state: SharedState) { + let session_id = Uuid::new_v4().to_string(); + + // Lazy-load TTS models (like listen.rs does for STT) + let tts = match state.get_tts_models().await { + Ok(tts) => tts, + Err(e) => { + send_error(&mut socket, "MODEL_LOADING", &e.to_string()).await; + return; + } + }; + + // Session loop: parse JSON messages, dispatch to TTS engine + let (mut sender, mut receiver) = socket.split(); + + while let Some(msg) = receiver.next().await { + match parse_client_message(msg) { + ClientMessage::Start(config) => { + // Load voice, send Ready + } + ClientMessage::Speak(text) => { + // Run inference, stream audio chunks + for chunk in tts.engine.generate(&text, voice_id, language).await? { + sender.send(Message::Binary(chunk.to_pcm16())).await?; + } + sender.send(complete_message()).await?; + } + ClientMessage::Stop => break, + ClientMessage::Cancel => { /* abort current generation */ } + } + } +} +``` + +### 5.2 State Changes + +Remove from `AppState`: +- `tts_client: Option>` +- `tts_config: Option` +- `with_tts()` method +- `is_tts_healthy()` method + +Add to `AppState`: +- `tts_models: OnceCell` — lazily loaded TTS engine +- `get_tts_models()` method (async, like `get_ml_models()`) + +### 5.3 Voice Assets + +``` +models/voices/makima/ +├── manifest.json # Voice metadata +├── reference.wav # 5-15 second reference audio +└── transcript.txt # Exact transcript of reference audio +``` + +--- + +## 6. Phase 3: Optimization + Integration + +### 6.1 Streaming Generation + +The 12Hz model's causal architecture enables token-by-token waveform generation: +- Each token = ~80ms of audio (12.5 Hz) +- After generating each token, decode immediately and send audio chunk +- Client receives audio before full generation completes + +### 6.2 GPU Memory Optimization + +- Load weights in bf16/f16 (candle supports both) +- Implement KV cache with fixed maximum size +- Clear cache between sessions +- CPU fallback when GPU is unavailable + +### 6.3 Listen Page Integration + +Following the pattern in `listen.rs`: +- TTS model protected behind `tokio::sync::Mutex` +- WebSocket endpoint emits audio chunks as tokens are generated +- Bidirectional: STT (listen) → process → TTS (speak) loop + +--- + +## 7. Testing Plan + +### 7.1 Unit Tests + +| Test Area | Coverage | Key Tests | +|-----------|----------|-----------| +| Config parsing | 100% | Load config from JSON, validate fields | +| Model layers | 80% | Attention, MLP, Conv1d shapes | +| Code predictor | 85% | Multi-codebook output shapes | +| Speech tokenizer | 80% | Encode/decode round-trip | +| Voice cache | 95% | LRU eviction, TTL expiration | +| Message parsing | 100% | All client/server message types | + +### 7.2 Integration Tests + +| Test | Description | +|------|-------------| +| WebSocket flow | Connect → Start → Speak → Audio chunks → Complete → Stop | +| Error handling | Invalid text, unknown voice, model loading failure | +| Cancellation | Cancel mid-generation | +| Voice cloning | Generate with custom reference audio | + +### 7.3 Latency Benchmarks + +| Metric | Target | Acceptable | Warning | +|--------|--------|------------|---------| +| First Audio (short text) | < 150ms | < 200ms | > 300ms | +| First Audio (medium text) | < 200ms | < 300ms | > 500ms | +| First Audio (long text) | < 300ms | < 500ms | > 800ms | +| Inter-chunk latency | < 30ms | < 50ms | > 100ms | +| GPU memory | < 4GB | < 6GB | > 8GB | + +--- + +## 8. Risk Assessment + +### 8.1 Technical Risks + +| Risk | Likelihood | Impact | Mitigation | +|------|------------|--------|------------| +| **Candle implementation takes longer** | Medium | Medium | Reference Crane's Spark-TTS; use qwen3-rs as LM reference | +| **Speech tokenizer ConvNet is complex** | Medium | High | Study PyTorch source; ConvNet layers are simpler than transformers | +| **Model quality differs from PyTorch** | Low | High | Validate with reference audio; ensure bf16 precision | +| **Crane ships Qwen3-TTS first** | Medium | Positive | Adopt their implementation or use as reference | +| **GPU memory issues** | Low | Medium | 0.6B model is small (~2.5GB); fits in 4GB VRAM | + +### 8.2 Contingency Plans + +| Scenario | Response | +|----------|----------| +| Candle implementation blocked | Use Crane crate as dependency if they ship Qwen3-TTS | +| ConvNet decoder too complex | Implement simplified decoder; optimize later | +| Latency exceeds targets | Start with batch mode + chunked delivery (acceptable UX) | +| No GPU available | CPU fallback with candle's MKL support (degraded performance) | + +--- + +## 9. Dependencies & Prerequisites + +### 9.1 Rust Dependencies + +Add to `Cargo.toml`: + +```toml +[dependencies] +candle-core = "0.8" +candle-nn = "0.8" +candle-transformers = "0.8" +# Keep existing: tokenizers, hf-hub, safetensors, ndarray +``` + +### 9.2 Hardware Requirements + +| Component | Minimum | Recommended | +|-----------|---------|-------------| +| GPU | CUDA 4GB VRAM / Metal (macOS) | NVIDIA RTX 3060+ (8GB+) | +| RAM | 8GB | 16GB | +| Storage | 5GB (model weights) | 10GB | + +### 9.3 Voice Asset Prerequisites + +Before Phase 2 voice testing: +1. Makima voice reference audio (5-15 seconds, clean speech) +2. Accurate transcript of reference audio +3. Format: WAV 24kHz mono, 16-bit PCM + +--- + +## 10. Success Criteria + +### 10.1 Phase 1 Completion + +- [ ] candle-based Qwen3 model loads safetensors weights +- [ ] Forward pass produces valid logits +- [ ] Speech tokenizer encodes/decodes audio +- [ ] Code predictor generates 16 codebook layers +- [ ] Unit tests pass with > 80% coverage + +### 10.2 Phase 2 Completion + +- [ ] `/api/v1/speak` endpoint produces audio from text +- [ ] No Python service required +- [ ] Voice cloning works with reference audio +- [ ] Error handling returns appropriate codes +- [ ] speak.rs calls TTS engine directly (no proxy) + +### 10.3 Phase 3 Completion + +- [ ] Streaming generation with < 200ms TTFA +- [ ] GPU memory usage < 6GB +- [ ] Integration tests pass +- [ ] Listen page bidirectional speech works +- [ ] Latency benchmarks documented + +### 10.4 Final Acceptance Criteria + +1. **Functional:** End-to-end TTS streaming via WebSocket, pure Rust, no Python +2. **Performance:** TTFA < 200ms, subsequent chunks < 100ms +3. **Quality:** Synthesized speech is intelligible and recognizable as Makima +4. **Reliability:** Error handling is robust; graceful degradation on GPU failure +5. **Architecture:** Clean `tts/` module with trait-based engine selection + +--- + +## Appendix A: Quick Start Commands + +### Development + +```bash +# Build with candle GPU support +cd makima +cargo build --features cuda # or --features metal for macOS + +# Run server with TTS enabled +TTS_ENGINE=qwen3 TTS_DEVICE=cuda:0 cargo run + +# Run TTS-specific tests +cargo test tts +``` + +### Benchmarks + +```bash +# Run latency benchmarks (requires GPU) +cargo bench --bench tts_latency +``` + +## Appendix B: Reference Implementations + +- [candle-transformers qwen2 model](https://docs.rs/candle-transformers/latest/candle_transformers/models/qwen2/index.html) — base attention/MLP layers +- [qwen3-rs](https://github.com/reinterpretcat/qwen3-rs) — educational Qwen3 in Rust +- [Crane](https://github.com/lucasjinreal/Crane) — Rust TTS engine (Qwen3-TTS on roadmap) +- [docs/research/rust-native-tts-research.md](../research/rust-native-tts-research.md) — full feasibility analysis diff --git a/docs/research/TTS_RESEARCH_NOTES.md b/docs/research/TTS_RESEARCH_NOTES.md new file mode 100644 index 0000000..72ac8c6 --- /dev/null +++ b/docs/research/TTS_RESEARCH_NOTES.md @@ -0,0 +1,405 @@ +# TTS Replacement Research Notes + +## Executive Summary + +This document summarizes research on replacing the existing TTS endpoint in makima with Qwen3-TTS-12Hz-0.6B-Base, with the goal of supporting voice cloning using Makima's Japanese voice speaking English, and achieving near-live/streaming TTS capabilities. + +--- + +## 1. Current TTS Implementation Analysis + +### 1.1 Current Model: Chatterbox-Turbo + +The existing TTS implementation in `makima/src/tts.rs` uses **ResembleAI/chatterbox-turbo-ONNX**: + +- **Architecture**: 4-component ONNX model pipeline + - `speech_encoder.onnx` - Encodes reference voice audio + - `embed_tokens.onnx` - Token embedding layer + - `language_model.onnx` - Autoregressive language model (24 layers, 16 KV heads, 64 head dim) + - `conditional_decoder.onnx` - Decodes speech tokens to audio waveform + +- **Sample Rate**: 24,000 Hz output +- **Voice Cloning**: Required (no default voice support) +- **Special Tokens**: + - START_SPEECH_TOKEN: 6561 + - STOP_SPEECH_TOKEN: 6562 + - SILENCE_TOKEN: 4299 + +### 1.2 Current API Surface + +**Core TTS Functions:** +```rust +pub fn generate_tts(&mut self, _text: &str) -> Result, TtsError> + // Returns VoiceRequired error - voice cloning is mandatory + +pub fn generate_tts_with_voice(&mut self, text: &str, sample_audio_path: &Path) -> Result, TtsError> + // Voice cloning from file path + +pub fn generate_tts_with_samples(&mut self, text: &str, samples: &[f32], sample_rate: u32) -> Result, TtsError> + // Voice cloning from raw samples +``` + +**Audio Processing:** +- Input audio resampled to 24kHz +- Reference voice encoded into: + - `audio_features` - Acoustic features + - `prompt_tokens` - Token representation + - `speaker_embeddings` - Speaker identity + - `speaker_features` - Voice characteristics + +### 1.3 Current Limitations + +1. **No Streaming Support**: Current implementation generates complete audio before returning +2. **No Default Voice**: Requires voice reference audio for every call +3. **No HTTP Endpoint**: TTS is only available as a Rust library, not exposed via REST API +4. **Single Language**: Optimized for English, unclear multilingual support +5. **High Latency**: Full autoregressive generation before any audio output + +### 1.4 Related Components + +**Audio Processing (`makima/src/audio.rs`):** +- Uses Symphonia for audio decoding (MP3, WAV, FLAC, OGG, etc.) +- Resampling via sinc interpolation +- Stereo to mono mixdown +- Target: 16kHz mono for STT + +**Listen Endpoint (`makima/src/server/handlers/listen.rs`):** +- WebSocket-based streaming STT +- Uses Parakeet for transcription +- Sortformer for speaker diarization +- Already has real-time audio streaming infrastructure + +--- + +## 2. Qwen3-TTS-12Hz-0.6B-Base Model Analysis + +### 2.1 Model Capabilities + +| Feature | Specification | +|---------|---------------| +| **Model Size** | 0.6B parameters (lightweight variant) | +| **Voice Cloning** | 3-second reference audio only | +| **Streaming** | Dual-track hybrid architecture | +| **Minimum Latency** | 97ms end-to-end | +| **Languages** | 10 (Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian) | +| **Cross-lingual Cloning** | Japanese voice to English speech supported | +| **Speaker Similarity** | 0.95 (near human-level) | +| **Output Sample Rate** | Up to 48kHz (standard 24kHz) | + +### 2.2 Voice Cloning Requirements + +**Reference Audio:** +- **Minimum Duration**: 3 seconds +- **Recommended Duration**: 5-15 seconds +- **Format**: WAV preferred; also supports URL, base64, numpy array +- **Quality**: Clean, noise-free audio essential +- **Transcript**: Providing `ref_text` significantly improves quality + +**Cross-Lingual Usage (Japanese to English):** +```python +ref_audio = "makima_japanese.wav" # Japanese reference +ref_text = "日本語のテキスト" # Japanese transcription + +wavs, sr = model.generate_voice_clone( + text="This is English text", # English output + language="English", + ref_audio=ref_audio, + ref_text=ref_text, +) +``` + +### 2.3 Technical Requirements + +**Python Dependencies:** +```bash +pip install -U qwen-tts +pip install -U flash-attn --no-build-isolation # For optimal performance +``` + +**Hardware:** +- CUDA-compatible GPU required +- FlashAttention 2 for optimal memory usage +- Float16/bfloat16 precision support +- For <96GB RAM: `MAX_JOBS=4` for flash-attn installation + +**Model Loading:** +```python +from qwen_tts import Qwen3TTSModel +import torch + +model = Qwen3TTSModel.from_pretrained( + "Qwen/Qwen3-TTS-12Hz-0.6B-Base", + device_map="cuda:0", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", +) +``` + +### 2.4 Streaming Architecture + +**Dual-Track Hybrid Design:** +- Single model supports both streaming and non-streaming +- Audio output begins after minimal text input +- 97ms minimum latency achieved through: + - Proprietary Qwen3-TTS-Tokenizer-12Hz (efficient acoustic compression) + - Discrete multi-codebook LM (eliminates LM+DiT bottleneck) + - Lightweight non-DiT vocoder + +**Reusable Voice Clone Prompt (Critical for Performance):** +```python +# Pre-compute once +prompt_items = model.create_voice_clone_prompt( + ref_audio=ref_audio, + ref_text=ref_text, + x_vector_only_mode=False +) + +# Reuse for multiple generations +wavs, sr = model.generate_voice_clone( + text=["Line 1", "Line 2"], + language=["English", "English"], + voice_clone_prompt=prompt_items, # Cached prompt +) +``` + +--- + +## 3. Makima Voice Audio Sources + +### 3.1 Character Information + +- **Character**: Makima from Chainsaw Man anime +- **Japanese Voice Actress**: Tomori Kusunoki (楠木ともり) +- **English Voice Actress**: Suzie Yeung + +### 3.2 Potential Audio Sources + +| Source | Type | Notes | +|--------|------|-------| +| **Voicy Network Soundboard** | Official clips | MP3 download available, 20+ sound effects | +| **101Soundboards** | Fan-curated clips | Various character sounds | +| **Anime Episodes** | Source material | Highest quality, requires extraction | +| **Nikke: Goddess of Victory** | Game voicelines | Same voice actress (Tomori Kusunoki) | +| **Ko-fi (erusha)** | WAV files | x5 character impression audio files | + +### 3.3 Recommended Approach + +1. **Primary Source**: Extract 5-15 seconds of clean dialogue from Chainsaw Man anime (Japanese audio track) +2. **Selection Criteria**: + - Clear, isolated dialogue (no background music/effects) + - Natural speaking tone (not shouting/whispering) + - Variety of phonemes for better cloning +3. **Transcription**: Provide accurate Japanese transcription for `ref_text` +4. **Processing**: Convert to WAV format, ensure clean audio quality + +### 3.4 Legal Considerations + +- Voice cloning of real voice actors for commercial use may have legal implications +- Synthetic voice generation based on copyrighted characters may require licenses +- Consider using for internal/personal use only, or creating disclaimer + +--- + +## 4. Feasibility Assessment + +### 4.1 Live/Streaming TTS Feasibility: HIGHLY FEASIBLE + +**Evidence:** +- Qwen3-TTS achieves 97ms latency (well under 200ms real-time threshold) +- Existing WebSocket infrastructure in makima (`/api/v1/listen`) can be adapted +- Streaming architecture designed for interactive scenarios + +**Implementation Approach:** +1. Create new WebSocket endpoint `/api/v1/speak` mirroring listen endpoint +2. Pre-compute voice clone prompt on connection start +3. Stream audio chunks as they're generated +4. Use chunked audio encoding (similar to listen's binary message handling) + +### 4.2 Voice Cloning with Japanese Voice: FULLY SUPPORTED + +**Evidence:** +- Qwen3-TTS explicitly supports cross-lingual voice cloning +- Japanese is one of 10 supported languages +- 0.95 speaker similarity maintained across languages + +**Implementation Approach:** +1. Pre-process Makima voice clips (5-15 seconds Japanese audio) +2. Include Japanese transcription +3. Generate English speech while preserving voice characteristics + +### 4.3 Integration Challenges + +| Challenge | Difficulty | Mitigation | +|-----------|-----------|------------| +| **Python to Rust Integration** | Medium | Use Python subprocess or microservice | +| **GPU Memory** | Low | 0.6B model is lightweight | +| **Latency Target** | Low | 97ms base latency is excellent | +| **Audio Format Conversion** | Low | Existing symphonia infrastructure | +| **Default Voice Setup** | Low | One-time voice prompt caching | + +### 4.4 Architecture Options + +**Option A: Python Microservice** +``` +[Makima Rust Server] --HTTP/WebSocket--> [Python TTS Service] + | + [Qwen3-TTS Model] +``` +Pros: Clean separation, easy Python integration +Cons: Network overhead, deployment complexity + +**Option B: PyO3 Rust Bindings** +``` +[Makima Rust Server] --FFI--> [pyo3 Python Bindings] --> [Qwen3-TTS] +``` +Pros: Single process, lower latency +Cons: Complex build, Python GIL issues + +**Option C: ONNX Export (Like Current Chatterbox)** +``` +[Makima Rust Server] --ort--> [Qwen3-TTS ONNX Models] +``` +Pros: Pure Rust, consistent with existing architecture +Cons: May not have ONNX export available for Qwen3-TTS + +**Recommended: Option A (Python Microservice)** +- Fastest time to implementation +- Aligns with Qwen3-TTS's native Python API +- Can use WebSocket for streaming audio chunks +- Easy to deploy alongside existing makima server + +--- + +## 5. Preliminary Technical Approach + +### 5.1 Phase 1: Python TTS Microservice + +```python +# tts_service.py +from fastapi import FastAPI, WebSocket +from qwen_tts import Qwen3TTSModel +import torch +import base64 + +app = FastAPI() +model = None +voice_prompt = None + +@app.on_event("startup") +async def load_model(): + global model, voice_prompt + model = Qwen3TTSModel.from_pretrained( + "Qwen/Qwen3-TTS-12Hz-0.6B-Base", + device_map="cuda:0", + dtype=torch.bfloat16, + ) + # Pre-load Makima voice + voice_prompt = model.create_voice_clone_prompt( + ref_audio="makima_voice.wav", + ref_text="日本語の台詞...", + ) + +@app.websocket("/ws/speak") +async def speak(websocket: WebSocket): + await websocket.accept() + while True: + text = await websocket.receive_text() + wavs, sr = model.generate_voice_clone( + text=text, + language="English", + voice_clone_prompt=voice_prompt, + ) + # Stream audio chunks + audio_bytes = wavs[0].tobytes() + await websocket.send_bytes(audio_bytes) +``` + +### 5.2 Phase 2: Rust Integration + +```rust +// makima/src/server/handlers/speak.rs +pub async fn websocket_handler( + ws: WebSocketUpgrade, + State(state): State, +) -> Response { + ws.on_upgrade(|socket| handle_speak_socket(socket, state)) +} + +async fn handle_speak_socket(socket: WebSocket, state: SharedState) { + // Connect to Python TTS service + let tts_ws = tokio_tungstenite::connect_async("ws://localhost:8001/ws/speak").await?; + + // Forward text to TTS, stream audio back to client + // ... +} +``` + +### 5.3 API Design + +**WebSocket Endpoint: `/api/v1/speak`** + +**Client to Server Messages:** +```json +{ + "type": "start", + "sample_rate": 24000, + "encoding": "pcm16" +} + +{ + "type": "speak", + "text": "Hello, I am Makima." +} + +{ + "type": "stop" +} +``` + +**Server to Client Messages:** +```json +{ + "type": "ready", + "session_id": "uuid" +} + +{ + "type": "audio_chunk", + "data": "", + "is_final": false +} + +{ + "type": "complete" +} +``` + +--- + +## 6. Next Steps + +### Immediate Actions +1. [ ] Obtain Makima voice clips (5-15 seconds clean Japanese audio) +2. [ ] Create Japanese transcription of voice clips +3. [ ] Test Qwen3-TTS voice cloning with Makima samples +4. [ ] Benchmark latency on target hardware + +### Development Phases +1. **Phase 1**: Python TTS microservice proof-of-concept +2. **Phase 2**: WebSocket streaming integration +3. **Phase 3**: Rust proxy endpoint in makima +4. **Phase 4**: Listen page integration for bidirectional speech + +### Hardware Requirements +- CUDA-compatible GPU (minimum) +- Recommended: 8GB+ VRAM for 0.6B model with FlashAttention 2 +- Python 3.12+ environment + +--- + +## References + +- [Qwen3-TTS-12Hz-0.6B-Base on HuggingFace](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-Base) +- [Qwen3-TTS GitHub Repository](https://github.com/QwenLM/Qwen3-TTS) +- [Behind The Voice Actors - Makima](https://www.behindthevoiceactors.com/tv-shows/Chainsaw-Man/Makima/) +- [Voicy Network Chainsaw Man Soundboard](https://www.voicy.network/official-soundboards/anime/chainsaw-man) diff --git a/docs/research/rust-native-tts-research.md b/docs/research/rust-native-tts-research.md new file mode 100644 index 0000000..5bc75f7 --- /dev/null +++ b/docs/research/rust-native-tts-research.md @@ -0,0 +1,297 @@ +# Rust-Native Qwen3-TTS Integration Research + +## Executive Summary + +This document researches integrating **Qwen3-TTS-12Hz-0.6B-Base** directly into the makima Rust codebase, replacing the current Chatterbox TTS implementation. The goal is a **pure Rust** solution — no Python, no separate microservice. + +**Bottom line:** A Rust-native integration is feasible but requires significant implementation work. The most viable path is using **candle** (HuggingFace's Rust ML framework) to implement the model architecture natively, loading safetensors weights directly. The existing ONNX-based approach used for Chatterbox TTS is **not viable** for Qwen3-TTS due to architecture incompatibilities with ONNX exporters. + +--- + +## 1. Current TTS Implementation Analysis + +The existing Chatterbox TTS in `makima/src/tts.rs` uses: + +- **ONNX Runtime** via the `ort` crate (v2.0.0-rc.10) +- **Four ONNX model files**: `speech_encoder.onnx`, `embed_tokens.onnx`, `language_model.onnx`, `conditional_decoder.onnx` +- **tokenizers** crate for text tokenization +- **ndarray** for tensor manipulation +- **hf-hub** for model downloading +- **Pipeline**: encode voice → tokenize text → autoregressive token generation with KV cache → decode tokens to waveform +- **Architecture constants**: 24 layers, 16 KV heads, 64 head dim, 24kHz sample rate + +The pattern is well-established: download ONNX models from HuggingFace, load sessions, run inference with manual KV cache management. + +### STT Pattern (listen.rs) + +The Listen/STT handler in `makima/src/server/handlers/listen.rs` demonstrates the broader ML pattern: +- WebSocket-based streaming +- Lazy model loading via `SharedState::get_ml_models()` +- Models held behind `tokio::sync::Mutex` for async access +- `parakeet-rs` local crate for STT, `sortformer` for diarization +- All models are Rust-native with ONNX backends + +--- + +## 2. Qwen3-TTS-12Hz-0.6B-Base Architecture + +### Model Overview + +| Property | Value | +|----------|-------| +| **Parameters** | 0.6B | +| **Architecture** | `Qwen3TTSForConditionalGeneration` | +| **Output Sample Rate** | 24,000 Hz | +| **Token Frame Rate** | 12.5 Hz (~80ms per token) | +| **Model Format** | SafeTensors (1.83 GB main + 682 MB tokenizer) | +| **Total Size** | ~2.52 GB | +| **Precision** | bfloat16/float16 | + +### Components + +The model has **three distinct components**: + +#### A. Main Language Model (Talker) — 1.83 GB safetensors +- Hidden size: 1024 +- Layers: 28 +- Attention heads: 16 (8 KV heads) +- Intermediate size: 3072 +- Head dimension: 128 +- Text vocab size: 151,936 +- Max position embeddings: 32,768 +- Autoregressive transformer predicting speech token sequences from text + +#### B. Code Predictor (Multi-Token Prediction) — embedded in main model +- Hidden size: 1024 +- Layers: 5 +- Attention heads: 16 +- Number of code groups: 16 +- Codebook vocab size: 2048 +- Predicts residual codebooks (16 layers) after the main LM predicts the zeroth codebook + +#### C. Speech Tokenizer (Qwen3-TTS-Tokenizer-12Hz) — 682 MB safetensors +- Separate model in `speech_tokenizer/` directory +- GAN-based codec: encoder + decoder +- 16-layer multi-codebook RVQ (Residual Vector Quantization) +- First codebook: semantic (WavLM-guided) +- Remaining 15: acoustic details +- **Decoder**: lightweight causal ConvNet (no DiT/diffusion needed) +- Encodes reference audio → discrete codes, decodes codes → waveform + +### Inference Pipeline + +``` +Text Input + Reference Audio + ↓ +[Speech Tokenizer Encoder] → reference audio codes + speaker embedding + ↓ +[Text Tokenizer] → text token IDs + ↓ +[Language Model] → autoregressive generation of zeroth codebook tokens + ↓ +[Code Predictor / MTP] → predict remaining 15 codebook layers + ↓ +[Speech Tokenizer Decoder / Causal ConvNet] → waveform output (24kHz) +``` + +--- + +## 3. ONNX Export Feasibility — NOT VIABLE + +### Status: No ONNX support exists + +- **No official ONNX export** from Qwen team +- **No community ONNX conversion** for Qwen3-TTS +- The Qwen3 architecture is **not supported** by HuggingFace Optimum's ONNX exporter +- Users attempting export get: `ValueError: Trying to export a qwen3 model, that is a custom or unsupported architecture, but no custom onnx configuration was passed` +- Even for base Qwen3 LLMs (non-TTS), ONNX export has significant issues with MoE routing, hybrid attention, and novel architecture components + +### Why ONNX Won't Work for Qwen3-TTS + +1. **Custom architecture** — `Qwen3TTSForConditionalGeneration` is not a standard transformer; it combines LM + code predictor + speech tokenizer +2. **Multi-codebook MTP module** — the code predictor generates 16 codebook layers, a non-standard operation +3. **Causal ConvNet decoder** — the speech tokenizer's decoder is a custom GAN-trained ConvNet, not a standard vocoder +4. **Dynamic control flow** — dual-track streaming architecture with conditional branching +5. **No Optimum support** — would require writing a custom ONNX config from scratch for each sub-component + +**Verdict: The ONNX path (matching our Chatterbox approach) is a dead end for Qwen3-TTS.** + +--- + +## 4. Rust-Native Inference Options + +### Option A: Candle (HuggingFace) — RECOMMENDED + +[candle](https://github.com/huggingface/candle) is HuggingFace's minimalist Rust ML framework. + +**Why candle is the best fit:** + +| Factor | Assessment | +|--------|------------| +| **Qwen model support** | ✅ Has `qwen2` module in candle-transformers; Qwen3 variants supported | +| **SafeTensors loading** | ✅ Native first-class support (safetensors is a Rust crate) | +| **GPU support** | ✅ CUDA backend, Metal (macOS), CPU with MKL | +| **Tokenizer support** | ✅ Uses the same `tokenizers` crate makima already depends on | +| **Audio models** | ✅ Supports EnCodec, Whisper, MetaVoice, Parler-TTS | +| **KV cache** | ✅ Well-established patterns in existing model implementations | +| **Community** | ✅ Active; Crane project already lists Qwen3-TTS as "highest priority" | +| **Binary size** | ✅ Compiles to single binary, no Python dependency | + +**What needs to be implemented:** + +1. **Qwen3-TTS transformer layers** — extend existing `qwen2` model code for the 28-layer LM with TTS-specific modifications (speaker encoder concatenation, code predictor output heads) +2. **Code Predictor (MTP)** — 5-layer module that generates 16 codebook predictions from the LM hidden states +3. **Speech Tokenizer Encoder** — ConvNet encoder that converts reference audio to discrete multi-codebook tokens + speaker embeddings +4. **Speech Tokenizer Decoder** — causal ConvNet that reconstructs waveforms from discrete codes +5. **Multi-codebook handling** — manage 16 parallel codebook sequences + +**Estimated effort:** Medium-High. The LM backbone can reuse existing Qwen2/3 code. The speech tokenizer (encoder + decoder) is the most novel component. + +**Key crate dependencies to add:** +```toml +candle-core = "0.8" +candle-nn = "0.8" +candle-transformers = "0.8" +# Keep existing: tokenizers, hf-hub, ndarray (for compatibility) +``` + +### Option B: Crane (Candle-based TTS Engine) + +[Crane](https://github.com/lucasjinreal/Crane) is a pure Rust LLM inference engine built on candle, specifically designed for multi-modal models including TTS. + +**Key facts:** +- Already supports Spark-TTS (codec-based TTS with similar architecture) +- **Qwen3-TTS is listed as "Highest Priority" on their roadmap** +- Handles multi-module architectures (codec + LLM pipelines) +- Supports Qwen2.5, Moonshine ASR +- Claims 50x faster than PyTorch on Apple Silicon + +**Strategy:** Monitor Crane's Qwen3-TTS implementation. If they ship it, we could either: +- Use Crane as a dependency directly +- Port their implementation into makima's codebase +- Contribute to Crane and depend on it + +**Risk:** Crane is a relatively new project; depending on it adds supply chain risk. + +### Option C: qwen3-rs (Educational Reference) + +[qwen3-rs](https://github.com/reinterpretcat/qwen3-rs) is an educational project implementing Qwen3 inference from scratch in Rust. + +**Useful for:** Reference implementation of Qwen3 transformer layers, tokenization, KV cache, and safetensors loading — all without heavy ML framework dependencies. However, it only implements the base LLM, not the TTS-specific components. + +### Option D: Direct ort (ONNX Runtime) with Custom Export — FALLBACK + +If we could manually export each sub-component to ONNX: + +1. Export the 28-layer LM backbone (similar complexity to Chatterbox) +2. Export the code predictor separately +3. Export the speech tokenizer encoder/decoder + +This would match our existing Chatterbox pattern but requires Python scripting for the one-time export, and the Qwen3 architecture is explicitly unsupported by standard exporters. **Not recommended unless ONNX support materializes upstream.** + +### Option E: PyTorch C++ (libtorch) via FFI — NOT RECOMMENDED + +Using libtorch via Rust FFI bindings (`tch-rs` crate). This would: +- Add a ~2GB libtorch dependency +- Require complex build setup +- Introduce C++ dependency management +- Defeat the purpose of a pure Rust solution + +--- + +## 5. Recommended Approach + +### Phase 1: Candle-Based Implementation + +**Architecture:** + +``` +makima/src/tts/ +├── mod.rs // TTS trait + factory (select Chatterbox vs Qwen3) +├── chatterbox.rs // Existing ONNX-based Chatterbox (moved from tts.rs) +├── qwen3/ +│ ├── mod.rs // Qwen3TTS public API +│ ├── model.rs // Qwen3 LM transformer (28 layers) +│ ├── code_predictor.rs // MTP module (5 layers, 16 codebooks) +│ ├── speech_tokenizer.rs // Encoder + Decoder (causal ConvNet) +│ ├── config.rs // Model config from config.json +│ └── generate.rs // Autoregressive generation loop with KV cache +``` + +**Key implementation details:** + +1. **Load safetensors directly** — candle's `safetensors` support reads the 1.83GB main model and 682MB speech tokenizer +2. **Reuse Qwen2 attention** — candle-transformers already has `qwen2::Model` with RoPE, GQA, and KV cache +3. **Implement ConvNet codec** — the speech tokenizer's encoder/decoder is a causal 1D ConvNet; candle has `Conv1d` layers +4. **Multi-codebook RVQ** — implement the 16-codebook residual vector quantization lookup +5. **Speaker embedding** — extract from reference audio via the speech tokenizer encoder +6. **Streaming support** — the 12Hz model's causal architecture enables token-by-token waveform generation + +### Phase 2: Voice Assets + +The model supports voice cloning with reference audio. For the default Makima voice: +- Need 5-15 second Japanese-accented English audio clip +- Reference audio + transcript fed to speech tokenizer encoder +- Speaker embedding cached for reuse + +### Phase 3: Integration with Listen Page + +Following the pattern in `listen.rs`: +- TTS model loaded lazily via `SharedState` +- Protected behind `tokio::sync::Mutex` (or `RwLock` for concurrent reads) +- WebSocket endpoint for streaming TTS (emit audio chunks as tokens are generated) +- Bidirectional: STT (listen) → process → TTS (speak) loop + +--- + +## 6. Comparison Matrix + +| Criteria | ONNX (current pattern) | Candle | Crane | libtorch | +|----------|----------------------|--------|-------|----------| +| Pure Rust | ✅ (ort crate) | ✅ | ✅ | ❌ (C++ FFI) | +| Qwen3-TTS support | ❌ No export | ⚠️ Needs impl | ⚠️ Planned | ✅ (full PyTorch) | +| Single binary | ✅ | ✅ | ✅ | ❌ | +| GPU acceleration | ✅ | ✅ | ✅ | ✅ | +| SafeTensors loading | ❌ (needs ONNX) | ✅ | ✅ | ✅ | +| Streaming TTS | ✅ | ✅ | ✅ | ✅ | +| Maintenance burden | Low | Medium | Low (if adopted) | High | +| Implementation effort | N/A (blocked) | Medium-High | Low (if available) | Medium | +| Dependency size | ~50MB | ~5MB | ~5MB | ~2GB | + +--- + +## 7. Risk Assessment + +| Risk | Likelihood | Impact | Mitigation | +|------|-----------|--------|------------| +| Candle implementation takes longer than expected | Medium | Medium | Reference Crane's Spark-TTS implementation; use qwen3-rs as LM reference | +| Speech tokenizer ConvNet is complex to port | Medium | High | Study the PyTorch source in qwen-tts package; ConvNet layers are simpler than transformers | +| Model quality differs from reference PyTorch | Low | High | Validate with reference audio samples; ensure bfloat16 precision | +| Crane ships Qwen3-TTS before we finish | Medium | Positive | Adopt their implementation | +| GPU memory issues on target hardware | Low | Medium | 0.6B model is small (~2.5GB); fits in 4GB VRAM with float16 | + +--- + +## 8. Next Steps + +1. **Immediate:** Add `candle-core`, `candle-nn`, `candle-transformers` to Cargo.toml +2. **Week 1:** Implement Qwen3 LM backbone in candle (extend existing qwen2 model) +3. **Week 2:** Implement speech tokenizer encoder/decoder (ConvNet + RVQ) +4. **Week 2:** Implement code predictor (MTP module) +5. **Week 3:** Integration testing with reference audio; validate output quality +6. **Week 3:** Wire into makima server as TTS endpoint +7. **Ongoing:** Monitor Crane project for Qwen3-TTS implementation + +--- + +## Sources + +- [Qwen3-TTS-12Hz-0.6B-Base on HuggingFace](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-Base) +- [Qwen3-TTS Technical Report (arXiv)](https://arxiv.org/html/2601.15621v1) +- [Qwen3-TTS GitHub Repository](https://github.com/QwenLM/Qwen3-TTS) +- [Candle — HuggingFace Rust ML Framework](https://github.com/huggingface/candle) +- [Crane — Rust LLM Inference Engine](https://github.com/lucasjinreal/Crane) +- [qwen3-rs — Educational Qwen3 Rust Implementation](https://github.com/reinterpretcat/qwen3-rs) +- [candle-transformers Qwen2 model](https://docs.rs/candle-transformers/latest/candle_transformers/models/qwen2/index.html) +- [Qwen3-TTS-Tokenizer-12Hz on HuggingFace](https://huggingface.co/Qwen/Qwen3-TTS-Tokenizer-12Hz) +- [ONNX export issues for Qwen3](https://huggingface.co/onnx-community/Qwen3-1.7B-ONNX/discussions/1) diff --git a/docs/research/tts-qwen3-research.md b/docs/research/tts-qwen3-research.md new file mode 100644 index 0000000..a961b4f --- /dev/null +++ b/docs/research/tts-qwen3-research.md @@ -0,0 +1,548 @@ +# TTS Research: Qwen3-TTS-12Hz-0.6B-Base Integration + +## Executive Summary + +This document evaluates replacing the current Chatterbox TTS implementation with Qwen3-TTS-12Hz-0.6B-Base for the makima system. The goal is to enable near-real-time voice synthesis with voice cloning capabilities, defaulting to Makima's Japanese voice (Tomori Kusunoki) speaking English. + +**Key Findings:** +- Qwen3-TTS offers superior streaming capabilities (~97ms latency) compared to the current batch-only Chatterbox implementation +- Voice cloning requires only 3 seconds of reference audio +- No official ONNX export exists; Python/PyTorch inference required +- The 0.6B model is optimized for resource-constrained environments + +--- + +## 1. Current TTS Implementation Analysis + +### 1.1 Architecture Overview + +The current implementation uses **Chatterbox-Turbo-ONNX** from ResembleAI: + +``` +Location: makima/src/tts.rs +Model ID: ResembleAI/chatterbox-turbo-ONNX +Sample Rate: 24,000 Hz +``` + +**Components:** +| Component | File | Purpose | +|-----------|------|---------| +| `speech_encoder.onnx` | ~XX MB | Encodes reference audio to speaker embeddings | +| `embed_tokens.onnx` | ~XX MB | Token embedding layer | +| `language_model.onnx` | ~XX MB | Autoregressive text-to-speech token generation | +| `conditional_decoder.onnx` | ~XX MB | Converts speech tokens to waveform | +| `tokenizer.json` | ~KB | Text tokenization | + +### 1.2 Current API Surface + +```rust +pub struct ChatterboxTTS { + speech_encoder: Session, + embed_tokens: Session, + language_model: Session, + conditional_decoder: Session, + tokenizer: Tokenizer, +} + +impl ChatterboxTTS { + // Load from pretrained models + pub fn from_pretrained(model_dir: Option<&str>) -> Result; + + // Generate speech (requires voice reference) + pub fn generate_tts(&mut self, _text: &str) -> Result, TtsError>; + + // Voice cloning from file path + pub fn generate_tts_with_voice( + &mut self, + text: &str, + sample_audio_path: &Path, + ) -> Result, TtsError>; + + // Voice cloning from raw samples + pub fn generate_tts_with_samples( + &mut self, + text: &str, + samples: &[f32], + sample_rate: u32, + ) -> Result, TtsError>; +} +``` + +### 1.3 Current Capabilities + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Voice Cloning | **Yes** | Required for all synthesis | +| Streaming | **No** | Batch generation only | +| Languages | Limited | English-focused | +| ONNX Runtime | **Yes** | CPU inference via `ort` crate | +| GPU Acceleration | Partial | ONNX supports CUDA EP | +| Real-time Factor | Unknown | Not benchmarked | + +### 1.4 Integration Points + +The TTS module is currently: +- Exposed as `pub mod tts` in `lib.rs` +- Used in `main.rs` for testing +- **Not integrated with the web server** (no `/api/v1/tts` endpoint) + +The audio processing infrastructure is shared with the Listen (STT) module: +- `audio.rs` provides format conversion utilities +- `symphonia` for decoding various audio formats +- Resampling to target sample rates (16kHz for STT, 24kHz for TTS) + +--- + +## 2. Qwen3-TTS-12Hz-0.6B-Base Analysis + +### 2.1 Model Overview + +**Source:** [Hugging Face - Qwen/Qwen3-TTS-12Hz-0.6B-Base](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-Base) + +| Specification | Value | +|---------------|-------| +| Parameters | 0.6B | +| Release Date | January 22, 2026 | +| Architecture | Dual-Track hybrid streaming LM | +| Tokenizer | Qwen3-TTS-Tokenizer-12Hz | +| Frame Rate | 12.5 Hz | +| Output Sample Rate | 24 kHz | +| Languages | 10 (Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian) | + +### 2.2 Key Features + +| Feature | Status | Details | +|---------|--------|---------| +| **Voice Cloning** | Yes | 3-second minimum reference audio | +| **Streaming** | Yes | 97ms end-to-end latency | +| **Real-time** | Yes | First audio packet after single character | +| **Multilingual** | Yes | 10 languages supported | +| **Instruction Control** | No | Base model limitation | + +### 2.3 Streaming Architecture + +The Dual-Track architecture enables: +1. **Streaming text input** - Processes text incrementally +2. **Streaming audio output** - Emits audio packets as generated +3. **Multi-Token Prediction (MTP)** - Enables immediate speech decoding from first codec frame + +**Latency Benchmarks:** +- First token latency: ~97ms (end-to-end) +- Optimized TTFT: ~170ms on RTX 5090 (community fork) +- Initial implementations: ~800ms TTFT (before optimization) + +### 2.4 Voice Cloning Requirements + +| Requirement | Specification | +|-------------|---------------| +| Reference Audio Length | **3 seconds minimum** | +| Audio Format | WAV, MP3, or common formats | +| Input Methods | File path, URL, base64, numpy array | +| Reference Text | **Required** (transcript of reference audio) | +| X-Vector Only Mode | Optional (speaker embedding only, lower quality) | + +### 2.5 Python API + +```python +from qwen_tts import Qwen3TTSModel + +# Load model +model = Qwen3TTSModel.from_pretrained( + "Qwen/Qwen3-TTS-12Hz-0.6B-Base", + device_map="cuda:0", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", +) + +# Voice cloning +wavs, sr = model.generate_voice_clone( + text="Hello, this is a test.", + language="English", + ref_audio="reference.wav", + ref_text="Original speaker text from reference", +) + +# Reusable prompt (efficient for multiple generations) +prompt = model.create_voice_clone_prompt( + ref_audio="reference.wav", + ref_text="Reference transcript", +) + +wavs, sr = model.generate_voice_clone( + text="New text", + language="English", + voice_clone_prompt=prompt, +) +``` + +### 2.6 Dependencies + +``` +pip install -U qwen-tts +pip install -U flash-attn --no-build-isolation # Optional, recommended +``` + +**Requirements:** +- Python 3.12 recommended +- CUDA-capable GPU (for optimal performance) +- FlashAttention 2 compatible hardware +- PyTorch with bfloat16 support + +--- + +## 3. Feasibility Assessment + +### 3.1 Streaming/Live TTS Feasibility + +**Assessment: FEASIBLE with caveats** + +| Factor | Current State | Path Forward | +|--------|---------------|--------------| +| Streaming API | Experimental (community fork) | Use [dffdeeq/Qwen3-TTS-streaming](https://github.com/dffdeeq/Qwen3-TTS-streaming) or wait for official support | +| Latency Target | 97ms advertised | Achievable with optimization | +| First Token | ~170ms optimized | Acceptable for conversational use | +| Audio Stability | First 1-2s may have timbre issues | Known limitation, may need buffering | + +**Streaming Implementation Status:** +- Official repository: Streaming documented but **not released** +- Community fork: Working implementation with ~170ms TTFT +- vLLM-Omni: Offline inference only (online serving planned) + +### 3.2 Voice Cloning for Makima + +**Assessment: FULLY FEASIBLE** + +Requirements for Makima voice cloning: +1. **3+ seconds of clean audio** - Tomori Kusunoki (Japanese VA) speaking +2. **Transcript of the audio** - Required for full quality +3. **Audio format** - WAV/MP3 acceptable + +**Audio Sources:** +- Chainsaw Man anime clips +- Voice actress promotional material +- Behind The Voice Actors database + +**Considerations:** +- Japanese VA speaking English may work with explicit `language="English"` setting +- May need English-speaking Makima clips (Suzie Yeung, English dub VA) as fallback +- X-vector mode available if transcript is difficult to obtain + +### 3.3 Integration Complexity + +| Component | Complexity | Notes | +|-----------|------------|-------| +| Model Loading | Medium | Python subprocess or PyO3 bridge required | +| Streaming API | High | WebSocket integration needed | +| Voice Caching | Low | `create_voice_clone_prompt()` supports this | +| Audio Format | Low | Both use 24kHz output | +| ONNX Migration | N/A | **No ONNX export available** | + +### 3.4 ONNX vs Python Inference + +**Current approach (Chatterbox):** Rust + ONNX Runtime +- Pros: Native Rust, low latency, CPU-friendly +- Cons: Limited model ecosystem, no streaming + +**Required approach (Qwen3-TTS):** Python + PyTorch +- Pros: Full model access, streaming support, GPU acceleration +- Cons: Python subprocess overhead, dependency management + +**Integration Options:** + +1. **Python Subprocess/Service** + - Run `qwen-tts` as separate Python service + - Communicate via HTTP/WebSocket + - Cleanest separation, easiest to implement + +2. **PyO3 Bridge** + - Embed Python in Rust binary + - Higher complexity, tighter integration + - May have GIL contention issues + +3. **Custom ONNX Export** (Future) + - Not currently available + - Would require community effort + - No timeline from Qwen team + +**Recommendation:** Python service with WebSocket streaming + +--- + +## 4. Audio Clip Requirements + +### 4.1 For Voice Cloning Setup + +| Requirement | Specification | +|-------------|---------------| +| Minimum Duration | 3 seconds | +| Recommended Duration | 5-10 seconds | +| Format | WAV (preferred), MP3 | +| Sample Rate | Any (will be resampled) | +| Content | Clear speech, minimal background noise | +| Transcript | Required for full quality | + +### 4.2 Makima Voice Sources + +**Priority 1: Japanese VA (Tomori Kusunoki) speaking Japanese** +- Source: Chainsaw Man anime +- Challenge: Need clear dialogue without music/SFX +- Fallback: May not transfer well to English output + +**Priority 2: English VA (Suzie Yeung)** +- Source: Chainsaw Man English dub +- Advantage: Native English speaker for English output +- Trade-off: Different vocal characteristics from Japanese VA + +**Recommended Approach:** +1. Extract 5-10 second clips of both VAs +2. Test voice cloning quality with each +3. Select based on English speech naturalness +4. Store reference audio + transcript in `models/voices/makima/` + +### 4.3 Transcript Requirements + +For optimal voice cloning: +``` +ref_audio: "models/voices/makima/makima-reference.wav" +ref_text: "The exact words spoken in the reference audio" +``` + +X-vector fallback (lower quality, no transcript needed): +```python +prompt = model.create_voice_clone_prompt( + ref_audio="reference.wav", + x_vector_only_mode=True, # No transcript required +) +``` + +--- + +## 5. Preliminary Technical Approach + +### 5.1 Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Makima Server (Rust) │ +├─────────────────────────────────────────────────────────────┤ +│ ┌─────────────┐ ┌─────────────┐ ┌──────────────────────┐│ +│ │ Listen (STT)│ │ TTS Proxy │ │ Chat/Contract APIs ││ +│ │ /api/v1/ │ │ /api/v1/tts │ │ /api/v1/... ││ +│ │ listen │ │ │ │ ││ +│ └──────┬──────┘ └──────┬──────┘ └──────────────────────┘│ +│ │ │ │ +│ │ ┌──────▼──────┐ │ +│ │ │ WebSocket │ │ +│ │ │ Bridge │ │ +│ │ └──────┬──────┘ │ +└─────────┼────────────────┼──────────────────────────────────┘ + │ │ + │ ┌──────▼──────┐ + │ │ Python TTS │ + │ │ Service │ + │ │ (Qwen3-TTS) │ + │ └─────────────┘ + │ + ┌──────▼──────┐ + │ ML Models │ + │ (Parakeet, │ + │ Sortformer) │ + └─────────────┘ +``` + +### 5.2 Python TTS Service + +**Proposed Architecture:** + +```python +# tts_service.py +import asyncio +from fastapi import FastAPI, WebSocket +from qwen_tts import Qwen3TTSModel + +app = FastAPI() +model = None +voice_prompts = {} + +@app.on_event("startup") +async def load_model(): + global model + model = Qwen3TTSModel.from_pretrained( + "Qwen/Qwen3-TTS-12Hz-0.6B-Base", + device_map="cuda:0", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + ) + + # Pre-load Makima voice prompt + voice_prompts["makima"] = model.create_voice_clone_prompt( + ref_audio="models/voices/makima/reference.wav", + ref_text="[Makima reference transcript]", + ) + +@app.websocket("/ws/tts") +async def tts_stream(websocket: WebSocket): + await websocket.accept() + while True: + data = await websocket.receive_json() + text = data["text"] + voice = data.get("voice", "makima") + language = data.get("language", "English") + + # Generate with streaming (when available) + prompt = voice_prompts.get(voice) + wavs, sr = model.generate_voice_clone( + text=text, + language=language, + voice_clone_prompt=prompt, + ) + + # Send audio chunks + await websocket.send_bytes(wavs[0].tobytes()) + +@app.post("/api/tts") +async def tts_batch(request: TTSRequest): + # Batch fallback for non-streaming use cases + ... +``` + +### 5.3 Rust Integration + +**New Endpoint: `/api/v1/tts`** + +```rust +// server/handlers/tts.rs +pub async fn tts_websocket_handler( + ws: WebSocketUpgrade, + State(state): State, +) -> Response { + ws.on_upgrade(|socket| handle_tts_socket(socket, state)) +} + +async fn handle_tts_socket(socket: WebSocket, state: SharedState) { + // Proxy WebSocket to Python TTS service + let tts_client = state.tts_client.lock().await; + + let (mut sender, mut receiver) = socket.split(); + + while let Some(msg) = receiver.next().await { + match msg { + Ok(Message::Text(text)) => { + // Forward to Python service + let response = tts_client.generate(text).await; + + // Stream audio back + for chunk in response.audio_chunks { + sender.send(Message::Binary(chunk)).await.ok(); + } + } + _ => {} + } + } +} +``` + +### 5.4 Voice Prompt Caching + +```rust +// Pre-computed voice prompts stored in state +pub struct TtsConfig { + pub default_voice: String, + pub voices: HashMap, +} + +pub struct VoicePrompt { + pub name: String, + pub ref_audio_path: PathBuf, + pub ref_text: String, + pub language: String, + // Cached prompt from Python service + pub cached_prompt_id: Option, +} +``` + +### 5.5 Message Protocol + +**Client -> Server:** +```json +{ + "type": "synthesize", + "text": "Hello, I am Makima.", + "voice": "makima", + "language": "English", + "stream": true +} +``` + +**Server -> Client:** +```json +// Audio chunk +{"type": "audio", "data": "", "sample_rate": 24000, "final": false} + +// Completion +{"type": "complete", "duration_ms": 1234} + +// Error +{"type": "error", "code": "SYNTHESIS_ERROR", "message": "..."} +``` + +--- + +## 6. Implementation Phases + +### Phase 1: Research & Setup (Current) +- [x] Analyze current TTS implementation +- [x] Research Qwen3-TTS capabilities +- [x] Document feasibility and approach +- [ ] Acquire Makima voice reference clips +- [ ] Test voice cloning quality + +### Phase 2: Python Service +- [ ] Create Python TTS service with FastAPI +- [ ] Implement batch TTS endpoint +- [ ] Implement WebSocket streaming (when available) +- [ ] Add voice prompt management +- [ ] GPU optimization with FlashAttention 2 + +### Phase 3: Rust Integration +- [ ] Add TTS proxy endpoints to makima server +- [ ] WebSocket bridge implementation +- [ ] Voice configuration management +- [ ] Error handling and fallbacks + +### Phase 4: Production Ready +- [ ] Health checks for Python service +- [ ] Voice prompt caching optimization +- [ ] Latency benchmarking +- [ ] Integration with Listen page + +--- + +## 7. Open Questions + +1. **Streaming API Availability**: When will official streaming support be released? + - Fallback: Use community fork or batch with chunked responses + +2. **Voice Quality**: How well does Japanese VA voice clone to English speech? + - Action: Test with both Japanese and English VA samples + +3. **GPU Requirements**: What's the minimum VRAM for 0.6B model? + - Estimate: ~2-4GB with bf16 quantization + +4. **Latency Target**: What's acceptable for "close to live" TTS? + - Proposal: <500ms first audio, <100ms subsequent chunks + +5. **Transcript Acquisition**: How to obtain accurate transcripts for voice clips? + - Options: Manual transcription, Whisper ASR, community resources + +--- + +## 8. References + +- [Qwen3-TTS-12Hz-0.6B-Base (Hugging Face)](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-Base) +- [Qwen3-TTS GitHub Repository](https://github.com/QwenLM/Qwen3-TTS) +- [Qwen3-TTS Technical Report (arXiv)](https://arxiv.org/abs/2601.15621) +- [Streaming Inference Issue #77](https://github.com/QwenLM/Qwen3-TTS/issues/77) +- [Community Streaming Fork](https://github.com/dffdeeq/Qwen3-TTS-streaming) +- [Makima Voice Actors](https://www.behindthevoiceactors.com/characters/Chainsaw-Man/Makima/) +- [Chatterbox-Turbo-ONNX (Current Model)](https://huggingface.co/ResembleAI/chatterbox-turbo-ONNX) diff --git a/docs/specs/qwen3-tts-spec.md b/docs/specs/qwen3-tts-spec.md new file mode 100644 index 0000000..91d447d --- /dev/null +++ b/docs/specs/qwen3-tts-spec.md @@ -0,0 +1,928 @@ +# Qwen3-TTS Integration Specification + +**Version:** 2.0 +**Date:** 2026-01-27 +**Status:** Draft +**Author:** Makima Engineering + +## Table of Contents + +1. [Overview](#1-overview) +2. [Functional Requirements](#2-functional-requirements) +3. [Non-Functional Requirements](#3-non-functional-requirements) +4. [Architecture Specification](#4-architecture-specification) +5. [API Contract](#5-api-contract) +6. [Voice Asset Requirements](#6-voice-asset-requirements) +7. [Testing Strategy](#7-testing-strategy) +8. [Implementation Phases](#8-implementation-phases) +9. [Appendix](#appendix) + +--- + +## 1. Overview + +### 1.1 Purpose + +This specification defines the integration of Qwen3-TTS-12Hz-0.6B-Base as a replacement for the existing Chatterbox-Turbo TTS implementation in the makima system. The new implementation is a **pure Rust** solution using the **candle** ML framework — no Python, no separate microservice. The TTS model runs directly inside the main makima process. The implementation will provide: + +- **Streaming TTS** with near-real-time audio synthesis +- **Voice cloning** with default Makima voice (Japanese voice actress speaking English) +- **Bidirectional speech integration** for the Listen page +- **WebSocket-based streaming API** for low-latency delivery + +### 1.2 Background + +The current TTS implementation (Chatterbox-Turbo-ONNX) has limitations: +- No streaming support (batch-only generation) +- No HTTP/WebSocket endpoint exposed +- High latency for interactive use cases + +Qwen3-TTS offers significant improvements: +- **97ms** end-to-end latency (vs. batch processing) +- **10 languages** supported including Japanese cross-lingual cloning +- **3-second** reference audio for voice cloning +- **Dual-track streaming architecture** + +### 1.3 Scope + +This specification covers: +- WebSocket endpoint `/api/v1/speak` for streaming TTS +- Pure Rust candle-based model inference running in-process +- Voice asset management +- Testing and benchmarking + +Out of scope: +- ONNX export of Qwen3-TTS (not available) +- Instruction-following TTS features (base model only) +- Full replacement of STT/Listen functionality + +--- + +## 2. Functional Requirements + +### 2.1 WebSocket Endpoint: `/api/v1/speak` + +The TTS service SHALL be exposed via a WebSocket endpoint at `/api/v1/speak` for streaming audio synthesis. + +#### 2.1.1 Connection Flow + +``` +Client Server (Rust/Axum) + | | + |--- WS Connect ------------------>| + | | [Load TTS model lazily if needed] + |<-- Ready (session_id) -----------| + | | + |--- Start (config) -------------->| + |<-- Started ----------------------| [Load voice prompt] + | | + |--- Speak (text) ---------------->| [Direct candle model inference] + | | + |<-- AudioChunk (binary) ----------| + |<-- AudioChunk (binary) ----------| + |<-- Complete ---------------------| + | | + |--- Stop ------------------------>| + |<-- Stopped ----------------------| + | | +``` + +#### 2.1.2 Voice Cloning + +The system SHALL support voice cloning with the following modes: + +| Mode | Description | Requirements | +|------|-------------|--------------| +| Default (Makima) | Pre-loaded Makima voice | None (auto-selected) | +| Custom Voice | User-provided reference | Audio file + transcript | +| X-Vector Only | Speaker embedding only | Audio file (no transcript) | + +**Default Voice Behavior:** +- If no voice is specified, use the pre-loaded Makima voice prompt +- Makima voice SHALL be a Japanese voice actress (Tomori Kusunoki) speaking English +- Voice prompt is pre-computed at model load time for zero-latency switching + +#### 2.1.3 Message Protocol + +##### Client-to-Server Messages + +All messages use JSON format with a `type` field for routing. + +**Start Message** - Initialize TTS session +```json +{ + "type": "start", + "sampleRate": 24000, + "encoding": "pcm16", + "voice": "makima", + "language": "English", + "authToken": "optional-jwt-token", + "contractId": "optional-contract-uuid" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | string | Yes | Must be "start" | +| `sampleRate` | number | No | Output sample rate (default: 24000) | +| `encoding` | string | No | Audio encoding: "pcm16", "pcm32f" (default: "pcm16") | +| `voice` | string | No | Voice ID or "makima" (default: "makima") | +| `language` | string | No | Output language (default: "English") | +| `authToken` | string | No | JWT for authenticated sessions | +| `contractId` | string | No | Contract ID for context | + +**Speak Message** - Request speech synthesis +```json +{ + "type": "speak", + "text": "Hello, I am Makima.", + "priority": "normal" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | string | Yes | Must be "speak" | +| `text` | string | Yes | Text to synthesize | +| `priority` | string | No | "high" or "normal" (default: "normal") | + +**Stop Message** - End session +```json +{ + "type": "stop", + "reason": "user_requested" +} +``` + +**Cancel Message** - Cancel current synthesis +```json +{ + "type": "cancel" +} +``` + +##### Server-to-Client Messages + +**Ready Message** - Session established +```json +{ + "type": "ready", + "sessionId": "uuid-string", + "voiceLoaded": "makima", + "capabilities": { + "streaming": true, + "languages": ["English", "Japanese", "Chinese", "Korean", "German", "French", "Russian", "Portuguese", "Spanish", "Italian"] + } +} +``` + +**Started Message** - TTS session configured +```json +{ + "type": "started", + "sampleRate": 24000, + "encoding": "pcm16", + "voice": "makima" +} +``` + +**AudioChunk Message** - Streaming audio data +```json +{ + "type": "audioChunk", + "data": "", + "sequenceNumber": 1, + "isFinal": false, + "timestampMs": 1234567890 +} +``` + +For binary transport (recommended for performance): +- Server MAY send raw binary WebSocket frames +- Binary frames contain PCM audio data directly +- JSON control messages indicate start/end of audio stream + +**Complete Message** - Synthesis finished +```json +{ + "type": "complete", + "durationMs": 1500, + "charactersProcessed": 25, + "audioLengthMs": 2100 +} +``` + +**Error Message** - Error occurred +```json +{ + "type": "error", + "code": "SYNTHESIS_ERROR", + "message": "Failed to generate audio", + "recoverable": true +} +``` + +**Stopped Message** - Session ended +```json +{ + "type": "stopped", + "reason": "user_requested" +} +``` + +#### 2.1.4 Integration with Listen Page + +The TTS endpoint SHALL integrate with the existing Listen page (`/api/v1/listen`) to enable bidirectional speech: + +**Bidirectional Flow:** +``` +User Speech -> /api/v1/listen (STT) -> Transcription + | + v + LLM Processing / Task Creation + | + v +Response Text -> /api/v1/speak (TTS) -> Audio -> User +``` + +**Implementation Requirements:** +1. Both endpoints SHALL support the same `contractId` for context sharing +2. TTS SHALL support interruption when new STT input is detected +3. Session management SHALL coordinate between STT and TTS + +### 2.2 Voice Configuration API + +#### 2.2.1 List Available Voices + +``` +GET /api/v1/voices +``` + +Response: +```json +{ + "voices": [ + { + "id": "makima", + "name": "Makima (Default)", + "language": "Japanese", + "description": "Default Makima voice (Tomori Kusunoki)", + "isDefault": true + } + ] +} +``` + +#### 2.2.2 Upload Custom Voice (Future) + +``` +POST /api/v1/voices +Content-Type: multipart/form-data + +audio: +transcript: "Text spoken in the audio" +name: "Custom Voice" +``` + +--- + +## 3. Non-Functional Requirements + +### 3.1 Latency Requirements + +| Metric | Target | Maximum | Notes | +|--------|--------|---------|-------| +| First Audio Byte | < 200ms | 500ms | From text submission to first audio chunk | +| Subsequent Chunks | < 50ms | 100ms | Inter-chunk latency | +| End-to-End Latency | < 300ms | 800ms | Total time for short phrases | +| Voice Prompt Loading | < 500ms | 2000ms | One-time at session start | + +**Measurement Points:** +- T0: Client sends "speak" message +- T1: First audio chunk received by client +- T2: Last audio chunk received ("complete" message) +- First Audio Latency = T1 - T0 +- Total Latency = T2 - T0 + +### 3.2 Audio Quality Requirements + +| Specification | Value | +|---------------|-------| +| Output Sample Rate | 24,000 Hz | +| Bit Depth | 16-bit (PCM16) or 32-bit float | +| Channels | Mono (1 channel) | +| Audio Codec | Raw PCM (WebSocket), WAV (download) | +| Voice Similarity | > 0.90 speaker similarity score | + +**Quality Metrics:** +- MOS (Mean Opinion Score): Target > 4.0 +- Speaker similarity to reference: Target > 0.90 +- No audible artifacts or glitches in streaming mode + +### 3.3 Hardware Requirements + +The TTS model runs directly in the makima process using candle. + +| Component | Minimum | Recommended | +|-----------|---------|-------------| +| GPU | CUDA-capable, 4GB VRAM (or Metal on macOS) | RTX 3060+ with 8GB+ VRAM | +| GPU Memory | 4GB | 8GB | +| System RAM | 8GB | 16GB | +| Storage | 5GB (model weights) | 10GB | + +**GPU Memory Breakdown:** +- Model weights (bf16): ~1.2GB +- Speech tokenizer: ~682MB +- KV cache during inference: ~1-2GB +- Safety margin: ~1GB + +**CPU Fallback:** +- candle supports CPU with MKL for systems without GPU +- Latency will be higher but functional + +### 3.4 Scalability Requirements + +| Metric | Target | +|--------|--------| +| Concurrent Sessions | 10 per GPU | +| Requests per Second | 50 text-to-speech requests | +| Audio Throughput | 10 hours of audio per hour | + +### 3.5 Availability Requirements + +| Metric | Target | +|--------|--------| +| Service Uptime | 99.5% | +| Recovery Time | < 30 seconds | +| Graceful Degradation | Fall back to batch mode if streaming fails | + +--- + +## 4. Architecture Specification + +### 4.1 System Architecture + +``` ++-------------------------------------------------------------------------+ +| Client Application | +| +-------------+ +-------------+ +------------------------------+ | +| | Listen | | Speak | | UI Components | | +| | (STT UI) | | (TTS UI) | | (Audio Player, Controls) | | +| +------+------+ +------+------+ +------------------------------+ | ++---------|--------------------|------------------------------------------+ + | WebSocket | WebSocket + | /api/v1/listen | /api/v1/speak + | | ++---------|--------------------|------------------------------------------+ +| | Makima Server (Rust/Axum) | +| +------v--------------------v------+ | +| | WebSocket Router | | +| | (axum WebSocket handlers) | | +| +------+--------------------+------+ | +| | | | +| +------v------+ +------v------+ +-----------------------------+ | +| | Listen | | Speak | | Shared State | | +| | Handler | | Handler | | - ML Models (STT) | | +| | (STT/ML) | | (TTS/ML) | | - TTS Model (candle) | | +| +-------------+ +------+------+ | - Voice Prompt Cache | | +| | | - Session Manager | | +| | +-----------------------------+ | +| +------v------+ | +| | TTS Module | | +| | (candle) | | +| +------+------+ | +| | | +| +------v------+ | +| | Qwen3-TTS | | +| | Components | | +| | - LM (28L) | | +| | - Code Pred | | +| | - Speech Tok| | +| +-------------+ | ++--------------------------------------------------------------------------+ +``` + +### 4.2 TTS Module Structure + +``` +makima/src/tts/ +├── mod.rs // TTS trait + factory (select Chatterbox vs Qwen3) +├── chatterbox.rs // Existing ONNX-based Chatterbox (moved from tts.rs) +├── qwen3/ +│ ├── mod.rs // Qwen3TTS public API +│ ├── model.rs // Qwen3 LM transformer (28 layers) +│ ├── code_predictor.rs // MTP module (5 layers, 16 codebooks) +│ ├── speech_tokenizer.rs // Encoder + Decoder (causal ConvNet) +│ ├── config.rs // Model config from config.json +│ └── generate.rs // Autoregressive generation loop with KV cache +``` + +#### 4.2.1 TTS Trait + +```rust +// makima/src/tts/mod.rs + +/// Trait for text-to-speech implementations. +#[async_trait] +pub trait TtsEngine: Send + Sync { + /// Generate audio from text with a given voice prompt. + async fn generate( + &self, + text: &str, + voice_id: &str, + language: &str, + ) -> Result, TtsError>; + + /// Load and cache a voice prompt from reference audio. + async fn load_voice(&self, voice_id: &str) -> Result<(), TtsError>; + + /// Check if the engine is ready for inference. + fn is_ready(&self) -> bool; +} + +/// Select the appropriate TTS engine based on configuration. +pub fn create_engine(config: &TtsConfig) -> Box { + match config.engine { + TtsEngineType::Qwen3 => Box::new(qwen3::Qwen3Tts::new(config)), + TtsEngineType::Chatterbox => Box::new(chatterbox::ChatterboxTts::new(config)), + } +} +``` + +#### 4.2.2 Qwen3 Candle Implementation + +The Qwen3 module implements the three core model components using candle: + +1. **Language Model** — 28-layer transformer using candle-transformers' Qwen2 attention with TTS-specific modifications +2. **Code Predictor** — 5-layer MTP module predicting 16 codebook layers +3. **Speech Tokenizer** — GAN-based codec with Conv1d encoder/decoder + +**Key candle features used:** +- `candle_core::Tensor` for all tensor operations +- `candle_nn::Module` for model layers +- `candle_nn::VarBuilder` for loading safetensors weights +- `candle_core::Device` for GPU/CPU selection + +#### 4.2.3 Model Loading + +Models are loaded lazily on first TTS request, following the pattern established by `listen.rs`: + +```rust +// Models held in SharedState behind async mutex +pub struct TtsModels { + pub engine: Box, + pub voice_cache: VoicePromptCache, +} + +impl AppState { + pub async fn get_tts_models(&self) -> Result<&TtsModels, TtsError> { + self.tts_models.get_or_try_init(|| async { + // Load safetensors weights via candle + // Initialize voice cache with default Makima voice + }).await + } +} +``` + +### 4.3 Speak Handler + +```rust +// makima/src/server/handlers/speak.rs + +/// WebSocket handler for TTS streaming. +/// Calls the TTS engine directly — no proxy, no external service. +pub async fn websocket_handler( + ws: WebSocketUpgrade, + State(state): State, +) -> Response { + ws.on_upgrade(|socket| handle_speak_socket(socket, state)) +} + +async fn handle_speak_socket(socket: WebSocket, state: SharedState) { + let session_id = Uuid::new_v4().to_string(); + + // Get or lazily load TTS models + let tts = match state.get_tts_models().await { + Ok(tts) => tts, + Err(e) => { + // Send error and close + return; + } + }; + + // Handle WebSocket messages directly + // Parse JSON commands, run inference, stream audio chunks back +} +``` + +### 4.4 Voice Prompt Caching + +Voice prompts are cached in-memory using an LRU cache: + +```rust +// makima/src/tts/mod.rs + +pub struct VoicePromptCache { + cache: tokio::sync::Mutex>, +} + +impl VoicePromptCache { + pub fn new(max_size: usize) -> Self { /* ... */ } + pub async fn get(&self, voice_id: &str) -> Option { /* ... */ } + pub async fn insert(&self, voice_id: String, prompt: VoicePrompt) { /* ... */ } +} +``` + +### 4.5 Error Handling and Recovery + +#### 4.5.1 Error Categories + +| Error Code | Category | Recoverable | Action | +|------------|----------|-------------|--------| +| `MODEL_LOADING` | Initialization | Yes | Wait and retry | +| `SYNTHESIS_ERROR` | Generation | Yes | Retry with same input | +| `INVALID_TEXT` | Input | No | Return error to client | +| `VOICE_NOT_FOUND` | Configuration | No | Fall back to default voice | +| `GPU_OUT_OF_MEMORY` | Resource | Yes | Clear cache, retry on CPU | +| `TIMEOUT` | Inference | Yes | Retry with backoff | + +--- + +## 5. API Contract + +### 5.1 WebSocket Message Formats + +#### 5.1.1 Client-to-Server Messages + +```typescript +// TypeScript type definitions for client implementation + +interface StartMessage { + type: "start"; + sampleRate?: number; // Default: 24000 + encoding?: "pcm16" | "pcm32f"; // Default: "pcm16" + voice?: string; // Default: "makima" + language?: string; // Default: "English" + authToken?: string; // JWT for authenticated sessions + contractId?: string; // Contract context +} + +interface SpeakMessage { + type: "speak"; + text: string; // Required: text to synthesize + priority?: "normal" | "high"; // Default: "normal" +} + +interface CancelMessage { + type: "cancel"; +} + +interface StopMessage { + type: "stop"; + reason?: string; +} + +type ClientMessage = StartMessage | SpeakMessage | CancelMessage | StopMessage; +``` + +#### 5.1.2 Server-to-Client Messages + +```typescript +interface ReadyMessage { + type: "ready"; + sessionId: string; + voiceLoaded: string; + capabilities: { + streaming: boolean; + languages: string[]; + }; +} + +interface StartedMessage { + type: "started"; + sampleRate: number; + encoding: string; + voice: string; +} + +interface AudioChunkMessage { + type: "audioChunk"; + data: string; // Base64-encoded PCM audio + sequenceNumber: number; + isFinal: boolean; + timestampMs: number; +} + +interface CompleteMessage { + type: "complete"; + durationMs: number; + charactersProcessed: number; + audioLengthMs: number; +} + +interface ErrorMessage { + type: "error"; + code: string; + message: string; + recoverable: boolean; +} + +interface StoppedMessage { + type: "stopped"; + reason: string; +} + +type ServerMessage = + | ReadyMessage + | StartedMessage + | AudioChunkMessage + | CompleteMessage + | ErrorMessage + | StoppedMessage; +``` + +### 5.2 Error Codes + +| Code | HTTP-like | Description | Recovery | +|------|-----------|-------------|----------| +| `MODEL_LOADING` | 503 | Model still loading | Wait and retry | +| `SYNTHESIS_ERROR` | 500 | Failed to generate audio | Retry | +| `INVALID_TEXT` | 400 | Text is empty or invalid | Fix input | +| `VOICE_NOT_FOUND` | 404 | Requested voice doesn't exist | Use default | +| `UNAUTHORIZED` | 401 | Invalid or missing auth token | Re-authenticate | +| `RATE_LIMITED` | 429 | Too many requests | Back off | +| `TIMEOUT` | 408 | Operation timed out | Retry | +| `CANCELLED` | 499 | Client cancelled request | N/A | + +### 5.3 Session Management + +#### 5.3.1 Session Lifecycle + +``` +DISCONNECTED -> CONNECTING -> READY -> STARTED -> SPEAKING -> READY -> ... -> STOPPED + | | | | + v v v v + ERROR ERROR ERROR STOPPED +``` + +--- + +## 6. Voice Asset Requirements + +### 6.1 Makima Voice Clip Specifications + +#### 6.1.1 Audio Requirements + +| Specification | Requirement | +|---------------|-------------| +| Duration | 5-10 seconds (minimum 3s) | +| Format | WAV (PCM) | +| Sample Rate | 24,000 Hz or higher | +| Bit Depth | 16-bit or higher | +| Channels | Mono (preferred) or Stereo | +| Content | Clear speech, natural tone | +| Background | Minimal noise/music | + +#### 6.1.2 Content Guidelines + +**DO:** +- Use dialogue with varied intonation +- Include multiple phonemes +- Capture natural speaking rhythm +- Extract from clean audio scenes + +**DON'T:** +- Include background music +- Use shouting or whispering +- Include sound effects +- Use heavily processed audio + +#### 6.1.3 Transcript Requirements + +| Specification | Requirement | +|---------------|-------------| +| Format | Plain text (.txt) or JSON | +| Encoding | UTF-8 | +| Content | Exact transcription of audio | +| Language | Japanese (for Japanese reference) | + +### 6.2 Storage Location and Management + +#### 6.2.1 Directory Structure + +``` +models/ +└── voices/ + ├── makima/ + │ ├── reference.wav # Primary reference audio + │ ├── transcript.txt # Plain text transcript + │ ├── transcript.json # Structured transcript (optional) + │ └── metadata.json # Voice metadata + ├── makima-alt/ # Alternative Makima clips (future) + │ └── ... + └── custom/ # User-uploaded voices (future) + └── {voice_id}/ + ├── reference.wav + ├── transcript.txt + └── metadata.json +``` + +--- + +## 7. Testing Strategy + +### 7.1 Unit Tests + +#### 7.1.1 Rust TTS Module Tests + +```rust +// makima/src/tts/qwen3/tests.rs + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_config_loading() { + let config = Qwen3Config::from_json("test_config.json").unwrap(); + assert_eq!(config.hidden_size, 1024); + assert_eq!(config.num_layers, 28); + } + + #[test] + fn test_voice_prompt_cache_lru() { + let cache = VoicePromptCache::new(2); + cache.insert("a", prompt_a); + cache.insert("b", prompt_b); + cache.get("a"); // access a + cache.insert("c", prompt_c); // should evict b + + assert!(cache.get("a").is_some()); + assert!(cache.get("b").is_none()); + assert!(cache.get("c").is_some()); + } + + #[tokio::test] + async fn test_speak_handler_message_parsing() { + let json = r#"{"type": "start", "voice": "makima"}"#; + let msg: SpeakClientMessage = serde_json::from_str(json).unwrap(); + + match msg { + SpeakClientMessage::Start(start) => { + assert_eq!(start.voice, Some("makima".to_string())); + } + _ => panic!("Expected Start message"), + } + } +} +``` + +### 7.2 Integration Tests + +```rust +// tests/tts_integration.rs + +#[tokio::test] +async fn test_speak_websocket_flow() { + // Start test server with TTS enabled + let state = create_test_state_with_tts().await; + let app = make_router(state); + + // Connect WebSocket + let ws = connect_ws("/api/v1/speak").await; + + // Send start + ws.send_json(json!({"type": "start", "voice": "makima"})).await; + let ready = ws.recv_json().await; + assert_eq!(ready["type"], "ready"); + + // Send speak + ws.send_json(json!({"type": "speak", "text": "Hello."})).await; + + // Collect audio chunks + let mut chunks = vec![]; + loop { + let msg = ws.recv().await; + match msg { + WsMsg::Binary(data) => chunks.push(data), + WsMsg::Text(json) => { + let data: Value = serde_json::from_str(&json).unwrap(); + if data["type"] == "complete" { break; } + } + } + } + assert!(!chunks.is_empty()); +} +``` + +### 7.3 Performance Targets + +| Metric | Target | Acceptable | Warning | +|--------|--------|------------|---------| +| First Audio (short) | < 150ms | < 200ms | > 300ms | +| First Audio (medium) | < 200ms | < 300ms | > 500ms | +| First Audio (long) | < 300ms | < 500ms | > 800ms | +| Inter-chunk | < 30ms | < 50ms | > 100ms | +| Memory (GPU) | < 4GB | < 6GB | > 8GB | +| Memory (CPU) | < 2GB | < 4GB | > 8GB | + +--- + +## 8. Implementation Phases + +### Phase 1: Candle-Based Qwen3-TTS Module (Week 1-2) + +**Deliverables:** +- [ ] `makima/src/tts/mod.rs` — TTS trait + factory +- [ ] `makima/src/tts/chatterbox.rs` — Move existing code from tts.rs +- [ ] `makima/src/tts/qwen3/model.rs` — 28-layer LM backbone (extend candle Qwen2) +- [ ] `makima/src/tts/qwen3/code_predictor.rs` — MTP module (5 layers, 16 codebooks) +- [ ] `makima/src/tts/qwen3/speech_tokenizer.rs` — ConvNet encoder/decoder + RVQ +- [ ] `makima/src/tts/qwen3/config.rs` — Config from safetensors +- [ ] `makima/src/tts/qwen3/generate.rs` — Autoregressive generation with KV cache +- [ ] Add `candle-core`, `candle-nn`, `candle-transformers` to Cargo.toml + +**Success Criteria:** +- Model loads safetensors weights successfully +- Can generate audio from text via direct inference +- First audio latency < 500ms (initial, unoptimized) + +### Phase 2: WebSocket Handler + Voice Assets (Week 2-3) + +**Deliverables:** +- [ ] Update `makima/src/server/handlers/speak.rs` — Direct TTS handler (no proxy) +- [ ] Lazy model loading via `SharedState` +- [ ] Voice prompt caching +- [ ] Makima voice asset acquisition and processing +- [ ] Basic error handling and session management + +**Success Criteria:** +- `/api/v1/speak` endpoint produces streaming audio +- Default Makima voice works +- Error handling matches specification + +### Phase 3: Optimization + Integration (Week 3-4) + +**Deliverables:** +- [ ] Streaming audio generation (token-by-token decoding) +- [ ] GPU memory optimization +- [ ] Listen page integration for bidirectional speech +- [ ] Session coordination between STT and TTS +- [ ] Full test suite (unit, integration) +- [ ] Latency benchmarks + +**Success Criteria:** +- First audio latency < 200ms +- Memory usage < 6GB +- All tests passing +- Documentation complete + +--- + +## Appendix + +### A. Dependencies + +#### Rust (Cargo.toml additions) + +```toml +[dependencies] +candle-core = "0.8" +candle-nn = "0.8" +candle-transformers = "0.8" +# Keep existing: tokenizers, hf-hub, ndarray (for compatibility) +``` + +### B. Environment Variables + +```bash +# TTS Configuration +TTS_ENGINE=qwen3 # "qwen3" or "chatterbox" +TTS_MODEL_ID=Qwen/Qwen3-TTS-12Hz-0.6B-Base +TTS_DEVICE=cuda:0 # "cuda:0", "metal", or "cpu" +TTS_VOICES_DIR=models/voices +TTS_DEFAULT_VOICE=makima +``` + +### C. References + +1. [Qwen3-TTS-12Hz-0.6B-Base (Hugging Face)](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-Base) +2. [Qwen3-TTS GitHub Repository](https://github.com/QwenLM/Qwen3-TTS) +3. [Qwen3-TTS Technical Report (arXiv:2601.15621)](https://arxiv.org/abs/2601.15621) +4. [Candle — HuggingFace Rust ML Framework](https://github.com/huggingface/candle) +5. [axum WebSocket Documentation](https://docs.rs/axum/latest/axum/extract/ws/index.html) +6. [docs/research/rust-native-tts-research.md](../research/rust-native-tts-research.md) — Detailed feasibility analysis + +### D. Glossary + +| Term | Definition | +|------|------------| +| **TTS** | Text-to-Speech: Converting text input to audio output | +| **STT** | Speech-to-Text: Converting audio input to text output | +| **Voice Cloning** | Creating synthetic speech that mimics a specific speaker | +| **Voice Prompt** | Pre-computed speaker embedding for voice cloning | +| **Candle** | HuggingFace's minimalist Rust ML framework | +| **SafeTensors** | Efficient, safe model weight serialization format | +| **RVQ** | Residual Vector Quantization — multi-codebook audio tokenization | +| **MTP** | Multi-Token Prediction — code predictor generating 16 codebook layers | +| **bf16** | Brain floating-point 16-bit format for GPU computation | diff --git a/makima/Cargo.toml b/makima/Cargo.toml index 950c123..b6b12dd 100644 --- a/makima/Cargo.toml +++ b/makima/Cargo.toml @@ -17,6 +17,12 @@ tokenizers = "0.21" hf-hub = "0.4" ndarray = "0.16" +# Candle ML framework (Qwen3-TTS native inference) +candle-core = "0.8" +candle-nn = "0.8" +candle-transformers = "0.8" +safetensors = "0.4" + # Web server axum = { version = "0.8", features = ["ws", "multipart"] } tokio = { version = "1.0", features = ["full", "signal", "process"] } diff --git a/makima/frontend/src/components/listen/ControlPanel.tsx b/makima/frontend/src/components/listen/ControlPanel.tsx index f0e5702..f482ec4 100644 --- a/makima/frontend/src/components/listen/ControlPanel.tsx +++ b/makima/frontend/src/components/listen/ControlPanel.tsx @@ -1,6 +1,7 @@ import { useState } from "react"; import { Logo } from "../Logo"; import type { MicrophoneStatus } from "../../hooks/useMicrophone"; +import type { ConnectionStatus } from "../../hooks/useWebSocket"; import { ContractPickerModal } from "./ContractPickerModal"; export interface ContractOption { @@ -22,6 +23,8 @@ interface ControlPanelProps { selectedContractId: string | null; onContractChange: (contractId: string | null) => void; contractsLoading?: boolean; + // Connection status for loading state + connectionStatus?: ConnectionStatus; } function getStatusText(isListening: boolean, micStatus: MicrophoneStatus): string { @@ -54,6 +57,7 @@ export function ControlPanel({ selectedContractId, onContractChange, contractsLoading, + connectionStatus, }: ControlPanelProps) { const [isModalOpen, setIsModalOpen] = useState(false); const statusText = getStatusText(isListening, micStatus); @@ -121,18 +125,36 @@ export function ControlPanel({ {/* Connection status */}
- - {isConnected ? "CONNECTED" : "DISCONNECTED"} +
+ + {isConnected + ? "CONNECTED" + : connectionStatus === "connecting" + ? "LOADING MODELS..." + : "DISCONNECTED"} +
+ {connectionStatus === "connecting" && ( +
+
+
+ )}
diff --git a/makima/frontend/src/hooks/useSpeakWebSocket.ts b/makima/frontend/src/hooks/useSpeakWebSocket.ts new file mode 100644 index 0000000..3ef8851 --- /dev/null +++ b/makima/frontend/src/hooks/useSpeakWebSocket.ts @@ -0,0 +1,329 @@ +import { useState, useCallback, useRef, useEffect } from "react"; +import { SPEAK_ENDPOINT } from "../lib/api"; + +export type SpeakStatus = + | "disconnected" + | "connecting" + | "connected" + | "loading_model" + | "speaking" + | "error"; + +export interface SpeakWebSocketState { + status: SpeakStatus; + error: string | null; +} + +export function useSpeakWebSocket() { + const [state, setState] = useState({ + status: "disconnected", + error: null, + }); + + const wsRef = useRef(null); + const audioContextRef = useRef(null); + const audioQueueRef = useRef([]); + const isPlayingRef = useRef(false); + const modelLoadingTimerRef = useRef | null>(null); + const nextPlayTimeRef = useRef(0); + + // Clean up on unmount + useEffect(() => { + return () => { + if (wsRef.current) { + wsRef.current.close(); + wsRef.current = null; + } + if (audioContextRef.current) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + if (modelLoadingTimerRef.current) { + clearTimeout(modelLoadingTimerRef.current); + modelLoadingTimerRef.current = null; + } + }; + }, []); + + const getAudioContext = useCallback((): AudioContext => { + if (!audioContextRef.current || audioContextRef.current.state === "closed") { + audioContextRef.current = new AudioContext({ sampleRate: 24000 }); + } + return audioContextRef.current; + }, []); + + const playAudioQueue = useCallback(() => { + if (isPlayingRef.current) return; + isPlayingRef.current = true; + + const ctx = getAudioContext(); + + function scheduleNext() { + const chunk = audioQueueRef.current.shift(); + if (!chunk) { + isPlayingRef.current = false; + return; + } + + const buffer = ctx.createBuffer(1, chunk.length, 24000); + buffer.copyToChannel(chunk, 0); + + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(ctx.destination); + + // Schedule playback at the right time to avoid gaps + const now = ctx.currentTime; + const startTime = Math.max(now, nextPlayTimeRef.current); + source.start(startTime); + nextPlayTimeRef.current = startTime + buffer.duration; + + source.onended = () => { + if (audioQueueRef.current.length > 0) { + scheduleNext(); + } else { + isPlayingRef.current = false; + } + }; + } + + scheduleNext(); + }, [getAudioContext]); + + const connect = useCallback((): Promise => { + return new Promise((resolve) => { + if (wsRef.current?.readyState === WebSocket.OPEN) { + resolve(true); + return; + } + + if (wsRef.current) { + wsRef.current.close(); + wsRef.current = null; + } + + setState({ status: "connecting", error: null }); + + try { + const ws = new WebSocket(SPEAK_ENDPOINT); + ws.binaryType = "arraybuffer"; + wsRef.current = ws; + + ws.onopen = () => { + setState({ status: "connected", error: null }); + resolve(true); + }; + + ws.onmessage = (event) => { + // Binary data = PCM audio chunk + if (event.data instanceof ArrayBuffer) { + // Clear model loading timer on first audio data + if (modelLoadingTimerRef.current) { + clearTimeout(modelLoadingTimerRef.current); + modelLoadingTimerRef.current = null; + } + + // Update status to speaking if not already + setState((s) => { + if (s.status === "loading_model" || s.status === "connected") { + return { ...s, status: "speaking" }; + } + return s; + }); + + // Convert PCM16 LE to Float32 + const pcm16 = new Int16Array(event.data); + const float32 = new Float32Array(pcm16.length); + for (let i = 0; i < pcm16.length; i++) { + float32[i] = pcm16[i] / 32768; + } + + audioQueueRef.current.push(float32); + playAudioQueue(); + return; + } + + // Text data = JSON message + try { + const message = JSON.parse(event.data); + + switch (message.type) { + case "audio_end": + // Clear model loading timer + if (modelLoadingTimerRef.current) { + clearTimeout(modelLoadingTimerRef.current); + modelLoadingTimerRef.current = null; + } + // Wait for audio queue to drain, then go back to connected + // Use a short delay to let buffered audio finish + { + const checkDone = () => { + if (audioQueueRef.current.length === 0 && !isPlayingRef.current) { + setState((s) => { + if (s.status === "speaking" || s.status === "loading_model") { + return { ...s, status: "connected" }; + } + return s; + }); + } else { + setTimeout(checkDone, 100); + } + }; + checkDone(); + } + break; + + case "error": + if (modelLoadingTimerRef.current) { + clearTimeout(modelLoadingTimerRef.current); + modelLoadingTimerRef.current = null; + } + setState({ + status: "error", + error: message.message || `Error: ${message.code}`, + }); + break; + } + } catch { + console.error("Failed to parse speak WebSocket message:", event.data); + } + }; + + ws.onerror = () => { + setState({ + status: "error", + error: "Failed to connect to speak server", + }); + resolve(false); + }; + + ws.onclose = (event) => { + if (modelLoadingTimerRef.current) { + clearTimeout(modelLoadingTimerRef.current); + modelLoadingTimerRef.current = null; + } + + let errorMessage: string | null = null; + if (event.code === 1006) { + errorMessage = "Connection failed - server may be unavailable"; + } else if (event.code !== 1000 && event.code !== 1001) { + errorMessage = `Connection closed unexpectedly (code: ${event.code})`; + } + + setState((s) => ({ + status: "disconnected", + error: errorMessage || s.error, + })); + wsRef.current = null; + }; + } catch (err) { + const message = + err instanceof Error ? err.message : "Failed to create WebSocket connection"; + setState({ status: "error", error: message }); + resolve(false); + } + }); + }, [playAudioQueue]); + + const speak = useCallback( + async (text: string) => { + if (!text.trim()) return; + + // Connect if not connected + if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) { + const connected = await connect(); + if (!connected) return; + } + + // Reset audio state + audioQueueRef.current = []; + isPlayingRef.current = false; + nextPlayTimeRef.current = 0; + + // Resume audio context if suspended (browser autoplay policy) + const ctx = getAudioContext(); + if (ctx.state === "suspended") { + await ctx.resume(); + } + + // Start loading timer - if no audio arrives in 2 seconds, show loading state + modelLoadingTimerRef.current = setTimeout(() => { + setState((s) => { + if (s.status === "connected" || s.status === "connecting") { + return { ...s, status: "loading_model" }; + } + return s; + }); + modelLoadingTimerRef.current = null; + }, 2000); + + // Send speak request + wsRef.current?.send( + JSON.stringify({ type: "speak", text }) + ); + + setState((s) => ({ ...s, error: null })); + }, + [connect, getAudioContext] + ); + + const cancel = useCallback(() => { + // Clear audio queue + audioQueueRef.current = []; + isPlayingRef.current = false; + nextPlayTimeRef.current = 0; + + // Clear model loading timer + if (modelLoadingTimerRef.current) { + clearTimeout(modelLoadingTimerRef.current); + modelLoadingTimerRef.current = null; + } + + // Send cancel message + if (wsRef.current?.readyState === WebSocket.OPEN) { + wsRef.current.send(JSON.stringify({ type: "cancel" })); + } + + setState((s) => ({ + ...s, + status: wsRef.current?.readyState === WebSocket.OPEN ? "connected" : "disconnected", + })); + }, []); + + const disconnect = useCallback(() => { + // Clear audio queue + audioQueueRef.current = []; + isPlayingRef.current = false; + nextPlayTimeRef.current = 0; + + if (modelLoadingTimerRef.current) { + clearTimeout(modelLoadingTimerRef.current); + modelLoadingTimerRef.current = null; + } + + if (wsRef.current) { + // Send stop message before closing + if (wsRef.current.readyState === WebSocket.OPEN) { + wsRef.current.send(JSON.stringify({ type: "stop" })); + } + wsRef.current.close(1000, "User disconnected"); + wsRef.current = null; + } + + setState({ status: "disconnected", error: null }); + }, []); + + return { + ...state, + isConnected: + state.status === "connected" || + state.status === "speaking" || + state.status === "loading_model", + isSpeaking: state.status === "speaking", + isModelLoading: state.status === "loading_model", + speak, + cancel, + connect, + disconnect, + }; +} diff --git a/makima/frontend/src/index.css b/makima/frontend/src/index.css index 5c08006..f29873b 100644 --- a/makima/frontend/src/index.css +++ b/makima/frontend/src/index.css @@ -64,6 +64,12 @@ body { background: rgba(117, 170, 252, 0.35); } +/* Loading bar animation for indeterminate progress */ +@keyframes loading-slide { + 0% { transform: translateX(-100%); } + 100% { transform: translateX(300%); } +} + /* Grid overlay */ .grid-overlay { position: fixed; diff --git a/makima/frontend/src/lib/api.ts b/makima/frontend/src/lib/api.ts index 4390b20..ca04ce7 100644 --- a/makima/frontend/src/lib/api.ts +++ b/makima/frontend/src/lib/api.ts @@ -99,6 +99,7 @@ async function authFetch(url: string, options: RequestInit = {}): Promise @@ -135,6 +136,14 @@ createRoot(document.getElementById("root")!).render( } /> + + + + } + /> diff --git a/makima/frontend/src/routes/listen.tsx b/makima/frontend/src/routes/listen.tsx index 55cf7e6..8af538e 100644 --- a/makima/frontend/src/routes/listen.tsx +++ b/makima/frontend/src/routes/listen.tsx @@ -207,6 +207,7 @@ export default function ListenPage() { selectedContractId={selectedContractId} onContractChange={setSelectedContractId} contractsLoading={contractsLoading} + connectionStatus={ws.status} /> diff --git a/makima/frontend/src/routes/speak.tsx b/makima/frontend/src/routes/speak.tsx new file mode 100644 index 0000000..c4692ff --- /dev/null +++ b/makima/frontend/src/routes/speak.tsx @@ -0,0 +1,159 @@ +import { useState, useCallback } from "react"; +import { Masthead } from "../components/Masthead"; +import { useSpeakWebSocket } from "../hooks/useSpeakWebSocket"; + +export default function SpeakPage() { + const [text, setText] = useState(""); + const tts = useSpeakWebSocket(); + + const handleSpeak = useCallback(() => { + if (!text.trim()) return; + tts.speak(text); + }, [text, tts]); + + const handleCancel = useCallback(() => { + tts.cancel(); + }, [tts]); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + // Ctrl/Cmd + Enter to speak + if ((e.ctrlKey || e.metaKey) && e.key === "Enter") { + e.preventDefault(); + handleSpeak(); + } + }, + [handleSpeak] + ); + + const statusLabel = (() => { + switch (tts.status) { + case "disconnected": + return "DISCONNECTED"; + case "connecting": + return "CONNECTING..."; + case "connected": + return "CONNECTED"; + case "loading_model": + return "LOADING TTS MODEL..."; + case "speaking": + return "SPEAKING"; + case "error": + return "ERROR"; + default: + return "IDLE"; + } + })(); + + const statusColor = (() => { + switch (tts.status) { + case "connected": + case "speaking": + return "border-[#3f6fb3] text-[#75aafc]"; + case "error": + return "border-red-400/50 text-red-400"; + default: + return "border-[rgba(117,170,252,0.25)] text-[#9bc3ff]"; + } + })(); + + const dotColor = (() => { + switch (tts.status) { + case "connected": + case "speaking": + return "bg-[#75aafc]"; + case "error": + return "bg-red-400"; + default: + return "bg-[#3f6fb3]"; + } + })(); + + return ( +
+ + +
+ {/* Text input area */} +
+