From eabd1304cce0e053cd32ec910d2f0ea429e8af14 Mon Sep 17 00:00:00 2001 From: soryu Date: Wed, 28 Jan 2026 02:54:17 +0000 Subject: Add Qwen3-TTS streaming endpoint for voice synthesis (#40) * Task completion checkpoint * Task completion checkpoint * Task completion checkpoint * Add Qwen3-TTS research document for live TTS replacement Research findings for replacing Chatterbox TTS with Qwen3-TTS-12Hz-0.6B-Base: - Current TTS: Chatterbox-Turbo-ONNX with batch-only generation, no streaming - Qwen3-TTS: 97ms end-to-end latency, streaming support, 3-second voice cloning - Voice cloning: Requires 3s reference audio + transcript (Makima voice planned) - Integration: Python service with WebSocket bridge (no ONNX export available) - Languages: 10 supported including English and Japanese Document includes: - Current architecture analysis (makima/src/tts.rs) - Qwen3-TTS capabilities and requirements - Feasibility assessment for live/streaming TTS - Audio clip requirements for voice cloning - Preliminary technical approach with architecture diagrams Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 03:11:15 UTC * Add Qwen3-TTS research documentation Comprehensive research on replacing Chatterbox TTS with Qwen3-TTS-12Hz-0.6B-Base: - Current TTS implementation analysis (Chatterbox-Turbo-ONNX in makima/src/tts.rs) - Qwen3-TTS capabilities: 97ms streaming latency, voice cloning with 3s reference - Cross-lingual support: Japanese voice (Makima/Tomori Kusunoki) speaking English - Python microservice architecture recommendation (FastAPI + WebSocket) - Implementation phases and technical approach - Hardware requirements and dependencies Key findings: - Live/streaming TTS is highly feasible with 97ms latency - Voice cloning fully supported with 0.95 speaker similarity - Recommended: Python microservice with WebSocket streaming Co-Authored-By: Claude Opus 4.5 * Add comprehensive Qwen3-TTS integration specification This specification document defines the complete integration of Qwen3-TTS-12Hz-0.6B-Base as a replacement for the existing Chatterbox-Turbo TTS implementation. The document covers: ## Functional Requirements - WebSocket endpoint /api/v1/speak for streaming TTS - Voice cloning with default Makima voice (Japanese VA speaking English) - Support for custom voice references - Detailed client-to-server and server-to-client message protocols - Integration with Listen page for bidirectional speech ## Non-Functional Requirements - Latency targets: < 200ms first audio byte - Audio quality: 24kHz, mono, PCM16/PCM32f - Hardware requirements: CUDA GPU with 4-8GB VRAM - Scalability: 10 concurrent sessions per GPU ## Architecture Specification - Python TTS microservice with FastAPI/WebSocket - Rust proxy endpoint in makima server - Voice prompt caching mechanism (LRU cache) - Error handling and recovery strategies ## API Contract - Complete WebSocket message format definitions (TypeScript) - Error codes and responses (TTS_UNAVAILABLE, SYNTHESIS_ERROR, etc.) - Session state machine and lifecycle management ## Voice Asset Requirements - Makima voice clip specifications (5-10s WAV, transcript required) - Storage location: models/voices/makima/ - Metadata format for voice management ## Testing Strategy - Unit tests for Python TTS service and Rust proxy - Integration tests for WebSocket flow - Latency benchmarks with performance targets - Test data fixtures for various text lengths Co-Authored-By: Claude Opus 4.5 * Add Qwen3-TTS implementation plan Comprehensive implementation plan for replacing Chatterbox-TTS with Qwen3-TTS streaming TTS service, including: - Task breakdown with estimated hours for each phase - Phase 1: Python TTS microservice (FastAPI, WebSocket) - Phase 2: Rust proxy integration (speak.rs, tts_client.rs) - Detailed file changes and new module structure - Testing plan with unit, integration, and latency benchmarks - Risk assessment with mitigation strategies - Success criteria for each phase Based on specification in docs/specs/qwen3-tts-spec.md Co-Authored-By: Claude Opus 4.5 * Add author and research references to TTS implementation plan Add links to research documentation and author attribution. Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 03:25:06 UTC * Add Python TTS service project structure (Phase 1.1-1.3) Create the initial makima-tts Python service directory structure with: - pyproject.toml with FastAPI, Qwen-TTS, and torch dependencies - config.py with pydantic-settings TTSConfig class - models.py with Pydantic message models (Start, Speak, Stop, Ready, etc.) This implements tasks P1.1, P1.2, and P1.3 from the Qwen3-TTS implementation plan. Co-Authored-By: Claude Opus 4.5 * Add TTS engine and voice manager for Qwen3-TTS (Phase 1.4-1.5) Implement core TTS functionality: - tts_engine.py: Qwen3-TTS wrapper with streaming audio chunk generation - voice_manager.py: Voice prompt caching with LRU eviction and TTL support Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 03:30:06 UTC * Add TTS proxy client and message types (Phase 2.1, 2.2, 2.4) - Add tts_client.rs with TtsConfig, TtsCircuitBreaker, TtsError, TtsProxyClient, and TtsConnection structs for WebSocket proxying - Add TTS message types to messages.rs (TtsAudioEncoding, TtsPriority, TtsStartMessage, TtsSpeakMessage, TtsStopMessage, TtsClientMessage, TtsReadyMessage, TtsAudioChunkMessage, TtsCompleteMessage, TtsErrorMessage, TtsStoppedMessage, TtsServerMessage) - Export tts_client module from server mod.rs - tokio-tungstenite already present in Cargo.toml Co-Authored-By: Claude Opus 4.5 * Add TTS WebSocket handler and route (Phase 2.3, 2.5, 2.6) - Create speak.rs WebSocket handler that proxies to Python TTS service - Add TtsState fields (tts_client, tts_config) to AppState - Add with_tts() builder and is_tts_healthy() methods to AppState - Register /api/v1/speak route in the router - Add speak module export in handlers/mod.rs The handler forwards WebSocket messages bidirectionally between the client and the Python TTS microservice with proper error handling. Co-Authored-By: Claude Opus 4.5 * Add Makima voice profile assets for TTS voice cloning Creates the voice assets directory structure with: - manifest.json containing voice configuration (voice_id, speaker, language, reference audio path, and Japanese transcript placeholder) - README.md with instructions for obtaining voice reference audio Co-Authored-By: Claude Opus 4.5 * Add Rust-native Qwen3-TTS integration research document Research findings for integrating Qwen3-TTS-12Hz-0.6B-Base directly into the makima Rust codebase without Python. Key conclusions: - ONNX export is not viable (unsupported architecture) - Candle (HF Rust ML framework) is the recommended approach - Model weights available in safetensors format (2.52GB total) - Three components needed: LM backbone, code predictor, speech tokenizer - Crane project has Qwen3-TTS as highest priority (potential upstream) Co-Authored-By: Claude Opus 4.5 * [WIP] Heartbeat checkpoint - 2026-01-27 11:21:43 UTC * [WIP] Heartbeat checkpoint - 2026-01-27 11:24:19 UTC * [WIP] Heartbeat checkpoint - 2026-01-27 11:26:43 UTC * feat: implement Rust-native Qwen3-TTS using candle framework Replace monolithic tts.rs with modular tts/ directory structure: - tts/mod.rs: TtsEngine trait, TtsEngineFactory, shared types (AudioChunk, TtsError), and utility functions (save_wav, resample, argmax) - tts/chatterbox.rs: existing ONNX-based ChatterboxTTS adapted to implement TtsEngine trait with Mutex-wrapped sessions for Send+Sync - tts/qwen3/mod.rs: Qwen3Tts entry point with HuggingFace model loading - tts/qwen3/config.rs: Qwen3TtsConfig parsing from HF config.json - tts/qwen3/model.rs: 28-layer Qwen3 transformer with RoPE, GQA (16 heads, 8 KV heads), SiLU MLP, RMS norm, and KV cache - tts/qwen3/code_predictor.rs: 5-layer MTP module predicting 16 codebooks - tts/qwen3/speech_tokenizer.rs: ConvNet encoder/decoder with 16-layer RVQ - tts/qwen3/generate.rs: autoregressive generation loop with streaming support Add candle-core, candle-nn, candle-transformers, safetensors to Cargo.toml. Co-Authored-By: Claude Opus 4.5 * feat: integrate TTS engine into speak WebSocket handler - Update speak.rs handler to use TTS engine directly from SharedState instead of returning a stub "not implemented" error - Add TtsEngine (OnceCell lazy-loaded) to AppState in state.rs with get_tts_engine() method for lazy initialization on first connection - Implement full WebSocket protocol: client sends JSON speak/cancel/stop messages, server streams binary PCM audio chunks and audio_end signals - Create voices/makima/manifest.json for Makima voice profile configuration - All files compile successfully with zero errors Co-Authored-By: Claude Opus 4.5 * feat: add /speak TTS page with WebSocket audio playback Add a new /speak frontend page for text-to-speech via WebSocket. The page accepts text input and streams synthesized PCM audio through the Web Audio API. Includes model loading indicator, cancel support, and connection status. Also adds a loading bar to the listen page ControlPanel during WebSocket connection. Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- makima/src/tts/qwen3/mod.rs | 287 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 makima/src/tts/qwen3/mod.rs (limited to 'makima/src/tts/qwen3/mod.rs') diff --git a/makima/src/tts/qwen3/mod.rs b/makima/src/tts/qwen3/mod.rs new file mode 100644 index 0000000..c55c118 --- /dev/null +++ b/makima/src/tts/qwen3/mod.rs @@ -0,0 +1,287 @@ +//! Qwen3-TTS — Pure Rust implementation using candle. +//! +//! Implements Qwen3-TTS-12Hz-0.6B-Base for text-to-speech synthesis +//! with voice cloning support. No Python, no ONNX — pure Rust inference +//! via the candle ML framework. +//! +//! # Architecture +//! +//! The model has three components: +//! - **Language Model** (28-layer transformer): generates zeroth codebook tokens +//! - **Code Predictor** (5-layer MTP): predicts remaining 15 codebook layers +//! - **Speech Tokenizer** (ConvNet codec): encodes/decodes audio ↔ codes +//! +//! # Usage +//! +//! ```rust,no_run +//! use makima::tts::qwen3::Qwen3Tts; +//! use candle_core::Device; +//! +//! let device = Device::Cpu; +//! let tts = Qwen3Tts::from_pretrained(None, &device).unwrap(); +//! // Use via TtsEngine trait or direct API +//! ``` + +pub mod code_predictor; +pub mod config; +pub mod generate; +pub mod model; +pub mod speech_tokenizer; + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use candle_core::{DType, Device}; +use candle_nn::VarBuilder; +use hf_hub::api::sync::Api; +use tokenizers::Tokenizer; + +use self::code_predictor::CodePredictor; +use self::config::Qwen3TtsConfig; +use self::generate::{GenerationConfig, GenerationContext}; +use self::model::Qwen3Model; +use self::speech_tokenizer::SpeechTokenizer; +use crate::tts::{AudioChunk, TtsEngine, TtsError, SAMPLE_RATE}; + +/// HuggingFace model IDs. +const LM_MODEL_ID: &str = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"; +const TOKENIZER_MODEL_ID: &str = "Qwen/Qwen3-TTS-Tokenizer-12Hz"; +const DEFAULT_MODEL_DIR: &str = "models/qwen3-tts"; + +/// Qwen3-TTS engine — pure Rust candle-based inference. +pub struct Qwen3Tts { + /// The 28-layer language model. + model: Qwen3Model, + /// Multi-token prediction code predictor. + code_predictor: CodePredictor, + /// Speech tokenizer (encoder + decoder + RVQ). + speech_tokenizer: SpeechTokenizer, + /// Text tokenizer. + tokenizer: Tokenizer, + /// Model configuration. + config: Qwen3TtsConfig, + /// Compute device (CPU/CUDA/Metal). + device: Device, + /// Whether the model is fully loaded and ready. + ready: AtomicBool, +} + +// SAFETY: All fields are either Send+Sync or behind appropriate synchronization. +// candle tensors are Send+Sync, Tokenizer is Send+Sync, AtomicBool is Send+Sync. +unsafe impl Send for Qwen3Tts {} +unsafe impl Sync for Qwen3Tts {} + +impl Qwen3Tts { + /// Load from a local directory or download from HuggingFace. + pub fn from_pretrained( + model_dir: Option<&str>, + device: &Device, + ) -> Result { + let model_path = PathBuf::from(model_dir.unwrap_or(DEFAULT_MODEL_DIR)); + + if !model_path.exists() { + Self::download_models(&model_path)?; + } + + Self::load_from_path(&model_path, device) + } + + /// Load all model components from a local directory. + pub fn load_from_path(model_dir: &Path, device: &Device) -> Result { + let dtype = DType::F32; // Use F32 for CPU; BF16/F16 for GPU + + // Load configuration + let config_path = model_dir.join("config.json"); + let config = if config_path.exists() { + Qwen3TtsConfig::from_json_path(&config_path)? + } else { + Qwen3TtsConfig::default() + }; + + // Load text tokenizer + let tokenizer_path = model_dir.join("tokenizer.json"); + let tokenizer = Tokenizer::from_file(&tokenizer_path) + .map_err(|e| TtsError::Tokenizer(format!("failed to load tokenizer: {e}")))?; + + // Load LM weights from safetensors + let lm_weights_path = model_dir.join("model.safetensors"); + let lm_data = std::fs::read(&lm_weights_path).map_err(|e| { + TtsError::ModelLoad(format!( + "failed to read LM weights from {}: {e}", + lm_weights_path.display() + )) + })?; + let lm_vb = VarBuilder::from_buffered_safetensors( + lm_data, + dtype, + device, + ).map_err(|e| TtsError::ModelLoad(format!("failed to create LM VarBuilder: {e}")))?; + + // Build language model + let model = Qwen3Model::new(&config.lm, lm_vb.clone()).map_err(|e| { + TtsError::ModelLoad(format!("failed to build LM model: {e}")) + })?; + + // Build code predictor (weights are in the same safetensors file) + let code_predictor = + CodePredictor::new(&config.code_predictor, &config.lm, lm_vb).map_err(|e| { + TtsError::ModelLoad(format!("failed to build code predictor: {e}")) + })?; + + // Load speech tokenizer from separate safetensors + let st_weights_path = model_dir.join("speech_tokenizer.safetensors"); + let st_data = std::fs::read(&st_weights_path).map_err(|e| { + TtsError::ModelLoad(format!( + "failed to read speech tokenizer weights from {}: {e}", + st_weights_path.display() + )) + })?; + let st_vb = VarBuilder::from_buffered_safetensors( + st_data, + dtype, + device, + ).map_err(|e| { + TtsError::ModelLoad(format!( + "failed to create speech tokenizer VarBuilder: {e}" + )) + })?; + + let speech_tokenizer = + SpeechTokenizer::new(&config.speech_tokenizer, st_vb, device).map_err(|e| { + TtsError::ModelLoad(format!("failed to build speech tokenizer: {e}")) + })?; + + Ok(Self { + model, + code_predictor, + speech_tokenizer, + tokenizer, + config, + device: device.clone(), + ready: AtomicBool::new(true), + }) + } + + /// Generate audio from text with optional voice reference. + pub fn generate_speech( + &self, + text: &str, + reference_audio: Option<&[f32]>, + gen_config: Option, + ) -> Result, TtsError> { + let config = gen_config.unwrap_or_default(); + + let ctx = GenerationContext::new( + &self.model, + &self.code_predictor, + &self.speech_tokenizer, + &self.tokenizer, + &self.device, + config, + ); + + ctx.generate(text, reference_audio) + } + + /// Download model files from HuggingFace Hub. + fn download_models(target_dir: &Path) -> Result<(), TtsError> { + std::fs::create_dir_all(target_dir)?; + + let api = Api::new().map_err(|e| TtsError::ModelLoad(e.to_string()))?; + + // Download LM model files + println!("Downloading Qwen3-TTS language model..."); + let lm_repo = api.model(LM_MODEL_ID.to_string()); + + let lm_files = [ + "model.safetensors", + "config.json", + "tokenizer.json", + "tokenizer_config.json", + ]; + + for file in &lm_files { + println!(" Downloading {file}..."); + let downloaded = lm_repo + .get(file) + .map_err(|e| TtsError::ModelLoad(format!("failed to download {file}: {e}")))?; + + let target = target_dir.join(file); + if !target.exists() { + std::fs::copy(&downloaded, &target)?; + } + } + + // Download speech tokenizer + println!("Downloading Qwen3-TTS speech tokenizer..."); + let st_repo = api.model(TOKENIZER_MODEL_ID.to_string()); + + let st_file = "model.safetensors"; + let downloaded = st_repo + .get(st_file) + .map_err(|e| { + TtsError::ModelLoad(format!("failed to download speech tokenizer: {e}")) + })?; + + let target = target_dir.join("speech_tokenizer.safetensors"); + if !target.exists() { + std::fs::copy(&downloaded, &target)?; + } + + println!("All models downloaded to {}", target_dir.display()); + Ok(()) + } + + /// Get the model configuration. + pub fn config(&self) -> &Qwen3TtsConfig { + &self.config + } + + /// Get the compute device. + pub fn device(&self) -> &Device { + &self.device + } +} + +#[async_trait::async_trait] +impl TtsEngine for Qwen3Tts { + async fn generate( + &self, + text: &str, + reference_audio: Option<&[f32]>, + _reference_sample_rate: Option, + ) -> Result, TtsError> { + // Note: reference audio should already be resampled to 24kHz + // by the caller. If a different sample rate is provided, + // the caller should resample using `resample_to_24k()`. + self.generate_speech(text, reference_audio, None) + } + + fn is_ready(&self) -> bool { + self.ready.load(Ordering::Relaxed) + } + + fn sample_rate(&self) -> u32 { + SAMPLE_RATE + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config() { + let config = Qwen3TtsConfig::default(); + assert_eq!(config.lm.hidden_size, 1024); + assert_eq!(config.lm.num_hidden_layers, 28); + assert_eq!(config.code_predictor.num_code_groups, 16); + assert_eq!(config.speech_tokenizer.sample_rate, 24_000); + } + + #[test] + fn test_model_ids() { + assert_eq!(LM_MODEL_ID, "Qwen/Qwen3-TTS-12Hz-0.6B-Base"); + assert_eq!(TOKENIZER_MODEL_ID, "Qwen/Qwen3-TTS-Tokenizer-12Hz"); + } +} -- cgit v1.2.3