Experiment: ChatterBoxTTS

4 files changed, 1050 insertions, 40 deletions

diff --git a/makima/src/audio.rs b/makima/src/audio.rs
new file mode 100644
index 0000000..acfe7ce
--- /dev/null
+++ b/makima/src/audio.rs
@@ -0,0 +1,375 @@
+use std::fs::File;
+use std::io::{self, Read, Seek};
+use std::path::Path;
+
+use symphonia::core::audio::{AudioBufferRef, Signal};
+use symphonia::core::codecs::{DecoderOptions, CODEC_TYPE_NULL};
+use symphonia::core::errors::Error as SymphoniaError;
+use symphonia::core::formats::FormatOptions;
+use symphonia::core::io::{MediaSourceStream, ReadOnlySource};
+use symphonia::core::meta::MetadataOptions;
+use symphonia::core::probe::Hint;
+
+pub const TARGET_SAMPLE_RATE: u32 = 16_000;
+pub const TARGET_CHANNELS: u16 = 1;
+
+#[derive(Debug, Clone)]
+pub struct PcmAudio {
+    pub samples: Vec<f32>,
+    pub sample_rate: u32,
+    pub channels: u16,
+}
+
+#[derive(Debug)]
+pub enum AudioError {
+    Io(io::Error),
+    Decode(String),
+    UnsupportedFormat,
+    NoAudioTrack,
+}
+
+impl std::fmt::Display for AudioError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            AudioError::Io(err) => write!(f, "io error: {err}"),
+            AudioError::Decode(err) => write!(f, "decode error: {err}"),
+            AudioError::UnsupportedFormat => write!(f, "unsupported audio format"),
+            AudioError::NoAudioTrack => write!(f, "no audio track found"),
+        }
+    }
+}
+
+impl std::error::Error for AudioError {}
+
+impl From<io::Error> for AudioError {
+    fn from(value: io::Error) -> Self {
+        AudioError::Io(value)
+    }
+}
+
+impl From<SymphoniaError> for AudioError {
+    fn from(value: SymphoniaError) -> Self {
+        match value {
+            SymphoniaError::IoError(e) => AudioError::Io(e),
+            SymphoniaError::Unsupported(_) => AudioError::UnsupportedFormat,
+            other => AudioError::Decode(other.to_string()),
+        }
+    }
+}
+
+pub fn to_16k_mono_from_path(path: impl AsRef<Path>) -> Result<PcmAudio, AudioError> {
+    let path = path.as_ref();
+    let file = File::open(path)?;
+
+    let mut hint = Hint::new();
+    if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
+        hint.with_extension(ext);
+    }
+
+    decode_to_16k_mono(file, hint)
+}
+
+pub fn to_16k_mono_from_reader<R: Read + Seek + Send + Sync + 'static>(
+    reader: R,
+) -> Result<PcmAudio, AudioError> {
+    decode_to_16k_mono(reader, Hint::new())
+}
+
+fn decode_to_16k_mono<R: Read + Seek + Send + Sync + 'static>(
+    reader: R,
+    hint: Hint,
+) -> Result<PcmAudio, AudioError> {
+    let source = MediaSourceStream::new(Box::new(ReadOnlySource::new(reader)), Default::default());
+
+    let format_opts = FormatOptions::default();
+    let metadata_opts = MetadataOptions::default();
+
+    let probed = symphonia::default::get_probe().format(&hint, source, &format_opts, &metadata_opts)?;
+    let mut format = probed.format;
+
+    let track = format
+        .tracks()
+        .iter()
+        .find(|t| t.codec_params.codec != CODEC_TYPE_NULL)
+        .ok_or(AudioError::NoAudioTrack)?;
+
+    let track_id = track.id;
+    let codec_params = track.codec_params.clone();
+
+    let sample_rate = codec_params.sample_rate.ok_or(AudioError::Decode(
+        "unknown sample rate".to_string(),
+    ))?;
+    let channels = codec_params
+        .channels
+        .map(|c| c.count() as u16)
+        .unwrap_or(1);
+
+    let decoder_opts = DecoderOptions::default();
+    let mut decoder = symphonia::default::get_codecs().make(&codec_params, &decoder_opts)?;
+
+    let mut interleaved: Vec<f32> = Vec::new();
+
+    loop {
+        let packet = match format.next_packet() {
+            Ok(p) => p,
+            Err(SymphoniaError::IoError(ref e)) if e.kind() == io::ErrorKind::UnexpectedEof => break,
+            Err(SymphoniaError::ResetRequired) => {
+                decoder.reset();
+                continue;
+            }
+            Err(e) => return Err(e.into()),
+        };
+
+        if packet.track_id() != track_id {
+            continue;
+        }
+
+        let decoded = match decoder.decode(&packet) {
+            Ok(d) => d,
+            Err(SymphoniaError::DecodeError(_)) => continue,
+            Err(e) => return Err(e.into()),
+        };
+
+        append_samples(&decoded, &mut interleaved);
+    }
+
+    let mono = mixdown_to_mono(&interleaved, channels);
+    let samples = resample_sinc(&mono, sample_rate, TARGET_SAMPLE_RATE);
+
+    Ok(PcmAudio {
+        samples,
+        sample_rate: TARGET_SAMPLE_RATE,
+        channels: TARGET_CHANNELS,
+    })
+}
+
+fn append_samples(buffer: &AudioBufferRef, out: &mut Vec<f32>) {
+    match buffer {
+        AudioBufferRef::U8(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push((plane[frame] as f32 - 128.0) / 128.0);
+                }
+            }
+        }
+        AudioBufferRef::U16(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push((plane[frame] as f32 - 32768.0) / 32768.0);
+                }
+            }
+        }
+        AudioBufferRef::U24(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push((plane[frame].inner() as f32 - 8388608.0) / 8388608.0);
+                }
+            }
+        }
+        AudioBufferRef::U32(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push((plane[frame] as f64 - 2147483648.0) as f32 / 2147483648.0);
+                }
+            }
+        }
+        AudioBufferRef::S8(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push(plane[frame] as f32 / 128.0);
+                }
+            }
+        }
+        AudioBufferRef::S16(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push(plane[frame] as f32 / 32768.0);
+                }
+            }
+        }
+        AudioBufferRef::S24(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push(plane[frame].inner() as f32 / 8388608.0);
+                }
+            }
+        }
+        AudioBufferRef::S32(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push(plane[frame] as f32 / 2147483648.0);
+                }
+            }
+        }
+        AudioBufferRef::F32(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push(plane[frame]);
+                }
+            }
+        }
+        AudioBufferRef::F64(buf) => {
+            for frame in 0..buf.frames() {
+                for plane in buf.planes().planes() {
+                    out.push(plane[frame] as f32);
+                }
+            }
+        }
+    }
+}
+
+fn mixdown_to_mono(interleaved: &[f32], channels: u16) -> Vec<f32> {
+    if channels <= 1 {
+        return interleaved.to_vec();
+    }
+
+    let channels = channels as usize;
+    let frames = interleaved.len() / channels;
+
+    let mut mono = Vec::with_capacity(frames);
+    for frame in 0..frames {
+        let base = frame * channels;
+        let mut acc = 0.0f32;
+        for c in 0..channels {
+            acc += interleaved[base + c];
+        }
+        mono.push(acc / channels as f32);
+    }
+
+    mono
+}
+
+fn resample_sinc(input: &[f32], input_rate: u32, output_rate: u32) -> Vec<f32> {
+    if input_rate == output_rate {
+        return input.to_vec();
+    }
+    if input.is_empty() {
+        return Vec::new();
+    }
+
+    let ratio = input_rate as f64 / output_rate as f64;
+    let output_len = ((input.len() as f64) / ratio).ceil() as usize;
+
+    let cutoff = (output_rate as f64 / input_rate as f64).min(1.0);
+
+    let radius: i32 = 32;
+    let radius_f = radius as f64;
+    let pi = std::f64::consts::PI;
+
+    let mut output = Vec::with_capacity(output_len);
+    for n in 0..output_len {
+        let t = n as f64 * ratio;
+        let center = t.floor() as i32;
+        let frac = t - (center as f64);
+
+        let mut acc = 0.0f64;
+        let mut norm = 0.0f64;
+
+        for k in -radius..=radius {
+            let idx = center + k;
+            if idx < 0 || (idx as usize) >= input.len() {
+                continue;
+            }
+
+            let x = (k as f64) - frac;
+            let d = x.abs();
+            if d > radius_f {
+                continue;
+            }
+
+            let window = 0.5 * (1.0 + (pi * d / radius_f).cos());
+
+            let z = x * cutoff;
+            let sinc = if z == 0.0 {
+                1.0
+            } else {
+                let pz = pi * z;
+                pz.sin() / pz
+            };
+
+            let weight = cutoff * sinc * window;
+            acc += input[idx as usize] as f64 * weight;
+            norm += weight;
+        }
+
+        let y = if norm == 0.0 { 0.0 } else { acc / norm };
+        output.push(y as f32);
+    }
+
+    output
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Cursor;
+
+    fn create_wav_buffer(sample_rate: u32, channels: u16, samples: &[i16]) -> Vec<u8> {
+        let mut buf = Vec::new();
+        let data_size = (samples.len() * 2) as u32;
+        let file_size = 36 + data_size;
+
+        buf.extend_from_slice(b"RIFF");
+        buf.extend_from_slice(&file_size.to_le_bytes());
+        buf.extend_from_slice(b"WAVE");
+
+        buf.extend_from_slice(b"fmt ");
+        buf.extend_from_slice(&16u32.to_le_bytes());
+        buf.extend_from_slice(&1u16.to_le_bytes());
+        buf.extend_from_slice(&channels.to_le_bytes());
+        buf.extend_from_slice(&sample_rate.to_le_bytes());
+        let byte_rate = sample_rate * channels as u32 * 2;
+        buf.extend_from_slice(&byte_rate.to_le_bytes());
+        let block_align = channels * 2;
+        buf.extend_from_slice(&block_align.to_le_bytes());
+        buf.extend_from_slice(&16u16.to_le_bytes());
+
+        buf.extend_from_slice(b"data");
+        buf.extend_from_slice(&data_size.to_le_bytes());
+        for &s in samples {
+            buf.extend_from_slice(&s.to_le_bytes());
+        }
+
+        buf
+    }
+
+    #[test]
+    fn converts_stereo_to_mono() {
+        let mut samples = Vec::new();
+        for _ in 0..(TARGET_SAMPLE_RATE / 10) {
+            samples.push(10_000i16);
+            samples.push(0i16);
+        }
+
+        let wav = create_wav_buffer(TARGET_SAMPLE_RATE, 2, &samples);
+        let cursor = Cursor::new(wav);
+
+        let normalized = to_16k_mono_from_reader(cursor).unwrap();
+
+        assert_eq!(normalized.sample_rate, TARGET_SAMPLE_RATE);
+        assert_eq!(normalized.channels, TARGET_CHANNELS);
+        let mean =
+            normalized.samples.iter().copied().sum::<f32>() / normalized.samples.len() as f32;
+        let expected = (10_000.0 / 32768.0) / 2.0;
+        assert!((mean - expected).abs() < 1e-3);
+    }
+
+    #[test]
+    fn resamples_to_16k() {
+        let samples: Vec<i16> = vec![0; 48_000];
+        let wav = create_wav_buffer(48_000, 1, &samples);
+        let cursor = Cursor::new(wav);
+
+        let normalized = to_16k_mono_from_reader(cursor).unwrap();
+
+        assert_eq!(normalized.sample_rate, TARGET_SAMPLE_RATE);
+        assert_eq!(normalized.channels, TARGET_CHANNELS);
+        assert_eq!(normalized.samples.len(), TARGET_SAMPLE_RATE as usize);
+        let max_abs = normalized
+            .samples
+            .iter()
+            .copied()
+            .fold(0.0f32, |m, v| m.max(v.abs()));
+        assert!(max_abs <= 1e-6);
+    }
+}
diff --git a/makima/src/listen.rs b/makima/src/listen.rs
index a0f4246..cd0a394 100644
--- a/makima/src/listen.rs
+++ b/makima/src/listen.rs
@@ -1,11 +1,12 @@
 use std::cmp::Ordering;
 use std::path::Path;
 
-use hound::WavReader;
 use parakeet_rs::sortformer::{DiarizationConfig, Sortformer, SpeakerSegment};
 use parakeet_rs::{ParakeetTDT, TimedToken, TimestampMode};
 
-const SAMPLE_RATE: u32 = 16_000;
+use crate::audio;
+
+const STREAM_CHUNK_MS: u32 = 5_000;
 
 pub struct DialogueSegment {
     pub speaker: String,
@@ -15,57 +16,54 @@ pub struct DialogueSegment {
 }
 
 pub(crate) fn listen() -> Result<Vec<DialogueSegment>, Box<dyn std::error::Error>> {
-    let audio_path = Path::new("audio.wav");
+    let audio_path = Path::new("audio-ftc.mp3");
 
-    let (audio, sample_rate, channels) = load_audio(audio_path)?;
+    let normalized = audio::to_16k_mono_from_path(audio_path)?;
 
     let mut parakeet = ParakeetTDT::from_pretrained("models/parakeet-tdt-0.6b-v3", None)?;
-    let transcription = parakeet.transcribe_samples(
-        audio.clone(),
-        sample_rate,
-        channels,
-        Some(TimestampMode::Sentences),
-    )?;
-
     let mut sortformer = Sortformer::with_config(
         "models/diarization/diar_streaming_sortformer_4spk-v2.onnx",
         None,
         DiarizationConfig::callhome(),
     )?;
-    let diarization_segments = sortformer.diarize(audio, sample_rate, channels)?;
-
-    let segments = align_speakers(&transcription.tokens, &diarization_segments);
-    for segment in &segments {
-        println!(
-            "[{:.2}s - {:.2}s] {}: {}",
-            segment.start, segment.end, segment.speaker, segment.text
-        );
-    }
 
-    Ok(segments)
-}
+    let chunk_samples = samples_per_chunk(normalized.sample_rate, STREAM_CHUNK_MS);
+    let mut cumulative_audio: Vec<f32> = Vec::new();
+    let mut last_printed_tokens = 0usize;
+    let mut final_segments: Vec<DialogueSegment> = Vec::new();
 
-fn load_audio(path: &Path) -> Result<(Vec<f32>, u32, u16), Box<dyn std::error::Error>> {
-    let mut reader = WavReader::open(path)?;
-    let spec = reader.spec();
+    for (chunk_idx, chunk) in normalized.samples.chunks(chunk_samples).enumerate() {
+        cumulative_audio.extend_from_slice(chunk);
 
-    if spec.sample_rate != SAMPLE_RATE {
-        return Err(format!(
-            "Expected {} Hz audio, got {} Hz",
-            SAMPLE_RATE, spec.sample_rate
-        )
-        .into());
-    }
+        let diarization_segments = sortformer.diarize(
+            cumulative_audio.clone(),
+            normalized.sample_rate,
+            normalized.channels,
+        )?;
+
+        let transcription = parakeet.transcribe_samples(
+            cumulative_audio.clone(),
+            normalized.sample_rate,
+            normalized.channels,
+            Some(TimestampMode::Sentences),
+        )?;
+
+        final_segments = align_speakers(&transcription.tokens, &diarization_segments);
 
-    let samples = match spec.sample_format {
-        hound::SampleFormat::Float => reader.samples::<f32>().collect::<Result<Vec<_>, _>>()?,
-        hound::SampleFormat::Int => reader
-            .samples::<i16>()
-            .map(|s| s.map(|s| s as f32 / 32768.0))
-            .collect::<Result<Vec<_>, _>>()?,
-    };
+        // Simulate "live" output by printing only newly emitted tokens.
+        if transcription.tokens.len() > last_printed_tokens {
+            let new_segments = &final_segments[last_printed_tokens..];
+            for segment in new_segments {
+                println!(
+                    "[chunk {}] [{:.2}s - {:.2}s] {}: {}",
+                    chunk_idx, segment.start, segment.end, segment.speaker, segment.text
+                );
+            }
+            last_printed_tokens = transcription.tokens.len();
+        }
+    }
 
-    Ok((samples, spec.sample_rate, spec.channels))
+    Ok(final_segments)
 }
 
 fn align_speakers(tokens: &[TimedToken], speakers: &[SpeakerSegment]) -> Vec<DialogueSegment> {
@@ -84,6 +82,25 @@ fn align_speakers(tokens: &[TimedToken], speakers: &[SpeakerSegment]) -> Vec<Dia
         .collect()
 }
 
+fn samples_per_chunk(sample_rate: u32, chunk_ms: u32) -> usize {
+    let samples = (sample_rate as u64)
+        .saturating_mul(chunk_ms as u64)
+        .saturating_div(1_000);
+    samples.max(1) as usize
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn samples_per_chunk_rounds_down_and_clamps() {
+        assert_eq!(samples_per_chunk(16_000, 1_000), 16_000);
+        assert_eq!(samples_per_chunk(16_000, 160), 2_560);
+        assert_eq!(samples_per_chunk(16_000, 0), 1);
+    }
+}
+
 fn speaker_for_span(start: f32, end: f32, speakers: &[SpeakerSegment]) -> Option<String> {
     speakers
         .iter()
diff --git a/makima/src/main.rs b/makima/src/main.rs
index 9097ef6..2348b23 100644
--- a/makima/src/main.rs
+++ b/makima/src/main.rs
@@ -1,6 +1,44 @@
+use std::path::Path;
+use crate::tts::{save_wav, ChatterboxTTS};
+
+mod audio;
 mod listen;
+pub mod tts;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("Loading ChatterboxTTS...");
+    let mut tts = ChatterboxTTS::from_pretrained(None)?;
+    println!("Model loaded successfully!");
+
+    // // Voice cloning using existing audio file
+    // println!("Generating TTS with voice cloning...");
+    // let audio = tts.generate_tts_with_voice(
+    //     "Hello, this is a test of the voice cloning system.",
+    //     Path::new("audio.wav")
+    // )?;
+    //
+    // println!("Generated {} samples", audio.len());
+    // save_wav(&audio, Path::new("output.wav"))?;
+    // println!("Saved to output.wav");
+
+
+    // Load reference audio from mp3
+    println!("Loading reference audio...");
+    let reference = audio::to_16k_mono_from_path(Path::new("audio.mp3"))?;
+    let samples = &reference.samples;
+    let sample_rate = reference.sample_rate;
+
+    // Voice cloning using audio samples
+    println!("Generating TTS with voice cloning...");
+    let audio = tts.generate_tts_with_samples(
+        "Hello, this is a test of the voice cloning system [chuckles]. Repeat after me \" I am Steve Jobs!\"",
+        samples,
+        sample_rate,
+    )?;
+
+    println!("Generated {} samples", audio.len());
+    save_wav(&audio, Path::new("output.wav"))?;
+    println!("Saved to output.wav");
     let segments = listen::listen()?;
     println!("Captured {} diarized segments", segments.len());
     Ok(())
diff --git a/makima/src/tts.rs b/makima/src/tts.rs
new file mode 100644
index 0000000..5198938
--- /dev/null
+++ b/makima/src/tts.rs
@@ -0,0 +1,580 @@
+use std::path::{Path, PathBuf};
+use std::fs;
+
+use hf_hub::api::sync::Api;
+use std::borrow::Cow;
+
+use ndarray::{ArrayD, Array2, Array3, Array4, IxDyn};
+use ort::session::Session;
+use ort::value::{Value, DynValue};
+use tokenizers::Tokenizer;
+
+use crate::audio;
+
+pub const SAMPLE_RATE: u32 = 24_000;
+const START_SPEECH_TOKEN: i64 = 6561;
+const STOP_SPEECH_TOKEN: i64 = 6562;
+const SILENCE_TOKEN: i64 = 4299;
+const NUM_LAYERS: usize = 24;
+const NUM_KV_HEADS: usize = 16;
+const HEAD_DIM: usize = 64;
+
+const MODEL_ID: &str = "ResembleAI/chatterbox-turbo-ONNX";
+const DEFAULT_MODEL_DIR: &str = "models/chatterbox-turbo";
+
+#[derive(Debug)]
+pub enum TtsError {
+    ModelLoad(String),
+    Inference(String),
+    Tokenizer(String),
+    Audio(audio::AudioError),
+    Io(std::io::Error),
+    VoiceRequired,
+}
+
+impl std::fmt::Display for TtsError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TtsError::ModelLoad(msg) => write!(f, "model load error: {msg}"),
+            TtsError::Inference(msg) => write!(f, "inference error: {msg}"),
+            TtsError::Tokenizer(msg) => write!(f, "tokenizer error: {msg}"),
+            TtsError::Audio(err) => write!(f, "audio error: {err}"),
+            TtsError::Io(err) => write!(f, "io error: {err}"),
+            TtsError::VoiceRequired => write!(f, "voice reference audio is required for chatterbox-turbo"),
+        }
+    }
+}
+
+impl std::error::Error for TtsError {}
+
+impl From<audio::AudioError> for TtsError {
+    fn from(value: audio::AudioError) -> Self {
+        TtsError::Audio(value)
+    }
+}
+
+impl From<std::io::Error> for TtsError {
+    fn from(value: std::io::Error) -> Self {
+        TtsError::Io(value)
+    }
+}
+
+impl From<ort::Error> for TtsError {
+    fn from(value: ort::Error) -> Self {
+        TtsError::ModelLoad(value.to_string())
+    }
+}
+
+pub struct ChatterboxTTS {
+    speech_encoder: Session,
+    embed_tokens: Session,
+    language_model: Session,
+    conditional_decoder: Session,
+    tokenizer: Tokenizer,
+}
+
+struct VoiceCondition {
+    audio_features: ArrayD<f32>,
+    prompt_tokens: ArrayD<i64>,
+    speaker_embeddings: ArrayD<f32>,
+    speaker_features: ArrayD<f32>,
+}
+
+fn extract_f32_tensor(value: &Value) -> Result<ArrayD<f32>, TtsError> {
+    let (shape, data) = value
+        .try_extract_tensor::<f32>()
+        .map_err(|e| TtsError::Inference(e.to_string()))?;
+
+    let dims: Vec<usize> = shape.iter().map(|&d| d as usize).collect();
+    ArrayD::from_shape_vec(IxDyn(&dims), data.to_vec())
+        .map_err(|e| TtsError::Inference(e.to_string()))
+}
+
+fn extract_i64_tensor(value: &Value) -> Result<ArrayD<i64>, TtsError> {
+    let (shape, data) = value
+        .try_extract_tensor::<i64>()
+        .map_err(|e| TtsError::Inference(e.to_string()))?;
+
+    let dims: Vec<usize> = shape.iter().map(|&d| d as usize).collect();
+    ArrayD::from_shape_vec(IxDyn(&dims), data.to_vec())
+        .map_err(|e| TtsError::Inference(e.to_string()))
+}
+
+impl ChatterboxTTS {
+    pub fn from_pretrained(model_dir: Option<&str>) -> Result<Self, TtsError> {
+        let model_path = PathBuf::from(model_dir.unwrap_or(DEFAULT_MODEL_DIR));
+
+        if !model_path.exists() {
+            download_models(&model_path)?;
+        }
+
+        Self::load_from_path(&model_path)
+    }
+
+    pub fn load_from_path(model_dir: &Path) -> Result<Self, TtsError> {
+        let speech_encoder = Session::builder()?
+            .with_intra_threads(4)?
+            .commit_from_file(model_dir.join("speech_encoder.onnx"))?;
+
+        let embed_tokens = Session::builder()?
+            .with_intra_threads(4)?
+            .commit_from_file(model_dir.join("embed_tokens.onnx"))?;
+
+        let language_model = Session::builder()?
+            .with_intra_threads(4)?
+            .commit_from_file(model_dir.join("language_model.onnx"))?;
+
+        let conditional_decoder = Session::builder()?
+            .with_intra_threads(4)?
+            .commit_from_file(model_dir.join("conditional_decoder.onnx"))?;
+
+        let tokenizer_path = model_dir.join("tokenizer.json");
+        let tokenizer = Tokenizer::from_file(&tokenizer_path)
+            .map_err(|e| TtsError::Tokenizer(e.to_string()))?;
+
+        Ok(Self {
+            speech_encoder,
+            embed_tokens,
+            language_model,
+            conditional_decoder,
+            tokenizer,
+        })
+    }
+
+    pub fn generate_tts(&mut self, _text: &str) -> Result<Vec<f32>, TtsError> {
+        // Chatterbox TTS requires voice reference audio
+        Err(TtsError::VoiceRequired)
+    }
+
+    pub fn generate_tts_with_voice(
+        &mut self,
+        text: &str,
+        sample_audio_path: &Path,
+    ) -> Result<Vec<f32>, TtsError> {
+        let audio = audio::to_16k_mono_from_path(sample_audio_path)?;
+        let resampled = resample_to_24k(&audio.samples, audio.sample_rate);
+        self.generate_tts_with_samples(text, &resampled, SAMPLE_RATE)
+    }
+
+    pub fn generate_tts_with_samples(
+        &mut self,
+        text: &str,
+        samples: &[f32],
+        sample_rate: u32,
+    ) -> Result<Vec<f32>, TtsError> {
+        let resampled = if sample_rate != SAMPLE_RATE {
+            resample_to_24k(samples, sample_rate)
+        } else {
+            samples.to_vec()
+        };
+
+        // 1. Encode reference audio
+        let voice_condition = self.encode_voice(&resampled)?;
+
+        // 2. Tokenize text
+        let encoding = self
+            .tokenizer
+            .encode(text, true)
+            .map_err(|e| TtsError::Tokenizer(e.to_string()))?;
+
+        let text_input_ids: Vec<i64> = encoding.get_ids().iter().map(|&id| id as i64).collect();
+
+        // 3. Generate speech tokens
+        let generated_tokens = self.generate_speech_tokens(
+            &text_input_ids,
+            &voice_condition.audio_features,
+        )?;
+
+        // 4. Prepare final speech tokens: prompt_tokens + generated + silence
+        let prompt_tokens: Vec<i64> = voice_condition.prompt_tokens.iter().copied().collect();
+        let silence_tokens = vec![SILENCE_TOKEN; 3];
+
+        let mut final_tokens = Vec::with_capacity(
+            prompt_tokens.len() + generated_tokens.len() + silence_tokens.len()
+        );
+        final_tokens.extend_from_slice(&prompt_tokens);
+        final_tokens.extend_from_slice(&generated_tokens);
+        final_tokens.extend_from_slice(&silence_tokens);
+
+        // 5. Decode to audio
+        let audio_samples = self.decode_speech_tokens(
+            &final_tokens,
+            &voice_condition.speaker_embeddings,
+            &voice_condition.speaker_features,
+        )?;
+
+        Ok(audio_samples)
+    }
+
+    fn encode_voice(&mut self, samples: &[f32]) -> Result<VoiceCondition, TtsError> {
+        let audio_arr = Array2::from_shape_vec((1, samples.len()), samples.to_vec())
+            .map_err(|e| TtsError::Inference(e.to_string()))?;
+
+        let audio_tensor = Value::from_array(audio_arr)?;
+
+        let outputs = self.speech_encoder.run(ort::inputs!["audio_values" => audio_tensor])?;
+
+        // Order: audio_features, audio_tokens (prompt_token), speaker_embeddings, speaker_features
+        let audio_features = extract_f32_tensor(&outputs[0])?;
+        let prompt_tokens = extract_i64_tensor(&outputs[1])?;
+        let speaker_embeddings = extract_f32_tensor(&outputs[2])?;
+        let speaker_features = extract_f32_tensor(&outputs[3])?;
+
+        Ok(VoiceCondition {
+            audio_features,
+            prompt_tokens,
+            speaker_embeddings,
+            speaker_features,
+        })
+    }
+
+    fn generate_speech_tokens(
+        &mut self,
+        text_input_ids: &[i64],
+        audio_features: &ArrayD<f32>,
+    ) -> Result<Vec<i64>, TtsError> {
+        let max_new_tokens: usize = 1024;
+        let repetition_penalty: f32 = 1.2;
+
+        // Start with START_SPEECH_TOKEN
+        let mut generate_tokens: Vec<i64> = vec![START_SPEECH_TOKEN];
+
+        // Initialize empty KV cache (seq_len = 0)
+        let mut past_key_values = self.init_kv_cache(0)?;
+
+        let mut first_iteration = true;
+        let mut total_seq_len: usize = 0;
+
+        for _ in 0..max_new_tokens {
+            // Get embeddings for current input_ids
+            let current_input_ids = if first_iteration {
+                // First iteration: use text input_ids
+                text_input_ids.to_vec()
+            } else {
+                // Subsequent iterations: use last generated token
+                vec![*generate_tokens.last().unwrap()]
+            };
+
+            let input_ids_arr = Array2::from_shape_vec(
+                (1, current_input_ids.len()),
+                current_input_ids
+            ).map_err(|e| TtsError::Inference(e.to_string()))?;
+
+            let input_ids_tensor = Value::from_array(input_ids_arr)?;
+
+            let inputs_embeds = {
+                let embed_outputs = self.embed_tokens.run(ort::inputs![input_ids_tensor])?;
+                extract_f32_tensor(&embed_outputs[0])?
+            };
+
+            // On first iteration, concatenate audio features with text embeddings
+            let inputs_embeds = if first_iteration {
+                let audio_feat_3d = audio_features.view()
+                    .into_dimensionality::<ndarray::Ix3>()
+                    .map_err(|e| TtsError::Inference(e.to_string()))?;
+                let text_emb_3d = inputs_embeds.view()
+                    .into_dimensionality::<ndarray::Ix3>()
+                    .map_err(|e| TtsError::Inference(e.to_string()))?;
+
+                ndarray::concatenate(ndarray::Axis(1), &[audio_feat_3d, text_emb_3d])
+                    .map_err(|e| TtsError::Inference(e.to_string()))?
+            } else {
+                inputs_embeds.view()
+                    .into_dimensionality::<ndarray::Ix3>()
+                    .map_err(|e| TtsError::Inference(e.to_string()))?
+                    .to_owned()
+            };
+
+            let seq_len = inputs_embeds.shape()[1];
+
+            // Set up attention mask and position ids
+            let (attention_mask, position_ids) = if first_iteration {
+                total_seq_len = seq_len;
+                let attention_mask: Array2<i64> = Array2::ones((1, seq_len));
+                let position_ids = Array2::from_shape_fn((1, seq_len), |(_, j)| j as i64);
+                (attention_mask, position_ids)
+            } else {
+                total_seq_len += 1;
+                let attention_mask: Array2<i64> = Array2::ones((1, total_seq_len));
+                let position_ids = Array2::from_shape_vec(
+                    (1, 1),
+                    vec![(total_seq_len - 1) as i64]
+                ).map_err(|e| TtsError::Inference(e.to_string()))?;
+                (attention_mask, position_ids)
+            };
+
+            // Run language model
+            let (logits, new_kv) = self.run_language_model(
+                inputs_embeds,
+                position_ids,
+                attention_mask,
+                past_key_values,
+            )?;
+
+            past_key_values = new_kv;
+
+            // Get last logits
+            let logits_3d = logits.view().into_dimensionality::<ndarray::Ix3>()
+                .map_err(|e| TtsError::Inference(e.to_string()))?;
+            let last_idx = logits_3d.shape()[1] - 1;
+
+            let mut current_logits: Vec<f32> = logits_3d
+                .slice(ndarray::s![0, last_idx, ..])
+                .iter()
+                .copied()
+                .collect();
+
+            // Apply repetition penalty
+            apply_repetition_penalty(&mut current_logits, &generate_tokens, repetition_penalty);
+
+            // Get next token
+            let next_token = argmax(&current_logits);
+
+            generate_tokens.push(next_token);
+
+            if next_token == STOP_SPEECH_TOKEN {
+                break;
+            }
+
+            first_iteration = false;
+        }
+
+        // Return tokens without START and STOP tokens: [1:-1]
+        if generate_tokens.len() > 2 {
+            Ok(generate_tokens[1..generate_tokens.len()-1].to_vec())
+        } else {
+            Ok(Vec::new())
+        }
+    }
+
+    fn init_kv_cache(&self, seq_len: usize) -> Result<Vec<Array4<f32>>, TtsError> {
+        let mut cache = Vec::with_capacity(NUM_LAYERS * 2);
+        for _ in 0..NUM_LAYERS {
+            let key = Array4::<f32>::zeros((1, NUM_KV_HEADS, seq_len, HEAD_DIM));
+            let value = Array4::<f32>::zeros((1, NUM_KV_HEADS, seq_len, HEAD_DIM));
+            cache.push(key);
+            cache.push(value);
+        }
+        Ok(cache)
+    }
+
+    fn run_language_model(
+        &mut self,
+        inputs_embeds: Array3<f32>,
+        position_ids: Array2<i64>,
+        attention_mask: Array2<i64>,
+        past_key_values: Vec<Array4<f32>>,
+    ) -> Result<(ArrayD<f32>, Vec<Array4<f32>>), TtsError> {
+        let mut inputs: Vec<(Cow<str>, DynValue)> = Vec::new();
+
+        inputs.push((Cow::from("inputs_embeds"), Value::from_array(inputs_embeds)?.into_dyn()));
+        inputs.push((Cow::from("position_ids"), Value::from_array(position_ids)?.into_dyn()));
+        inputs.push((Cow::from("attention_mask"), Value::from_array(attention_mask)?.into_dyn()));
+
+        // Add KV cache inputs
+        for layer_idx in 0..NUM_LAYERS {
+            let key_name = format!("past_key_values.{}.key", layer_idx);
+            let value_name = format!("past_key_values.{}.value", layer_idx);
+
+            let key_tensor = Value::from_array(past_key_values[layer_idx * 2].clone())?.into_dyn();
+            let value_tensor = Value::from_array(past_key_values[layer_idx * 2 + 1].clone())?.into_dyn();
+
+            inputs.push((Cow::from(key_name), key_tensor));
+            inputs.push((Cow::from(value_name), value_tensor));
+        }
+
+        let outputs = self.language_model.run(inputs)?;
+
+        let logits = extract_f32_tensor(&outputs[0])?;
+
+        let mut new_kv = Vec::with_capacity(NUM_LAYERS * 2);
+        for layer_idx in 0..NUM_LAYERS {
+            let key_idx = 1 + layer_idx * 2;
+            let value_idx = 2 + layer_idx * 2;
+
+            let key_arr = extract_f32_tensor(&outputs[key_idx])?;
+            let value_arr = extract_f32_tensor(&outputs[value_idx])?;
+
+            let key_4d = key_arr.into_dimensionality::<ndarray::Ix4>()
+                .map_err(|e| TtsError::Inference(e.to_string()))?;
+            let value_4d = value_arr.into_dimensionality::<ndarray::Ix4>()
+                .map_err(|e| TtsError::Inference(e.to_string()))?;
+
+            new_kv.push(key_4d.to_owned());
+            new_kv.push(value_4d.to_owned());
+        }
+
+        Ok((logits, new_kv))
+    }
+
+    fn decode_speech_tokens(
+        &mut self,
+        speech_tokens: &[i64],
+        speaker_embeddings: &ArrayD<f32>,
+        speaker_features: &ArrayD<f32>,
+    ) -> Result<Vec<f32>, TtsError> {
+        if speech_tokens.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let tokens_arr = Array2::from_shape_vec((1, speech_tokens.len()), speech_tokens.to_vec())
+            .map_err(|e| TtsError::Inference(e.to_string()))?;
+
+        let mut inputs: Vec<(Cow<str>, DynValue)> = Vec::new();
+        inputs.push((Cow::from("speech_tokens"), Value::from_array(tokens_arr)?.into_dyn()));
+        inputs.push((Cow::from("speaker_embeddings"), Value::from_array(speaker_embeddings.clone())?.into_dyn()));
+        inputs.push((Cow::from("speaker_features"), Value::from_array(speaker_features.clone())?.into_dyn()));
+
+        let outputs = self.conditional_decoder.run(inputs)?;
+
+        let waveform = extract_f32_tensor(&outputs[0])?;
+
+        Ok(waveform.iter().copied().collect())
+    }
+}
+
+fn download_models(target_dir: &Path) -> Result<(), TtsError> {
+    fs::create_dir_all(target_dir)?;
+
+    let api = Api::new().map_err(|e| TtsError::ModelLoad(e.to_string()))?;
+    let repo = api.model(MODEL_ID.to_string());
+
+    let model_files = [
+        "onnx/speech_encoder.onnx",
+        "onnx/speech_encoder.onnx_data",
+        "onnx/embed_tokens.onnx",
+        "onnx/embed_tokens.onnx_data",
+        "onnx/language_model.onnx",
+        "onnx/language_model.onnx_data",
+        "onnx/conditional_decoder.onnx",
+        "onnx/conditional_decoder.onnx_data",
+        "tokenizer.json",
+    ];
+
+    for file in &model_files {
+        println!("Downloading {}...", file);
+        let downloaded_path = repo.get(file).map_err(|e| TtsError::ModelLoad(e.to_string()))?;
+
+        let filename = Path::new(file).file_name().unwrap();
+        let target_path = target_dir.join(filename);
+
+        if !target_path.exists() {
+            fs::copy(&downloaded_path, &target_path)?;
+        }
+    }
+
+    println!("Models downloaded to {:?}", target_dir);
+    Ok(())
+}
+
+fn resample_to_24k(samples: &[f32], input_rate: u32) -> Vec<f32> {
+    if input_rate == SAMPLE_RATE {
+        return samples.to_vec();
+    }
+    if samples.is_empty() {
+        return Vec::new();
+    }
+
+    let ratio = input_rate as f64 / SAMPLE_RATE as f64;
+    let output_len = ((samples.len() as f64) / ratio).ceil() as usize;
+
+    let mut output = Vec::with_capacity(output_len);
+    for i in 0..output_len {
+        let src_idx = (i as f64 * ratio) as usize;
+        let sample = samples.get(src_idx).copied().unwrap_or(0.0);
+        output.push(sample);
+    }
+
+    output
+}
+
+fn apply_repetition_penalty(logits: &mut [f32], generated: &[i64], penalty: f32) {
+    for &token in generated {
+        if (token as usize) < logits.len() {
+            let score = logits[token as usize];
+            // Note: opposite of standard - if score < 0, multiply; if > 0, divide
+            logits[token as usize] = if score < 0.0 {
+                score * penalty
+            } else {
+                score / penalty
+            };
+        }
+    }
+}
+
+fn argmax(logits: &[f32]) -> i64 {
+    logits
+        .iter()
+        .enumerate()
+        .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
+        .map(|(idx, _)| idx as i64)
+        .unwrap_or(0)
+}
+
+pub fn save_wav(samples: &[f32], path: &Path) -> Result<(), TtsError> {
+    let mut file = fs::File::create(path)?;
+    write_wav(&mut file, samples, SAMPLE_RATE)?;
+    Ok(())
+}
+
+fn write_wav<W: std::io::Write>(writer: &mut W, samples: &[f32], sample_rate: u32) -> Result<(), std::io::Error> {
+    let num_samples = samples.len() as u32;
+    let num_channels: u16 = 1;
+    let bits_per_sample: u16 = 16;
+    let byte_rate = sample_rate * num_channels as u32 * bits_per_sample as u32 / 8;
+    let block_align = num_channels * bits_per_sample / 8;
+    let data_size = num_samples * num_channels as u32 * bits_per_sample as u32 / 8;
+    let file_size = 36 + data_size;
+
+    writer.write_all(b"RIFF")?;
+    writer.write_all(&file_size.to_le_bytes())?;
+    writer.write_all(b"WAVE")?;
+
+    writer.write_all(b"fmt ")?;
+    writer.write_all(&16u32.to_le_bytes())?;
+    writer.write_all(&1u16.to_le_bytes())?;
+    writer.write_all(&num_channels.to_le_bytes())?;
+    writer.write_all(&sample_rate.to_le_bytes())?;
+    writer.write_all(&byte_rate.to_le_bytes())?;
+    writer.write_all(&block_align.to_le_bytes())?;
+    writer.write_all(&bits_per_sample.to_le_bytes())?;
+
+    writer.write_all(b"data")?;
+    writer.write_all(&data_size.to_le_bytes())?;
+
+    for &sample in samples {
+        let clamped = sample.clamp(-1.0, 1.0);
+        let int_sample = (clamped * 32767.0) as i16;
+        writer.write_all(&int_sample.to_le_bytes())?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_argmax() {
+        let logits = vec![0.1, 0.5, 0.3, 0.8, 0.2];
+        assert_eq!(argmax(&logits), 3);
+    }
+
+    #[test]
+    fn test_resample_same_rate() {
+        let samples = vec![0.1, 0.2, 0.3];
+        let resampled = resample_to_24k(&samples, SAMPLE_RATE);
+        assert_eq!(resampled, samples);
+    }
+
+    #[test]
+    fn test_repetition_penalty() {
+        let mut logits = vec![1.0, 2.0, 3.0, 4.0];
+        let generated = vec![1, 3];
+        apply_repetition_penalty(&mut logits, &generated, 1.2);
+        // score > 0 -> divide
+        assert!((logits[1] - 2.0 / 1.2).abs() < 1e-6);
+        assert!((logits[3] - 4.0 / 1.2).abs() < 1e-6);
+    }
+}