first
This commit is contained in:
valenti
@@ -0,0 +1,273 @@
|
||||
//! Fish Audio / SGLang Drama 3 TTS plugin.
|
||||
//!
|
||||
//! POSTs text to `/v1/audio/speech` (OpenAI-compatible) requesting PCM
|
||||
//! streaming. Fish Audio Drama 3 outputs at 44.1 kHz, which matches the
|
||||
//! mixer's output rate, so samples are pushed directly into the TTS queue
|
||||
//! without resampling. A [`CancellationToken`] allows immediate interruption
|
||||
//! (barge-in).
|
||||
|
||||
use crate::audio::SampleRingBuffer;
|
||||
use crate::config::Config;
|
||||
use crate::tts::TtsEngine;
|
||||
use anyhow::{Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use futures_util::StreamExt;
|
||||
use reqwest::Client;
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
/// A voice-cloning reference: path to a WAV file + its transcript.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct Reference {
|
||||
audio_path: String,
|
||||
text: String,
|
||||
}
|
||||
|
||||
/// Request body for the SGLang `/v1/audio/speech` endpoint.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SpeechRequest<'a> {
|
||||
model: &'a str,
|
||||
input: &'a str,
|
||||
voice: &'a str,
|
||||
response_format: &'a str,
|
||||
stream: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
references: Option<Vec<Reference>>,
|
||||
}
|
||||
|
||||
/// Streaming TTS client backed by Fish Audio / SGLang-Omni Drama 3.
|
||||
#[derive(Clone)]
|
||||
pub struct FishAudioTts {
|
||||
http: Client,
|
||||
endpoint: String,
|
||||
model: String,
|
||||
voice: String,
|
||||
/// Current voice-cloning reference (config default, overridable at runtime).
|
||||
reference: Arc<parking_lot::RwLock<Option<Reference>>>,
|
||||
}
|
||||
|
||||
impl FishAudioTts {
|
||||
pub fn new(config: &Config) -> Self {
|
||||
let reference = config.tts_reference().map(|(audio_path, text)| Reference {
|
||||
audio_path,
|
||||
text,
|
||||
});
|
||||
if let Some(r) = &reference {
|
||||
info!(
|
||||
audio_path = %r.audio_path,
|
||||
text_len = r.text.len(),
|
||||
"TTS voice-cloning reference configured"
|
||||
);
|
||||
}
|
||||
Self {
|
||||
http: Client::builder()
|
||||
.build()
|
||||
.expect("failed to build reqwest client"),
|
||||
endpoint: config.speech_endpoint(),
|
||||
model: config.tts_model.clone(),
|
||||
voice: config.tts_voice.clone(),
|
||||
reference: Arc::new(parking_lot::RwLock::new(reference)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Override the voice-cloning reference at runtime. Pass `None` to clear.
|
||||
pub fn set_reference(&self, audio_path: Option<String>, text: Option<String>) {
|
||||
let mut guard = self.reference.write();
|
||||
match (audio_path, text) {
|
||||
(Some(audio_path), text) => {
|
||||
let text = text.unwrap_or_default();
|
||||
info!(audio_path = %audio_path, text_len = text.len(), "TTS reference updated");
|
||||
*guard = Some(Reference { audio_path, text });
|
||||
}
|
||||
_ => {
|
||||
info!("TTS reference cleared");
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TtsEngine for FishAudioTts {
|
||||
fn name(&self) -> &str {
|
||||
"fish-audio"
|
||||
}
|
||||
|
||||
fn sample_rate(&self) -> u32 {
|
||||
// Fish Audio Drama 3 outputs natively at 44.1 kHz, matching the mixer.
|
||||
44_100
|
||||
}
|
||||
|
||||
async fn synthesize(
|
||||
&self,
|
||||
text: &str,
|
||||
tts_queue: &SampleRingBuffer,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<()> {
|
||||
let references = self.reference.read().clone().map(|r| vec![r]);
|
||||
|
||||
let body = SpeechRequest {
|
||||
model: &self.model,
|
||||
input: text,
|
||||
voice: &self.voice,
|
||||
response_format: "pcm",
|
||||
stream: true,
|
||||
references,
|
||||
};
|
||||
|
||||
info!(model = %self.model, text_len = text.len(), "SGLang TTS request");
|
||||
debug!(
|
||||
endpoint = %self.endpoint,
|
||||
model = %self.model,
|
||||
voice = %self.voice,
|
||||
response_format = "pcm",
|
||||
stream = true,
|
||||
has_reference = body.references.is_some(),
|
||||
text = %text,
|
||||
"TTS request dispatched"
|
||||
);
|
||||
|
||||
let req = self
|
||||
.http
|
||||
.post(&self.endpoint)
|
||||
.json(&body)
|
||||
.build()
|
||||
.context("failed to build SGLang request")?;
|
||||
|
||||
let response = self
|
||||
.http
|
||||
.execute(req)
|
||||
.await
|
||||
.context("SGLang request failed")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let err_text = response.text().await.unwrap_or_default();
|
||||
error!(%status, body = %err_text, "SGLang returned an error");
|
||||
anyhow::bail!("SGLang error: {} {}", status, err_text);
|
||||
}
|
||||
|
||||
// Read the actual sample rate from the response header for logging.
|
||||
let tts_sr = response
|
||||
.headers()
|
||||
.get("x-sample-rate")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.parse::<u32>().ok())
|
||||
.unwrap_or(44100);
|
||||
info!(tts_sample_rate = tts_sr, "TTS response sample rate");
|
||||
|
||||
let mut byte_stream = response.bytes_stream();
|
||||
let mut carry: Option<u8> = None;
|
||||
let mut chunk_count = 0usize;
|
||||
let mut total_samples = 0usize;
|
||||
let t0 = std::time::Instant::now();
|
||||
let mut last_log = t0;
|
||||
|
||||
while let Some(chunk) = tokio::select! {
|
||||
c = byte_stream.next() => c,
|
||||
_ = cancel.cancelled() => {
|
||||
info!("TTS stream cancelled (barge-in)");
|
||||
return Ok(());
|
||||
}
|
||||
} {
|
||||
let chunk = match chunk {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
error!(error = %e, "error reading TTS stream chunk");
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
let n = ingest_pcm_chunk(&chunk, &mut carry, tts_queue);
|
||||
chunk_count += 1;
|
||||
total_samples += n;
|
||||
if chunk_count == 1 {
|
||||
debug!(
|
||||
ttfb_ms = t0.elapsed().as_millis(),
|
||||
bytes = chunk.len(),
|
||||
"TTS time-to-first-byte"
|
||||
);
|
||||
}
|
||||
debug!(
|
||||
chunk = chunk_count,
|
||||
bytes = chunk.len(),
|
||||
samples = n,
|
||||
queue_len = tts_queue.len(),
|
||||
elapsed_ms = t0.elapsed().as_millis(),
|
||||
"TTS chunk received"
|
||||
);
|
||||
if last_log.elapsed() > std::time::Duration::from_millis(500) {
|
||||
last_log = std::time::Instant::now();
|
||||
info!(
|
||||
chunk = chunk_count,
|
||||
total_samples,
|
||||
queue_len = tts_queue.len(),
|
||||
elapsed_ms = t0.elapsed().as_millis(),
|
||||
"TTS stream progress"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
chunk_count,
|
||||
total_samples,
|
||||
duration_s = format!("{:.2}", t0.elapsed().as_secs_f32()),
|
||||
"TTS stream complete"
|
||||
);
|
||||
debug!(
|
||||
chunk_count,
|
||||
total_samples,
|
||||
avg_chunk_bytes = if chunk_count > 0 { (total_samples * 2) / chunk_count } else { 0 },
|
||||
audio_s = format!("{:.2}", total_samples as f32 / tts_sr as f32),
|
||||
elapsed_ms = t0.elapsed().as_millis(),
|
||||
queue_len = tts_queue.len(),
|
||||
"TTS stream summary"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_reference(&self, audio_path: Option<String>, text: Option<String>) {
|
||||
FishAudioTts::set_reference(self, audio_path, text);
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse 16-bit LE PCM samples from a chunk and push them directly to the
|
||||
/// queue (Fish Audio's 44.1 kHz output matches the mixer rate).
|
||||
/// Returns the number of samples pushed.
|
||||
fn ingest_pcm_chunk(
|
||||
chunk: &[u8],
|
||||
carry: &mut Option<u8>,
|
||||
tts_queue: &SampleRingBuffer,
|
||||
) -> usize {
|
||||
let mut samples: Vec<i16> = Vec::with_capacity(chunk.len() / 2 + 1);
|
||||
let mut iter = chunk.iter().copied();
|
||||
|
||||
if let Some(lo) = carry.take() {
|
||||
if let Some(hi) = iter.next() {
|
||||
samples.push(i16::from_le_bytes([lo, hi]));
|
||||
} else {
|
||||
*carry = Some(lo);
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
let lo = match iter.next() {
|
||||
Some(b) => b,
|
||||
None => break,
|
||||
};
|
||||
let hi = match iter.next() {
|
||||
Some(b) => b,
|
||||
None => {
|
||||
*carry = Some(lo);
|
||||
break;
|
||||
}
|
||||
};
|
||||
samples.push(i16::from_le_bytes([lo, hi]));
|
||||
}
|
||||
|
||||
if !samples.is_empty() {
|
||||
tts_queue.push(&samples);
|
||||
}
|
||||
samples.len()
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! Kokoro TTS plugin (stub).
|
||||
//!
|
||||
//! Kokoro is a lightweight, open-source TTS model. This stub shows the pattern for
|
||||
//! adding a new engine plugin: implement [`TtsEngine`] and wire the HTTP or
|
||||
//! local inference calls.
|
||||
//!
|
||||
//! # Configuration
|
||||
//! Set `TTS_ENGINE=kokoro` and provide:
|
||||
//! - `KOKORO_URL` — base URL of the Kokoro HTTP server (default: `http://localhost:8880`)
|
||||
//! - `KOKORO_VOICE` — voice name (default: `af_heart`)
|
||||
//!
|
||||
//! # Resampling note
|
||||
//! Kokoro typically outputs at 24 kHz. Since the mixer runs at
|
||||
//! [`crate::config::OUTPUT_SAMPLE_RATE`] (44.1 kHz), this plugin must
|
||||
//! resample internally before pushing samples. A basic linear resampler is
|
||||
//! provided below; replace with a polyphase/FIR resampler for higher quality.
|
||||
|
||||
use crate::audio::SampleRingBuffer;
|
||||
use crate::config::{Config, OUTPUT_SAMPLE_RATE};
|
||||
use crate::tts::TtsEngine;
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
|
||||
/// Kokoro native output sample rate (Hz).
|
||||
const KOKORO_SAMPLE_RATE: u32 = 24_000;
|
||||
|
||||
/// Kokoro TTS plugin.
|
||||
#[derive(Clone)]
|
||||
pub struct KokoroTts {
|
||||
endpoint: String,
|
||||
voice: String,
|
||||
http: reqwest::Client,
|
||||
_config: Arc<Config>,
|
||||
}
|
||||
|
||||
impl KokoroTts {
|
||||
pub fn new(config: &Config) -> Self {
|
||||
let endpoint = std::env::var("KOKORO_URL")
|
||||
.unwrap_or_else(|_| "http://localhost:8880".to_string());
|
||||
let voice = std::env::var("KOKORO_VOICE")
|
||||
.unwrap_or_else(|_| "af_heart".to_string());
|
||||
info!(endpoint = %endpoint, voice = %voice, "Kokoro TTS plugin initialized");
|
||||
Self {
|
||||
endpoint,
|
||||
voice,
|
||||
http: reqwest::Client::builder()
|
||||
.build()
|
||||
.expect("failed to build reqwest client"),
|
||||
_config: Arc::new(config.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TtsEngine for KokoroTts {
|
||||
fn name(&self) -> &str {
|
||||
"kokoro"
|
||||
}
|
||||
|
||||
fn sample_rate(&self) -> u32 {
|
||||
// We resample to the mixer rate before pushing.
|
||||
OUTPUT_SAMPLE_RATE
|
||||
}
|
||||
|
||||
async fn synthesize(
|
||||
&self,
|
||||
text: &str,
|
||||
tts_queue: &SampleRingBuffer,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<()> {
|
||||
// TODO: implement the actual Kokoro HTTP streaming call.
|
||||
//
|
||||
// Example request (adjust to be completed by the user):
|
||||
//
|
||||
// ```ignore
|
||||
// let resp = self.http
|
||||
// .post(format!("{}/v1/audio/speech", self.endpoint))
|
||||
// .json(&serde_json::json!({
|
||||
// "model": "kokoro",
|
||||
// "input": text,
|
||||
// "voice": self.voice,
|
||||
// "response_format": "pcm",
|
||||
// "stream": true,
|
||||
// }))
|
||||
// .send()
|
||||
// .await?;
|
||||
//
|
||||
// let mut stream = resp.bytes_stream();
|
||||
// while let Some(chunk) = stream.next().await {
|
||||
// if cancel.is_cancelled() { break; }
|
||||
// let bytes = chunk?;
|
||||
// // Kokoro outputs 16-bit mono PCM at 24 kHz.
|
||||
// let samples = pcm_to_i16(&bytes);
|
||||
// let resampled = resample_24k_to_output(&samples);
|
||||
// tts_queue.push(&resampled);
|
||||
// }
|
||||
// ```
|
||||
let _ = (text, tts_queue, cancel);
|
||||
anyhow::bail!(
|
||||
"Kokoro TTS plugin is not yet implemented — \
|
||||
complete the HTTP streaming call in src/tts/kokoro.rs"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Resample a 24 kHz mono `i16` buffer to [`OUTPUT_SAMPLE_RATE` using linear
|
||||
/// interpolation. Replace with a windowed-sinc FIR resampler for higher fidelity.
|
||||
#[allow(dead_code)]
|
||||
fn resample_24k_to_output(samples: &[i16]) -> Vec<i16> {
|
||||
if samples.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let ratio = OUTPUT_SAMPLE_RATE as f64 / KOKORO_SAMPLE_RATE as f64;
|
||||
let out_len = (samples.len() as f64 * ratio).round() as usize;
|
||||
let mut out = Vec::with_capacity(out_len);
|
||||
for i in 0..out_len {
|
||||
let src_pos = i as f64 / ratio;
|
||||
let idx = src_pos.floor() as usize;
|
||||
let frac = src_pos - idx as f64;
|
||||
let a = samples[idx.min(samples.len() - 1)] as f32;
|
||||
let b = samples[(idx + 1).min(samples.len() - 1)] as f32;
|
||||
out.push((a + (b - a) * frac as f32).round() as i16);
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//! Pluggable Text-to-Speech engine subsystem.
|
||||
//!
|
||||
//! The engine is selected at runtime via the `TTS_ENGINE` environment variable
|
||||
//! (see [`Config::tts_engine`]). Every plugin implements the [`TtsEngine`]
|
||||
//! trait so the orchestration layer is engine-agnostic.
|
||||
//!
|
||||
//! Currently bundled plugins:
|
||||
//! - [`fish_audio::FishAudioTts`] — Fish Audio / SGLang Drama 3 (OpenAI-compatible HTTP).
|
||||
//! - [`kokoro::KokoroTts`] — Kokoro TTS plugin (stub; implement the HTTP/API calls).
|
||||
//!
|
||||
//! To add a new engine, create a module under `src/tts/`, implement
|
||||
//! [`TtsEngine`], and register it in [`create_tts_engine`].
|
||||
|
||||
use crate::audio::SampleRingBuffer;
|
||||
use crate::config::Config;
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
|
||||
pub mod fish_audio;
|
||||
pub mod kokoro;
|
||||
|
||||
use fish_audio::FishAudioTts;
|
||||
use kokoro::KokoroTts;
|
||||
|
||||
/// Trait implemented by every TTS engine plugin.
|
||||
///
|
||||
/// Implementations must stream **16-bit mono PCM samples at
|
||||
/// [`crate::config::OUTPUT_SAMPLE_RATE`]** into the provided `tts_queue`.
|
||||
/// If the engine's native sample rate differs, it must resample internally
|
||||
/// before pushing samples (the mixer runs at a fixed rate).
|
||||
#[async_trait]
|
||||
pub trait TtsEngine: Send + Sync {
|
||||
/// Human-readable engine name (used for logging).
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// The sample rate (Hz) at which this engine pushes samples into the
|
||||
/// mixer queue. Must equal [`crate::config::OUTPUT_SAMPLE_RATE`].
|
||||
fn sample_rate(&self) -> u32;
|
||||
|
||||
/// Synthesize `text` and stream PCM samples into `tts_queue`.
|
||||
///
|
||||
/// The implementation **must** honor `cancel` for barge-in: when the token
|
||||
/// is cancelled, stop pushing samples and return `Ok(())` promptly.
|
||||
async fn synthesize(
|
||||
&self,
|
||||
text: &str,
|
||||
tts_queue: &SampleRingBuffer,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<()>;
|
||||
|
||||
/// Update the voice-cloning reference at runtime. Default: no-op (engines
|
||||
/// that support voice cloning override this).
|
||||
fn set_reference(&self, _audio_path: Option<String>, _text: Option<String>) {}
|
||||
}
|
||||
|
||||
/// Build the TTS engine selected by the `TTS_ENGINE` env var.
|
||||
///
|
||||
/// Supported values (case-insensitive): `fish`, `fishaudio`, `fish_audio`,
|
||||
/// `sglang` → Fish Audio Drama 3; `kokoro` → Kokoro TTS.
|
||||
///
|
||||
/// # Panics
|
||||
/// Panics if `TTS_ENGINE` is set to an unknown value.
|
||||
pub fn create_tts_engine(config: &Config) -> Arc<dyn TtsEngine> {
|
||||
match config.tts_engine.to_lowercase().as_str() {
|
||||
"fish" | "fishaudio" | "fish_audio" | "sglang" => {
|
||||
info!(engine = "fish", "selected TTS engine");
|
||||
Arc::new(FishAudioTts::new(config))
|
||||
}
|
||||
"kokoro" => {
|
||||
info!(engine = "kokoro", "selected TTS engine");
|
||||
Arc::new(KokoroTts::new(config))
|
||||
}
|
||||
other => panic!(
|
||||
"unknown TTS_ENGINE='{}' — supported values: fish, kokoro",
|
||||
other
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user