step3
This commit is contained in:
valenti
+63
-7
@@ -34,6 +34,12 @@ use tracing::{debug, info};
|
||||
/// re-introduces the pre-buffer latency mid-reply.
|
||||
const TTS_DRAIN_GRACE_FRAMES: usize = 10;
|
||||
|
||||
/// Number of 20 ms frames over which the latency-mask filler (end-of-turn
|
||||
/// filler and start filler) is faded out once the reply's playback starts,
|
||||
/// before being cut. The filler is only meant to mask latency, so it must hand
|
||||
/// off cleanly to the answer instead of running underneath it. 4 × 20 ms = 80 ms.
|
||||
const FILLER_FADEOUT_FRAMES: u32 = 4;
|
||||
|
||||
/// Shared audio mixer holding the TTS, filler, start-filler and backchannel
|
||||
/// sample queues.
|
||||
#[derive(Clone)]
|
||||
@@ -91,6 +97,11 @@ pub struct AudioMixer {
|
||||
/// `mark_stream_start` and `flush`. Used to decide whether a turn-start
|
||||
/// filler is still useful.
|
||||
tts_first_chunk: Arc<AtomicBool>,
|
||||
/// Remaining frames of the filler fade-out envelope. `0` means no fade is in
|
||||
/// progress. Armed to [`FILLER_FADEOUT_FRAMES`] when the reply's playback
|
||||
/// starts while a filler is still queued, so the latency-mask cue is faded
|
||||
/// out and cut instead of playing underneath the answer.
|
||||
filler_fade_frames: Arc<AtomicU32>,
|
||||
}
|
||||
|
||||
impl AudioMixer {
|
||||
@@ -121,6 +132,7 @@ impl AudioMixer {
|
||||
tts_stream_active: Arc::new(AtomicBool::new(false)),
|
||||
tts_playback_started: Arc::new(AtomicBool::new(false)),
|
||||
tts_first_chunk: Arc::new(AtomicBool::new(false)),
|
||||
filler_fade_frames: Arc::new(AtomicU32::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,11 +145,13 @@ impl AudioMixer {
|
||||
/// Mixing formula:
|
||||
/// `mixed = tts + (filler + start_filler) * filler_gain
|
||||
/// + backchannel * backchannel_gain + external * external_gain`.
|
||||
/// The filler gain is dynamically attenuated to `filler_gain_duck` while
|
||||
/// TTS is being played out, otherwise it stays at `filler_gain_idle`; the
|
||||
/// backchannel gain is fixed and low so the cue stays in the background.
|
||||
/// The start-filler channel shares the filler gain, so the immediate
|
||||
/// acknowledgment ducks out of the way once the reply starts. The external
|
||||
/// The filler gain is `filler_gain_idle` while TTS is silent and ducks to
|
||||
/// `filler_gain_duck` while TTS is being played out; the backchannel gain is
|
||||
/// fixed and low so the cue stays in the background. The start-filler
|
||||
/// channel shares the filler gain. When the reply's playback starts, a
|
||||
/// still-queued filler/start-filler is instead faded out over
|
||||
/// `FILLER_FADEOUT_FRAMES` and then cut, so the latency-mask cue hands off
|
||||
/// cleanly to the answer instead of running underneath it. The external
|
||||
/// channel is a user-injected background source with its own (runtime
|
||||
/// adjustable) gain.
|
||||
pub fn pull_frame(&self) -> Option<Vec<u8>> {
|
||||
@@ -176,6 +190,13 @@ impl AudioMixer {
|
||||
self.tts_primed.store(true, Ordering::Relaxed);
|
||||
self.tts_drain_frames.store(0, Ordering::Relaxed);
|
||||
self.tts_playback_started.store(true, Ordering::Relaxed);
|
||||
// The reply is about to play: arm the filler fade-out so any
|
||||
// latency-mask cue still queued hands off cleanly instead of
|
||||
// running underneath the answer.
|
||||
if self.filler_queue.len() > 0 || self.start_filler_queue.len() > 0 {
|
||||
self.filler_fade_frames
|
||||
.store(FILLER_FADEOUT_FRAMES, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
let tts_active = self.tts_primed.load(Ordering::Relaxed);
|
||||
@@ -201,8 +222,15 @@ impl AudioMixer {
|
||||
);
|
||||
}
|
||||
|
||||
// Determine filler gain based on whether TTS is actively playing.
|
||||
let filler_gain = if tts_active {
|
||||
// Determine filler gain. Normally the filler ducks to `filler_gain_duck`
|
||||
// while TTS plays; but once the reply's playback starts a still-queued
|
||||
// latency-mask filler is faded out (idle gain → 0) over
|
||||
// `FILLER_FADEOUT_FRAMES` and then cut, so it never runs underneath the
|
||||
// answer.
|
||||
let fade_remaining = self.filler_fade_frames.load(Ordering::Relaxed);
|
||||
let filler_gain = if fade_remaining > 0 {
|
||||
self.filler_gain_idle * (fade_remaining as f32 / FILLER_FADEOUT_FRAMES as f32)
|
||||
} else if tts_active {
|
||||
self.filler_gain_duck
|
||||
} else {
|
||||
self.filler_gain_idle
|
||||
@@ -227,6 +255,18 @@ impl AudioMixer {
|
||||
let backchannel_n = self.backchannel_queue.pop_into(&mut backchannel_buf);
|
||||
let external_n = self.external_queue.pop_into(&mut external_buf);
|
||||
|
||||
// Filler fade-out bookkeeping, done after the pops so this frame's
|
||||
// samples are already captured: decay one step per frame and drop the
|
||||
// remaining queued filler once the envelope reaches zero.
|
||||
if fade_remaining == 1 {
|
||||
self.filler_queue.clear();
|
||||
self.start_filler_queue.clear();
|
||||
self.filler_fade_frames.store(0, Ordering::Relaxed);
|
||||
} else if fade_remaining > 0 {
|
||||
self.filler_fade_frames
|
||||
.store(fade_remaining - 1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
// If no channel produced anything, skip sending a frame.
|
||||
if tts_n == 0
|
||||
&& filler_n == 0
|
||||
@@ -272,6 +312,7 @@ impl AudioMixer {
|
||||
self.tts_first_chunk.store(false, Ordering::Relaxed);
|
||||
self.tts_primed.store(false, Ordering::Relaxed);
|
||||
self.tts_drain_frames.store(0, Ordering::Relaxed);
|
||||
self.filler_fade_frames.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Signal that the TTS stream has finished producing audio. The mixer will
|
||||
@@ -294,6 +335,20 @@ impl AudioMixer {
|
||||
|| self.start_filler_queue.len() > 0
|
||||
}
|
||||
|
||||
/// Whether the agent has *audible* output: the reply is being played out, or
|
||||
/// a latency-mask cue (filler / start filler) is queued and about to play.
|
||||
///
|
||||
/// Unlike [`Self::is_active`] this is `false` while the reply is still being
|
||||
/// synthesised or is sitting in the pre-buffer (only the TTS stream is in
|
||||
/// flight) — nothing has reached the user's ears yet. Used to gate
|
||||
/// VAD-driven barge-in so a transient or the tail of the user's own utterance
|
||||
/// cannot cancel a reply before the user has heard any of it.
|
||||
pub fn is_audible(&self) -> bool {
|
||||
self.tts_primed.load(Ordering::Relaxed)
|
||||
|| self.filler_queue.len() > 0
|
||||
|| self.start_filler_queue.len() > 0
|
||||
}
|
||||
|
||||
/// Whether the response audio has started playing since the last
|
||||
/// `mark_stream_start`. `false` means the agent is still silent (TTS
|
||||
/// request in flight, pre-buffer not reached yet), i.e. the latency filler
|
||||
@@ -328,6 +383,7 @@ impl AudioMixer {
|
||||
self.tts_stream_active.store(false, Ordering::Relaxed);
|
||||
self.tts_playback_started.store(false, Ordering::Relaxed);
|
||||
self.tts_first_chunk.store(false, Ordering::Relaxed);
|
||||
self.filler_fade_frames.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Set the gain applied to the external audio channel. Used by the client's
|
||||
|
||||
+36
-2
@@ -84,6 +84,28 @@ pub struct Config {
|
||||
/// recitative register; clip banks that carry their own tag are left
|
||||
/// untouched.
|
||||
pub tts_style_tag: String,
|
||||
/// Post-process a freshly recorded custom voice before it is installed
|
||||
/// (`CUSTOMVOICE_POSTPROCESS`, default `true`): rumble removal, silence
|
||||
/// trimming and EBU R128 loudness normalization. Disabling it stores the
|
||||
/// plain transcode (see [`crate::customvoice`]).
|
||||
pub customvoice_postprocess: bool,
|
||||
/// Enable the noise-reduction / de-essing stage of the custom-voice
|
||||
/// post-processing (`CUSTOMVOICE_DENOISE`, default `true`). Turn it off for
|
||||
/// an already clean studio recording.
|
||||
pub customvoice_denoise: bool,
|
||||
/// Integrated loudness target in LUFS for the custom-voice normalization
|
||||
/// (`CUSTOMVOICE_TARGET_LUFS`, default `-26`). Every take lands on the same
|
||||
/// programme level, so recordings do not jump in volume. Set **below** the
|
||||
/// shipped reference (`voice_profiles/solis.wav` measures ≈-23.4 LUFS):
|
||||
/// a user take is far more dynamic than the mastered reference, so at equal
|
||||
/// LUFS its much hotter peaks make the cloned voice sound louder than the
|
||||
/// default one. The gap buys that headroom back.
|
||||
pub customvoice_target_lufs: f32,
|
||||
/// True-peak ceiling in dBTP for the custom-voice normalization
|
||||
/// (`CUSTOMVOICE_TRUE_PEAK`, default `-5`) — headroom against clipping.
|
||||
/// Kept well below 0 on purpose: a dynamic user take would otherwise peak
|
||||
/// several dB hotter than the mastered reference even at the same LUFS.
|
||||
pub customvoice_true_peak: f32,
|
||||
/// Filesystem directory containing the Parakeet TDT ONNX pipeline
|
||||
/// (`nemo128.onnx`, `encoder-model*.onnx`, `decoder_joint-model.onnx`,
|
||||
/// `vocab.txt`, `config.json`).
|
||||
@@ -148,7 +170,7 @@ pub struct Config {
|
||||
/// [`Config::start_filler_go_delay_ms`]).
|
||||
pub start_filler_enabled: bool,
|
||||
/// How long (ms) the agent waits for the response audio to start before
|
||||
/// playing the start filler (`START_FILLER_GO_DELAY`, default `1200`). Same
|
||||
/// playing the start filler (`START_FILLER_GO_DELAY`, default `1800`). Same
|
||||
/// idea as [`Config::filler_delay_ms`] — a late reply is masked with a
|
||||
/// short acknowledgment, an on-time one is not — but on the dedicated
|
||||
/// start-filler channel and with its own (shorter) delay. If playback has
|
||||
@@ -290,6 +312,10 @@ impl Default for Config {
|
||||
tts_top_p: Some(0.75),
|
||||
tts_top_k: None,
|
||||
tts_style_tag: "[conversational, casual]".to_string(),
|
||||
customvoice_postprocess: true,
|
||||
customvoice_denoise: true,
|
||||
customvoice_target_lufs: -26.0,
|
||||
customvoice_true_peak: -5.0,
|
||||
stt_model_path: PathBuf::from("models/parakeet"),
|
||||
stt_tokenizer_path: None,
|
||||
fillers_dir: PathBuf::from("assets/fillers"),
|
||||
@@ -304,7 +330,7 @@ impl Default for Config {
|
||||
media_ytdlp_bin: "yt-dlp".to_string(),
|
||||
media_ytdlp_args: Vec::new(),
|
||||
start_filler_enabled: true,
|
||||
start_filler_go_delay_ms: 1200,
|
||||
start_filler_go_delay_ms: 1800,
|
||||
start_filler_dir: PathBuf::from("assets/start_filler"),
|
||||
start_filler_texts: vec!["[calm] Un momento...".to_string()],
|
||||
tls_cert: None,
|
||||
@@ -383,6 +409,14 @@ impl Config {
|
||||
if let Ok(v) = std::env::var("TTS_STYLE_TAG") {
|
||||
cfg.tts_style_tag = v.trim().to_string();
|
||||
}
|
||||
// Custom-voice post-processing (see `crate::customvoice`).
|
||||
cfg.customvoice_postprocess =
|
||||
env_bool("CUSTOMVOICE_POSTPROCESS", cfg.customvoice_postprocess);
|
||||
cfg.customvoice_denoise = env_bool("CUSTOMVOICE_DENOISE", cfg.customvoice_denoise);
|
||||
cfg.customvoice_target_lufs =
|
||||
env_f32("CUSTOMVOICE_TARGET_LUFS", cfg.customvoice_target_lufs);
|
||||
cfg.customvoice_true_peak =
|
||||
env_f32("CUSTOMVOICE_TRUE_PEAK", cfg.customvoice_true_peak);
|
||||
if let Ok(p) = std::env::var("STT_MODEL_PATH") {
|
||||
cfg.stt_model_path = PathBuf::from(p);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
//! Custom-voice builder: turns a recording (or an uploaded audio file) into a
|
||||
//! Fish Audio voice-cloning reference.
|
||||
//!
|
||||
//! The `/customvoice` page records the user reading a phrase (at least 10 s
|
||||
//! long) **or** lets them upload an audio file and type the matching
|
||||
//! transcript. The bytes are POSTed here, transcoded to a mono PCM WAV in the
|
||||
//! directory that holds the configured reference — the same directory the Fish
|
||||
//! Audio backend already reads, so the new file is immediately usable — the
|
||||
//! transcript is written as a sibling `.txt` (the convention
|
||||
//! [`crate::config::Config::tts_reference`] falls back to), and the
|
||||
//! running TTS engine is pointed at the new file via
|
||||
//! [`crate::tts::TtsEngine::set_reference`].
|
||||
//!
|
||||
//! Before installation the take goes through a **post-processing chain**
|
||||
//! (see [`postprocess`]): high-pass to drop rumble/DC, optional FFT denoise and
|
||||
//! de-essing, leading/trailing silence trim, and a **two-pass EBU R128
|
||||
//! loudness normalization** (single linear gain, so takes never jump in
|
||||
//! volume). Any step failing falls back to the plain transcode, so a take is
|
||||
//! never lost to a filter quirk.
|
||||
//!
|
||||
//! The predefined phrases live server-side so a plain recording always carries
|
||||
//! a transcript matching the text the page asked the user to read; a
|
||||
//! caller-supplied transcript (the upload flow) overrides it.
|
||||
|
||||
use crate::config::{Config, OUTPUT_SAMPLE_RATE};
|
||||
use crate::ws::AppState;
|
||||
use axum::body::Bytes;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::Json;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
use std::ffi::OsString;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Output, Stdio};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::process::Command;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
/// A predefined reading phrase, keyed by ISO 639-1 code. Long enough
|
||||
/// (~10 s at a natural pace) to give the cloning model a usable sample.
|
||||
struct Phrase {
|
||||
code: &'static str,
|
||||
label: &'static str,
|
||||
phrase: &'static str,
|
||||
}
|
||||
|
||||
/// Reading phrases for the languages supported by [`crate::lang`].
|
||||
const PHRASES: &[Phrase] = &[
|
||||
Phrase {
|
||||
code: "es",
|
||||
label: "Español",
|
||||
phrase: "Hola, esta es mi voz personal. Estoy leyendo esta frase para crear una voz \
|
||||
personalizada para mí. Hablo de forma natural y clara, para que el sistema pueda \
|
||||
aprender el timbre y el ritmo de mi voz.",
|
||||
},
|
||||
Phrase {
|
||||
code: "en",
|
||||
label: "English",
|
||||
phrase: "Hello, this is my personal voice. I am reading this sentence to create a custom \
|
||||
voice for me. I speak in a natural and clear way, so the system can learn the \
|
||||
tone and the rhythm of my voice.",
|
||||
},
|
||||
Phrase {
|
||||
code: "it",
|
||||
label: "Italiano",
|
||||
phrase: "Ciao, questa è la mia voce personale. Sto leggendo questa frase per creare una \
|
||||
voce su misura per me. Parlo in modo naturale e chiaro, così il sistema può \
|
||||
imparare il timbro e il ritmo della mia voce.",
|
||||
},
|
||||
Phrase {
|
||||
code: "fr",
|
||||
label: "Français",
|
||||
phrase: "Bonjour, c'est ma voix personnelle. Je lis cette phrase pour créer une voix \
|
||||
personnalisée pour moi. Je parle de manière naturelle et claire, afin que le \
|
||||
système puisse apprendre le timbre et le rythme de ma voix.",
|
||||
},
|
||||
Phrase {
|
||||
code: "de",
|
||||
label: "Deutsch",
|
||||
phrase: "Hallo, das ist meine persönliche Stimme. Ich lese diesen Satz, um eine eigene \
|
||||
Stimme für mich zu erstellen. Ich spreche natürlich und deutlich, damit das \
|
||||
System die Klangfarbe und den Rhythmus meiner Stimme lernen kann.",
|
||||
},
|
||||
Phrase {
|
||||
code: "pt",
|
||||
label: "Português",
|
||||
phrase: "Olá, esta é a minha voz pessoal. Estou a ler esta frase para criar uma voz \
|
||||
personalizada para mim. Falo de forma natural e clara, para que o sistema possa \
|
||||
aprender o timbre e o ritmo da minha voz.",
|
||||
},
|
||||
];
|
||||
|
||||
/// `GET /api/customvoice/phrases` — the reading phrases and the configured
|
||||
/// default language, used to populate the page's language selector.
|
||||
pub async fn phrases(State(state): State<AppState>) -> Json<Value> {
|
||||
let languages: Vec<Value> = PHRASES
|
||||
.iter()
|
||||
.map(|p| json!({ "code": p.code, "label": p.label, "phrase": p.phrase }))
|
||||
.collect();
|
||||
Json(json!({
|
||||
"default": state.config.lang_default,
|
||||
"languages": languages,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Query string of `POST /api/customvoice`.
|
||||
#[derive(Deserialize)]
|
||||
pub struct SetParams {
|
||||
/// ISO 639-1 code of the language the user recorded in.
|
||||
pub lang: String,
|
||||
/// Optional transcript override: the text actually spoken in the audio.
|
||||
/// Used by the upload flow, where the user types (or pastes) it. When
|
||||
/// absent or blank, the language's predefined reading phrase is used.
|
||||
pub text: Option<String>,
|
||||
}
|
||||
|
||||
/// `POST /api/customvoice?lang=<code>[&text=<transcript>]` — body is the raw
|
||||
/// audio (a browser recording or an uploaded file; whatever container the
|
||||
/// client sent, ffmpeg probes the content). Transcodes it to a WAV reference,
|
||||
/// persists the transcript, and installs the voice for the running session.
|
||||
pub async fn set_custom_voice(
|
||||
State(state): State<AppState>,
|
||||
Query(params): Query<SetParams>,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
let lang = params.lang.trim().to_ascii_lowercase();
|
||||
let Some(entry) = PHRASES.iter().find(|p| p.code == lang) else {
|
||||
return error_response(StatusCode::BAD_REQUEST, "unsupported language");
|
||||
};
|
||||
if body.is_empty() {
|
||||
return error_response(StatusCode::BAD_REQUEST, "empty recording");
|
||||
}
|
||||
|
||||
// Transcript attached to the reference: the caller-supplied text when it is
|
||||
// non-blank (the upload flow lets the user type it), otherwise the
|
||||
// language's predefined reading phrase.
|
||||
let transcript = params
|
||||
.text
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|t| !t.is_empty())
|
||||
.unwrap_or(entry.phrase)
|
||||
.to_string();
|
||||
|
||||
// Save next to the configured reference: that directory is already readable
|
||||
// by the Fish Audio backend, so no extra path plumbing is needed.
|
||||
let dir = reference_dir(&state.config);
|
||||
if let Err(e) = tokio::fs::create_dir_all(&dir).await {
|
||||
error!(error = %e, dir = %dir.display(), "failed to create reference directory");
|
||||
return error_response(StatusCode::INTERNAL_SERVER_ERROR, "cannot create reference directory");
|
||||
}
|
||||
|
||||
let stamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_millis())
|
||||
.unwrap_or(0);
|
||||
let base = format!("custom_{}_{}", entry.code, stamp);
|
||||
let raw_path = dir.join(format!("{base}.upload"));
|
||||
let wav_path = dir.join(format!("{base}.wav"));
|
||||
let txt_path = dir.join(format!("{base}.txt"));
|
||||
|
||||
if let Err(e) = tokio::fs::write(&raw_path, &body).await {
|
||||
error!(error = %e, path = %raw_path.display(), "failed to store upload");
|
||||
return error_response(StatusCode::INTERNAL_SERVER_ERROR, "cannot store upload");
|
||||
}
|
||||
|
||||
match render_reference(&state.config, &raw_path, &wav_path).await {
|
||||
Ok(()) => {}
|
||||
Err(msg) => {
|
||||
let _ = tokio::fs::remove_file(&raw_path).await;
|
||||
error!(error = %msg, "custom-voice transcode failed");
|
||||
return error_response(StatusCode::BAD_GATEWAY, &msg);
|
||||
}
|
||||
}
|
||||
let _ = tokio::fs::remove_file(&raw_path).await;
|
||||
|
||||
if let Err(e) = tokio::fs::write(&txt_path, &transcript).await {
|
||||
// Non-fatal: the in-memory reference below still carries the transcript.
|
||||
warn!(error = %e, path = %txt_path.display(), "failed to persist reference transcript");
|
||||
}
|
||||
|
||||
let audio_path = wav_path.to_string_lossy().into_owned();
|
||||
state
|
||||
.tts
|
||||
.set_reference(Some(audio_path.clone()), Some(transcript.clone()));
|
||||
info!(lang = %entry.code, audio = %audio_path, "custom voice installed");
|
||||
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"ok": true,
|
||||
"lang": entry.code,
|
||||
"audio": audio_path,
|
||||
"phrase": transcript,
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// `POST /api/customvoice/clear` — drop the custom voice and restore the
|
||||
/// reference configured at startup (the `TTS_REFERENCE_AUDIO` voice), or no
|
||||
/// reference at all when none is configured.
|
||||
pub async fn clear_custom_voice(State(state): State<AppState>) -> Json<Value> {
|
||||
match state.config.tts_reference() {
|
||||
Some((audio_path, text)) => {
|
||||
info!(audio = %audio_path, "custom voice cleared — restored configured reference");
|
||||
state.tts.set_reference(Some(audio_path), Some(text));
|
||||
}
|
||||
None => {
|
||||
info!("custom voice cleared — no reference configured");
|
||||
state.tts.set_reference(None, None);
|
||||
}
|
||||
}
|
||||
Json(json!({ "ok": true }))
|
||||
}
|
||||
|
||||
/// The directory holding voice-cloning references: the parent of the
|
||||
/// configured reference WAV, or `voice_profiles` when nothing is configured.
|
||||
fn reference_dir(config: &Config) -> PathBuf {
|
||||
config
|
||||
.tts_reference_audio
|
||||
.as_ref()
|
||||
.and_then(|p| p.parent())
|
||||
.filter(|p| !p.as_os_str().is_empty())
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("voice_profiles"))
|
||||
}
|
||||
|
||||
/// Produce the final WAV reference: transcode the upload and, unless disabled,
|
||||
/// run the post-processing chain. Any post-processing failure degrades to the
|
||||
/// plain transcode rather than losing the take.
|
||||
async fn render_reference(config: &Config, input: &Path, output: &Path) -> Result<(), String> {
|
||||
if config.customvoice_postprocess {
|
||||
match postprocess(config, input, output).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(e) => warn!(
|
||||
error = %e,
|
||||
"custom-voice post-processing failed — storing the plain transcode"
|
||||
),
|
||||
}
|
||||
}
|
||||
transcode_to_wav(config, input, output).await
|
||||
}
|
||||
|
||||
/// Two-pass EBU R128 normalization over the cleaned signal: measure, then apply
|
||||
/// the measurement as a **single linear gain**. One gain for the whole take is
|
||||
/// what keeps the level consistent instead of pumping frame by frame — the
|
||||
/// usual cause of uneven volume in a cloned reference.
|
||||
async fn postprocess(config: &Config, input: &Path, output: &Path) -> Result<(), String> {
|
||||
let chain = cleanup_chain(config.customvoice_denoise);
|
||||
let opts = format!(
|
||||
"I={}:TP={}:LRA=11",
|
||||
config.customvoice_target_lufs, config.customvoice_true_peak
|
||||
);
|
||||
|
||||
// Pass 1 — measure. loudnorm prints its JSON summary through the logger, so
|
||||
// this run keeps the default log level.
|
||||
let measure_args: Vec<OsString> = vec![
|
||||
"-hide_banner".into(),
|
||||
"-nostdin".into(),
|
||||
"-nostats".into(),
|
||||
"-i".into(),
|
||||
input.into(),
|
||||
"-af".into(),
|
||||
format!("{chain},loudnorm={opts}:print_format=json").into(),
|
||||
"-f".into(),
|
||||
"null".into(),
|
||||
"-".into(),
|
||||
];
|
||||
let measured = run_ffmpeg(config, &measure_args).await?;
|
||||
let measured_stderr = String::from_utf8_lossy(&measured.stderr);
|
||||
let m = parse_loudnorm(&measured_stderr)
|
||||
.ok_or_else(|| "could not read the loudness measurement".to_string())?;
|
||||
|
||||
// Pass 2 — apply the measurement as a linear gain and write the reference.
|
||||
let apply = format!(
|
||||
"{chain},loudnorm={opts}:measured_I={}:measured_TP={}:measured_LRA={}:\
|
||||
measured_thresh={}:offset={}:linear=true",
|
||||
m.input_i, m.input_tp, m.input_lra, m.input_thresh, m.target_offset
|
||||
);
|
||||
let args: Vec<OsString> = vec![
|
||||
"-hide_banner".into(),
|
||||
"-nostdin".into(),
|
||||
"-loglevel".into(),
|
||||
"error".into(),
|
||||
"-y".into(),
|
||||
"-i".into(),
|
||||
input.into(),
|
||||
"-af".into(),
|
||||
apply.into(),
|
||||
"-ar".into(),
|
||||
OUTPUT_SAMPLE_RATE.to_string().into(),
|
||||
"-ac".into(),
|
||||
"1".into(),
|
||||
"-c:a".into(),
|
||||
"pcm_s16le".into(),
|
||||
output.into(),
|
||||
];
|
||||
run_ffmpeg(config, &args).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// The cleanup filters shared by both loudnorm passes: downmix to mono, then
|
||||
/// rumble/DC removal, optional FFT denoise and de-essing, and a
|
||||
/// leading/trailing silence trim (the clip is reversed to trim the tail with
|
||||
/// the same filter).
|
||||
///
|
||||
/// The mono downmix comes **first** on purpose: loudnorm measures whatever
|
||||
/// reaches it, so measuring a stereo input and only folding it to mono at the
|
||||
/// output stage would land the written file ~3 dB below the target (the energy
|
||||
/// of two channels collapses into one).
|
||||
fn cleanup_chain(denoise: bool) -> String {
|
||||
let trim = "silenceremove=start_periods=1:start_duration=0.1:\
|
||||
start_threshold=-50dB:start_silence=0.05";
|
||||
let mut filters: Vec<&str> = vec!["aformat=channel_layouts=mono", "highpass=f=80"];
|
||||
if denoise {
|
||||
filters.push("afftdn=nr=12:nf=-40");
|
||||
filters.push("deesser");
|
||||
}
|
||||
filters.extend([trim, "areverse", trim, "areverse"]);
|
||||
filters.join(",")
|
||||
}
|
||||
|
||||
/// Transcode `input` to a mono 16-bit PCM WAV at the mixer's sample rate.
|
||||
async fn transcode_to_wav(config: &Config, input: &Path, output: &Path) -> Result<(), String> {
|
||||
let args: Vec<OsString> = vec![
|
||||
"-hide_banner".into(),
|
||||
"-nostdin".into(),
|
||||
"-loglevel".into(),
|
||||
"error".into(),
|
||||
"-y".into(),
|
||||
"-i".into(),
|
||||
input.into(),
|
||||
"-ac".into(),
|
||||
"1".into(),
|
||||
"-ar".into(),
|
||||
OUTPUT_SAMPLE_RATE.to_string().into(),
|
||||
"-c:a".into(),
|
||||
"pcm_s16le".into(),
|
||||
output.into(),
|
||||
];
|
||||
run_ffmpeg(config, &args).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Run ffmpeg with `args`. Output is discarded; on a non-zero exit the last
|
||||
/// stderr line is surfaced as the error message.
|
||||
async fn run_ffmpeg(config: &Config, args: &[OsString]) -> Result<Output, String> {
|
||||
let out = Command::new(&config.media_ffmpeg_bin)
|
||||
.args(args)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("failed to spawn ffmpeg ({}): {e}", config.media_ffmpeg_bin))?;
|
||||
|
||||
if !out.status.success() {
|
||||
let err = String::from_utf8_lossy(&out.stderr);
|
||||
let last = err.trim().lines().last().unwrap_or("").to_string();
|
||||
return Err(if last.is_empty() {
|
||||
format!("ffmpeg exited with {}", out.status)
|
||||
} else {
|
||||
format!("ffmpeg: {last}")
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The loudness figures loudnorm reports in its JSON summary.
|
||||
struct Loudness {
|
||||
input_i: f64,
|
||||
input_tp: f64,
|
||||
input_lra: f64,
|
||||
input_thresh: f64,
|
||||
target_offset: f64,
|
||||
}
|
||||
|
||||
/// Extract the loudnorm JSON summary from ffmpeg's log output.
|
||||
fn parse_loudnorm(stderr: &str) -> Option<Loudness> {
|
||||
let start = stderr.rfind('{')?;
|
||||
let end = start + stderr[start..].find('}')?;
|
||||
let value: Value = serde_json::from_str(&stderr[start..=end]).ok()?;
|
||||
let num = |key: &str| value.get(key)?.as_str()?.trim().parse::<f64>().ok();
|
||||
Some(Loudness {
|
||||
input_i: num("input_i")?,
|
||||
input_tp: num("input_tp")?,
|
||||
input_lra: num("input_lra")?,
|
||||
input_thresh: num("input_thresh")?,
|
||||
target_offset: num("target_offset")?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a JSON error response.
|
||||
fn error_response(status: StatusCode, msg: &str) -> Response {
|
||||
(status, Json(json!({ "ok": false, "error": msg }))).into_response()
|
||||
}
|
||||
+59
@@ -61,6 +61,14 @@ impl ChatMessage {
|
||||
}
|
||||
}
|
||||
|
||||
/// A completed non-streaming reply: the visible text plus any tool calls the
|
||||
/// server returned, parsed either from the standard `tool_calls` array or from
|
||||
/// the model's native `<tool_call>` text (see [`crate::tools`]).
|
||||
pub struct ChatReply {
|
||||
pub text: String,
|
||||
pub tool_calls: Vec<crate::tools::ToolCall>,
|
||||
}
|
||||
|
||||
/// Client for an OpenAI-compatible `/chat/completions` endpoint.
|
||||
#[derive(Clone)]
|
||||
pub struct LlmClient {
|
||||
@@ -218,6 +226,57 @@ impl LlmClient {
|
||||
Ok(self.clean(&self.delta_text(message)))
|
||||
}
|
||||
|
||||
/// Non-streaming request that also surfaces tool calls.
|
||||
///
|
||||
/// Used as a fallback for a streaming round that came back empty. Several
|
||||
/// OpenAI-compatible servers (notably vLLM with a `--tool-call-parser` that
|
||||
/// does not match the model's native format) drop the call on the SSE path
|
||||
/// — the stream ends with no content — while the very same request without
|
||||
/// `stream` returns it intact. The call is parsed from the standard
|
||||
/// `tool_calls` field when present, otherwise from the `<tool_call>` text
|
||||
/// inside `content`.
|
||||
pub async fn chat_reply(
|
||||
&self,
|
||||
messages: &[ChatMessage],
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<ChatReply> {
|
||||
let body = self.request_body(messages, false);
|
||||
let req = self
|
||||
.http
|
||||
.post(&self.endpoint)
|
||||
.header("Authorization", format!("Bearer {}", self.api_key))
|
||||
.json(&body)
|
||||
.send();
|
||||
|
||||
let resp = tokio::select! {
|
||||
r = req => r?,
|
||||
_ = cancel.cancelled() => {
|
||||
return Ok(ChatReply { text: String::new(), tool_calls: Vec::new() });
|
||||
}
|
||||
};
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!("LLM HTTP {}: {}", status, text.trim()));
|
||||
}
|
||||
let value: serde_json::Value = resp.json().await?;
|
||||
let message = &value["choices"][0]["message"];
|
||||
|
||||
let mut tool_calls = crate::tools::parse_openai_tool_calls(message);
|
||||
let mut text = self.clean(&self.delta_text(message));
|
||||
if tool_calls.is_empty() {
|
||||
// No structured call: the server may have left the model's native
|
||||
// `<tool_call>` block in the text (mis-matched tool parser).
|
||||
let (visible, calls) = crate::tools::split_tool_calls(&text);
|
||||
if !calls.is_empty() {
|
||||
tool_calls = calls;
|
||||
text = visible;
|
||||
}
|
||||
}
|
||||
Ok(ChatReply { text, tool_calls })
|
||||
}
|
||||
|
||||
/// One-off non-streaming completion with an explicit system/user pair, its
|
||||
/// own token budget and temperature, and **no tools advertised**.
|
||||
///
|
||||
|
||||
+18
-2
@@ -7,6 +7,7 @@ mod assets;
|
||||
mod audio;
|
||||
mod audit;
|
||||
mod config;
|
||||
mod customvoice;
|
||||
mod lang;
|
||||
mod llm;
|
||||
mod media;
|
||||
@@ -25,11 +26,12 @@ use crate::orchestrator::FillerBank;
|
||||
use crate::stt::{ParakeetStt, Tokenizer};
|
||||
use crate::tts::create_tts_engine;
|
||||
use crate::vad::SileroVad;
|
||||
use crate::web::{MIVOCERO_HTML, PLAYGROUND_HTML};
|
||||
use crate::web::{CUSTOMVOICE_HTML, MIVOCERO_HTML, PLAYGROUND_HTML};
|
||||
use crate::ws::{ws_handler, AppState};
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::http::header::{CONTENT_TYPE, CACHE_CONTROL};
|
||||
use axum::response::{Html, IntoResponse, Response};
|
||||
use axum::routing::get;
|
||||
use axum::routing::{get, post};
|
||||
use axum::Router;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
@@ -232,6 +234,15 @@ async fn main() -> anyhow::Result<()> {
|
||||
let app = Router::new()
|
||||
.route("/", get(playground))
|
||||
.route("/mivocero", get(mivocero))
|
||||
.route("/customvoice", get(customvoice_page))
|
||||
// A recording reference is a few hundred KB, but the client may upload
|
||||
// an uncompressed WAV; lift the default 2 MB body cap for this route.
|
||||
.route(
|
||||
"/api/customvoice",
|
||||
post(customvoice::set_custom_voice).layer(DefaultBodyLimit::max(32 * 1024 * 1024)),
|
||||
)
|
||||
.route("/api/customvoice/clear", post(customvoice::clear_custom_voice))
|
||||
.route("/api/customvoice/phrases", get(customvoice::phrases))
|
||||
.route("/health", get(health))
|
||||
.route("/ws", get(ws_handler))
|
||||
.with_state(app_state);
|
||||
@@ -281,6 +292,11 @@ async fn mivocero() -> Response {
|
||||
html_response(MIVOCERO_HTML)
|
||||
}
|
||||
|
||||
/// `GET /customvoice` — serve the voice-cloning builder page.
|
||||
async fn customvoice_page() -> Response {
|
||||
html_response(CUSTOMVOICE_HTML)
|
||||
}
|
||||
|
||||
/// Build an HTML response with caching disabled.
|
||||
fn html_response(body: &'static str) -> Response {
|
||||
let mut resp = Html(body).into_response();
|
||||
|
||||
@@ -371,13 +371,19 @@ impl TurnManager {
|
||||
// external tokenizer only if no text was produced.
|
||||
let text = if !transcript.text.trim().is_empty() {
|
||||
transcript.text.trim().to_string()
|
||||
} else if transcript.token_ids.is_empty() {
|
||||
// No tokens at all: nothing was said (a VAD false-trigger that
|
||||
// slipped past the gate). Skip the turn instead of forwarding the
|
||||
// "<transcript 0 tokens>" placeholder to the LLM.
|
||||
debug!("no tokens decoded, skipping turn");
|
||||
return;
|
||||
} else {
|
||||
match &self.tokenizer {
|
||||
Some(tok) => tok.decode(&transcript.token_ids),
|
||||
None => format!("<transcript {} tokens>", transcript.token_ids.len()),
|
||||
}
|
||||
};
|
||||
if text.is_empty() {
|
||||
if text.trim().is_empty() {
|
||||
debug!("empty transcript after decode, skipping turn");
|
||||
return;
|
||||
}
|
||||
@@ -592,9 +598,10 @@ impl TurnManager {
|
||||
// Tool-calling loop. Each round streams one reply; tool call blocks are
|
||||
// stripped from the spoken/displayed text (the model emits them as
|
||||
// text — see `crate::tools`), the calls run over HTTP, and the results
|
||||
// are handed back for the next round. Bounded by
|
||||
// `LLM_TOOL_MAX_ITERATIONS`. With no tools configured the loop runs
|
||||
// exactly once.
|
||||
// are handed back for the next round. A round that streams back empty
|
||||
// is re-asked without streaming, where the server returns the call it
|
||||
// dropped from the SSE path. Bounded by `LLM_TOOL_MAX_ITERATIONS`. With
|
||||
// no tools configured the loop runs exactly once.
|
||||
let registry = llm.tools().cloned();
|
||||
let max_rounds = self.config.llm_tool_max_iterations.max(1);
|
||||
let mut spoken = String::new();
|
||||
@@ -678,7 +685,36 @@ impl TurnManager {
|
||||
break;
|
||||
}
|
||||
|
||||
let calls = stripper.take_calls();
|
||||
let mut calls = stripper.take_calls();
|
||||
|
||||
// Streaming rescue. Some OpenAI-compatible servers swallow the
|
||||
// model's native tool call on the SSE path — vLLM with a
|
||||
// `--tool-call-parser` that does not match the model's format ends
|
||||
// the stream with no content at all — while the same request
|
||||
// *without* streaming returns the call intact. When a round
|
||||
// produced nothing and tools are configured, re-ask once,
|
||||
// non-streaming, and take the call from there.
|
||||
if calls.is_empty()
|
||||
&& registry.is_some()
|
||||
&& !cancel.is_cancelled()
|
||||
&& spoken.trim().is_empty()
|
||||
&& round + 1 < max_rounds
|
||||
{
|
||||
match llm.chat_reply(&messages, &cancel).await {
|
||||
Ok(reply) => {
|
||||
calls = reply.tool_calls;
|
||||
if calls.is_empty() && !reply.text.trim().is_empty() {
|
||||
// No call after all: the blocking reply holds the
|
||||
// actual answer — speak it rather than go silent.
|
||||
self.emit(json!({ "type": "assistant_delta", "text": &reply.text }));
|
||||
spoken.push_str(&reply.text);
|
||||
let _ = sentence_tx.send(reply.text.trim().to_string());
|
||||
}
|
||||
}
|
||||
Err(e) => error!(error = %e, "tool-call fallback request failed"),
|
||||
}
|
||||
}
|
||||
|
||||
if calls.is_empty() {
|
||||
break;
|
||||
}
|
||||
@@ -730,11 +766,21 @@ impl TurnManager {
|
||||
}
|
||||
|
||||
/// Whether the agent is currently speaking (TTS in flight or queued audio).
|
||||
/// Used by the VAD-driven barge-in to avoid flushing an idle mixer.
|
||||
/// Used by the backchannel gate to avoid cuing on top of the agent's own
|
||||
/// TTS — see [`Self::is_audible`] for the barge-in gate.
|
||||
pub fn is_speaking(&self) -> bool {
|
||||
self.mixer.is_active()
|
||||
}
|
||||
|
||||
/// Whether the agent has *audible* output (reply playing out, or a
|
||||
/// latency-mask cue queued). Unlike [`Self::is_speaking`] this is `false`
|
||||
/// while the reply is still being synthesised or sits in the pre-buffer.
|
||||
/// Used by the VAD-driven barge-in: a transient must not cancel a reply
|
||||
/// before the user has heard any of it.
|
||||
pub fn is_audible(&self) -> bool {
|
||||
self.mixer.is_audible()
|
||||
}
|
||||
|
||||
/// Update the voice-cloning reference at runtime. Pass `None` for
|
||||
/// `audio_path` to clear the reference.
|
||||
pub fn set_reference(&self, audio_path: Option<String>, text: Option<String>) {
|
||||
|
||||
@@ -341,6 +341,32 @@ fn coerce_value(raw: &str) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse the structured `tool_calls` array of an OpenAI chat `message` object.
|
||||
///
|
||||
/// Used when the server *does* run a matching tool parser and returns the call
|
||||
/// in the standard shape — `message.tool_calls[].function.{name,arguments}` —
|
||||
/// as opposed to the native `<tool_call>` text handled by [`split_tool_calls`].
|
||||
/// Per the OpenAI spec `arguments` is a JSON-encoded string; a payload that does
|
||||
/// not decode to an object yields an empty argument set.
|
||||
pub fn parse_openai_tool_calls(message: &Value) -> Vec<ToolCall> {
|
||||
let Some(items) = message["tool_calls"].as_array() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut calls = Vec::new();
|
||||
for item in items {
|
||||
let Some(name) = item["function"]["name"].as_str() else {
|
||||
continue;
|
||||
};
|
||||
let raw = item["function"]["arguments"].as_str().unwrap_or("");
|
||||
let arguments = serde_json::from_str::<Value>(raw.trim())
|
||||
.ok()
|
||||
.filter(Value::is_object)
|
||||
.unwrap_or_else(|| Value::Object(Map::new()));
|
||||
calls.push(ToolCall { name: name.to_string(), arguments });
|
||||
}
|
||||
calls
|
||||
}
|
||||
|
||||
// --- Streaming stripper -----------------------------------------------------
|
||||
|
||||
/// Incremental filter that turns a stream of raw model deltas into visible
|
||||
@@ -472,4 +498,32 @@ mod tests {
|
||||
assert_eq!(s.push("Nessun tool qui."), "Nessun tool qui.");
|
||||
assert!(s.take_calls().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_structured_tool_calls() {
|
||||
let message = serde_json::json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": { "name": "get_weather", "arguments": "{\"city\":\"Paris\"}" }
|
||||
}]
|
||||
});
|
||||
let calls = parse_openai_tool_calls(&message);
|
||||
assert_eq!(calls.len(), 1);
|
||||
assert_eq!(calls[0].name, "get_weather");
|
||||
assert_eq!(calls[0].arguments["city"], "Paris");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structured_tool_calls_absent_or_malformed() {
|
||||
assert!(parse_openai_tool_calls(&serde_json::json!({ "content": "hi" })).is_empty());
|
||||
let message = serde_json::json!({
|
||||
"tool_calls": [{ "function": { "name": "f", "arguments": "not json" } }]
|
||||
});
|
||||
let calls = parse_openai_tool_calls(&message);
|
||||
assert_eq!(calls.len(), 1);
|
||||
assert!(calls[0].arguments.as_object().unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
+19
-1
@@ -44,6 +44,20 @@ pub struct VadSettings {
|
||||
pub speech_pad_ms: u32,
|
||||
/// Speech required to fire the barge-in onset (`VAD_BARGE_IN_MIN_SPEECH_MS`).
|
||||
pub barge_in_min_speech_ms: u32,
|
||||
/// Drop an utterance shorter than this before transcription
|
||||
/// (`VAD_MIN_UTTERANCE_MS`, default 300). `0` disables the check. Filters
|
||||
/// the short VAD false-triggers (breaths, clicks, echo).
|
||||
pub min_utterance_ms: u32,
|
||||
/// Drop an utterance whose average speech probability is below this before
|
||||
/// transcription (`VAD_MIN_AVG_PROB`, default 0.25). `0.0` disables the
|
||||
/// check. Filters the soft/noisy segments that the STT would otherwise
|
||||
/// turn into hallucinated text. The average is taken over the **voiced
|
||||
/// windows only** (probability ≥ `neg_threshold`), so the trailing
|
||||
/// hangover silence (`min_silence_ms`) does not dilute short utterances
|
||||
/// like "uno" / "due". A genuine utterance still fluctuates around the
|
||||
/// speech threshold on its onset/release, so keep this well below
|
||||
/// `threshold` (default 0.5).
|
||||
pub min_avg_prob: f32,
|
||||
/// Feed the STT engine with VAD-segmented utterances instead of letting it
|
||||
/// run its own RMS segmentation (`VAD_SEGMENT_STT`).
|
||||
pub segment_stt: bool,
|
||||
@@ -62,9 +76,11 @@ impl Default for VadSettings {
|
||||
threshold: 0.5,
|
||||
neg_threshold: 0.35,
|
||||
min_speech_ms: 250,
|
||||
min_silence_ms: 300,
|
||||
min_silence_ms: 700,
|
||||
speech_pad_ms: 30,
|
||||
barge_in_min_speech_ms: 150,
|
||||
min_utterance_ms: 300,
|
||||
min_avg_prob: 0.25,
|
||||
segment_stt: true,
|
||||
barge_in: true,
|
||||
debug: false,
|
||||
@@ -91,6 +107,8 @@ impl VadSettings {
|
||||
s.min_silence_ms = env_u32("VAD_MIN_SILENCE_MS", s.min_silence_ms);
|
||||
s.speech_pad_ms = env_u32("VAD_SPEECH_PAD_MS", s.speech_pad_ms);
|
||||
s.barge_in_min_speech_ms = env_u32("VAD_BARGE_IN_MIN_SPEECH_MS", s.barge_in_min_speech_ms);
|
||||
s.min_utterance_ms = env_u32("VAD_MIN_UTTERANCE_MS", s.min_utterance_ms);
|
||||
s.min_avg_prob = env_f32("VAD_MIN_AVG_PROB", s.min_avg_prob);
|
||||
|
||||
// Hysteresis must stay below the speech threshold, and the fast
|
||||
// barge-in onset can never be slower than the utterance onset.
|
||||
|
||||
+11
-3
@@ -219,8 +219,10 @@ pub struct VadDetector {
|
||||
speech_run_ms: u32,
|
||||
/// Consecutive silence milliseconds while in speech.
|
||||
silence_run_ms: u32,
|
||||
/// Running probability average over the current utterance.
|
||||
/// Running sum of speech probabilities over the voiced windows of the
|
||||
/// current utterance.
|
||||
prob_sum: f32,
|
||||
/// Number of voiced windows contributing to `prob_sum`.
|
||||
speech_windows: u64,
|
||||
/// Windows analysed since the detector was created (for logging).
|
||||
window_count: u64,
|
||||
@@ -365,8 +367,14 @@ impl VadDetector {
|
||||
}
|
||||
|
||||
self.utterance.extend_from_slice(window);
|
||||
self.prob_sum += prob;
|
||||
self.speech_windows += 1;
|
||||
// Average only over the voiced windows. The trailing hangover (the
|
||||
// `min_silence_ms` of silence that closes the segment) must not dilute
|
||||
// the score: for a short word like "uno" it is most of the segment and
|
||||
// would drag the mean below any sane gate.
|
||||
if prob >= self.settings.neg_threshold {
|
||||
self.prob_sum += prob;
|
||||
self.speech_windows += 1;
|
||||
}
|
||||
|
||||
if prob < self.settings.neg_threshold {
|
||||
self.silence_run_ms = self.silence_run_ms.saturating_add(WINDOW_MS);
|
||||
|
||||
@@ -0,0 +1,568 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Custom Voice — Mivocero</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0b0f1a;
|
||||
--panel: #141b2d;
|
||||
--accent: #4f8cff;
|
||||
--accent-dim: #2a4a8a;
|
||||
--ok: #3ddc97;
|
||||
--warn: #ffb454;
|
||||
--err: #ff5470;
|
||||
--text: #e6edf7;
|
||||
--muted: #8a94a6;
|
||||
--border: #243049;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html, body { height: 100%; }
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 18px;
|
||||
}
|
||||
.app {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
max-width: 820px;
|
||||
height: calc(100vh - 36px);
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
overflow: hidden;
|
||||
}
|
||||
header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 14px 18px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
header h1 { font-size: 16px; margin: 0; flex: 1; }
|
||||
main { flex: 1; overflow-y: auto; padding: 20px 22px; }
|
||||
.step { margin-bottom: 22px; }
|
||||
.step h2 {
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .7px;
|
||||
color: var(--muted);
|
||||
margin: 0 0 8px;
|
||||
}
|
||||
select {
|
||||
width: 100%;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 10px 12px;
|
||||
font-size: 14px;
|
||||
}
|
||||
.phrase {
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
border-left: 3px solid var(--accent);
|
||||
border-radius: 8px;
|
||||
padding: 14px 16px;
|
||||
font-size: 15px;
|
||||
line-height: 1.55;
|
||||
color: var(--text);
|
||||
}
|
||||
.actions { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; }
|
||||
button {
|
||||
background: var(--accent-dim);
|
||||
color: var(--text);
|
||||
border: 1px solid var(--accent);
|
||||
border-radius: 8px;
|
||||
padding: 9px 16px;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
transition: all .15s;
|
||||
}
|
||||
button:hover:not(:disabled) { background: var(--accent); }
|
||||
button:disabled { opacity: .4; cursor: not-allowed; }
|
||||
button.danger { border-color: var(--err); background: rgba(255,84,112,.15); }
|
||||
button.danger:hover:not(:disabled) { background: var(--err); }
|
||||
button.ghost { background: transparent; border-color: var(--border); }
|
||||
button.ghost:hover:not(:disabled) { background: var(--bg); border-color: var(--accent); }
|
||||
.timer {
|
||||
font-variant-numeric: tabular-nums;
|
||||
font-size: 15px;
|
||||
color: var(--muted);
|
||||
min-width: 74px;
|
||||
}
|
||||
.timer.ready { color: var(--ok); }
|
||||
.timer.rec { color: var(--err); }
|
||||
.rec-dot {
|
||||
width: 10px; height: 10px; border-radius: 50%;
|
||||
background: var(--err); display: none; flex: none;
|
||||
}
|
||||
.recording .rec-dot { display: inline-block; animation: pulse 1s infinite; }
|
||||
@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: .25; } }
|
||||
.bar { height: 5px; background: var(--bg); border: 1px solid var(--border); border-radius: 999px; overflow: hidden; margin-top: 12px; }
|
||||
.bar > div { height: 100%; width: 0; background: var(--accent); transition: width .12s linear; }
|
||||
.bar.ready > div { background: var(--ok); }
|
||||
audio { width: 100%; margin-top: 12px; }
|
||||
.tabs { display: flex; gap: 8px; }
|
||||
.tab {
|
||||
flex: 1;
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--muted);
|
||||
border-radius: 8px;
|
||||
padding: 10px 12px;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
transition: all .15s;
|
||||
}
|
||||
.tab:hover { border-color: var(--accent); color: var(--text); }
|
||||
.tab.active { background: var(--accent-dim); border-color: var(--accent); color: var(--text); }
|
||||
input[type=file] {
|
||||
width: 100%;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
border: 1px dashed var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 12px;
|
||||
font-size: 13px;
|
||||
}
|
||||
input[type=file]::file-selector-button {
|
||||
background: var(--accent-dim);
|
||||
color: var(--text);
|
||||
border: 1px solid var(--accent);
|
||||
border-radius: 6px;
|
||||
padding: 6px 12px;
|
||||
margin-right: 12px;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
}
|
||||
textarea {
|
||||
width: 100%;
|
||||
min-height: 90px;
|
||||
resize: vertical;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 10px 12px;
|
||||
font-size: 14px;
|
||||
font-family: inherit;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.note { font-size: 13px; color: var(--muted); line-height: 1.5; margin: 8px 0 0; }
|
||||
.log {
|
||||
margin-top: 16px;
|
||||
padding: 10px 12px;
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
font-family: ui-monospace, Menlo, monospace;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
white-space: pre-wrap;
|
||||
min-height: 20px;
|
||||
}
|
||||
.log.ok { color: var(--ok); border-color: var(--ok); }
|
||||
.log.err { color: var(--err); border-color: var(--err); }
|
||||
|
||||
/* --- Shared side menu --- */
|
||||
.menu-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
border-radius: 8px;
|
||||
padding: 6px 11px;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
}
|
||||
.menu-btn:hover { background: var(--bg); }
|
||||
.sidemenu {
|
||||
position: fixed;
|
||||
top: 0; left: 0;
|
||||
height: 100%;
|
||||
width: 240px;
|
||||
background: var(--panel);
|
||||
border-right: 1px solid var(--border);
|
||||
transform: translateX(-100%);
|
||||
transition: transform .2s ease;
|
||||
z-index: 40;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 18px 14px;
|
||||
gap: 6px;
|
||||
}
|
||||
.sidemenu.open { transform: translateX(0); }
|
||||
.sidemenu h2 {
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .7px;
|
||||
color: var(--muted);
|
||||
margin: 0 0 10px;
|
||||
padding: 0 12px;
|
||||
}
|
||||
.sidemenu a {
|
||||
display: block;
|
||||
padding: 10px 12px;
|
||||
border-radius: 8px;
|
||||
color: var(--text);
|
||||
text-decoration: none;
|
||||
font-size: 14px;
|
||||
border: 1px solid transparent;
|
||||
}
|
||||
.sidemenu a:hover { background: var(--bg); }
|
||||
.sidemenu a.active { background: var(--accent-dim); border-color: var(--accent); }
|
||||
.backdrop {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0,0,0,.45);
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity .2s;
|
||||
z-index: 30;
|
||||
}
|
||||
.backdrop.open { opacity: 1; pointer-events: auto; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="backdrop" id="menuBackdrop"></div>
|
||||
<nav class="sidemenu" id="sidemenu">
|
||||
<h2>Menu</h2>
|
||||
<a href="/mivocero">🎙️ Mivocero</a>
|
||||
<a href="/customvoice" class="active">🎨 Custom Voice</a>
|
||||
</nav>
|
||||
|
||||
<div class="app">
|
||||
<header>
|
||||
<button class="menu-btn" id="menuBtn" title="Menu">☰</button>
|
||||
<h1>🎨 Custom Voice</h1>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="step">
|
||||
<h2>1 · Language</h2>
|
||||
<select id="langSelect"></select>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h2>2 · Source</h2>
|
||||
<div class="tabs">
|
||||
<button type="button" class="tab active" id="tabRecord">🎙️ Record now</button>
|
||||
<button type="button" class="tab" id="tabUpload">📁 Upload a file</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="recordSection">
|
||||
<div class="step">
|
||||
<h2>3 · Read this phrase out loud</h2>
|
||||
<div class="phrase" id="phrase">Loading…</div>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h2>4 · Record (at least 10 seconds)</h2>
|
||||
<div class="actions" id="recordRow">
|
||||
<button id="btnRecord">● Start recording</button>
|
||||
<button id="btnStop" class="danger" disabled>■ Stop</button>
|
||||
<button id="btnReset" class="ghost" disabled>Discard</button>
|
||||
<span class="rec-dot"></span>
|
||||
<span class="timer" id="timer">0:00 / 0:10</span>
|
||||
</div>
|
||||
<div class="bar" id="bar"><div id="barFill"></div></div>
|
||||
<audio id="preview" controls hidden></audio>
|
||||
<p class="note">
|
||||
Speak naturally in a quiet room. The recording is transcoded to a WAV
|
||||
voice-cloning reference and installed as the active voice for the agent.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="uploadSection" hidden>
|
||||
<div class="step">
|
||||
<h2>3 · Upload an audio file</h2>
|
||||
<input type="file" id="fileInput" accept="audio/*" />
|
||||
<audio id="uploadPreview" controls hidden></audio>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h2>4 · Type the transcript</h2>
|
||||
<textarea id="uploadText" placeholder="Type (or paste) the exact text spoken in the audio…"></textarea>
|
||||
<p class="note">
|
||||
The transcript must match what is said in the file — it drives the
|
||||
voice-cloning reference, so accuracy matters. Any format ffmpeg can
|
||||
read (wav, mp3, m4a, ogg, flac…) works.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h2>5 · Use it</h2>
|
||||
<div class="actions">
|
||||
<button id="btnSave" disabled>✓ Use as custom voice</button>
|
||||
<button id="btnClear" class="ghost">↺ Reset to default voice</button>
|
||||
</div>
|
||||
<div class="log" id="log"></div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(() => {
|
||||
// --- Side menu -----------------------------------------------------------
|
||||
const menu = document.getElementById('sidemenu');
|
||||
const backdrop = document.getElementById('menuBackdrop');
|
||||
const setMenu = (open) => {
|
||||
menu.classList.toggle('open', open);
|
||||
backdrop.classList.toggle('open', open);
|
||||
};
|
||||
document.getElementById('menuBtn').addEventListener('click', () => setMenu(true));
|
||||
backdrop.addEventListener('click', () => setMenu(false));
|
||||
|
||||
// --- Elements ------------------------------------------------------------
|
||||
const langSelect = document.getElementById('langSelect');
|
||||
const phraseBox = document.getElementById('phrase');
|
||||
const btnRecord = document.getElementById('btnRecord');
|
||||
const btnStop = document.getElementById('btnStop');
|
||||
const btnReset = document.getElementById('btnReset');
|
||||
const btnSave = document.getElementById('btnSave');
|
||||
const btnClear = document.getElementById('btnClear');
|
||||
const timerEl = document.getElementById('timer');
|
||||
const bar = document.getElementById('bar');
|
||||
const barFill = document.getElementById('barFill');
|
||||
const preview = document.getElementById('preview');
|
||||
const logEl = document.getElementById('log');
|
||||
const tabRecord = document.getElementById('tabRecord');
|
||||
const tabUpload = document.getElementById('tabUpload');
|
||||
const recordSection = document.getElementById('recordSection');
|
||||
const uploadSection = document.getElementById('uploadSection');
|
||||
const fileInput = document.getElementById('fileInput');
|
||||
const uploadText = document.getElementById('uploadText');
|
||||
const uploadPreview = document.getElementById('uploadPreview');
|
||||
|
||||
const MIN_SECONDS = 10;
|
||||
const AUTO_STOP_SECONDS = 20;
|
||||
|
||||
let languages = [];
|
||||
let stream = null;
|
||||
let recorder = null;
|
||||
let chunks = [];
|
||||
let blob = null;
|
||||
let uploadFile = null;
|
||||
let mode = 'record';
|
||||
let elapsed = 0;
|
||||
let ticker = null;
|
||||
let startedAt = 0;
|
||||
|
||||
const fmt = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, '0')}`;
|
||||
const log = (msg, cls = '') => { logEl.textContent = msg; logEl.className = 'log ' + cls; };
|
||||
|
||||
const currentPhrase = () => {
|
||||
const entry = languages.find(l => l.code === langSelect.value);
|
||||
return entry ? entry.phrase : '';
|
||||
};
|
||||
|
||||
// The save button is enabled once the active source holds a usable take:
|
||||
// a recording (record mode) or a file plus a non-blank transcript (upload).
|
||||
const refreshSave = () => {
|
||||
btnSave.disabled = mode === 'record'
|
||||
? !blob
|
||||
: !(uploadFile && uploadText.value.trim());
|
||||
};
|
||||
|
||||
// --- Source toggle -------------------------------------------------------
|
||||
function setMode(next) {
|
||||
mode = next;
|
||||
const rec = next === 'record';
|
||||
tabRecord.classList.toggle('active', rec);
|
||||
tabUpload.classList.toggle('active', !rec);
|
||||
recordSection.hidden = !rec;
|
||||
uploadSection.hidden = rec;
|
||||
log('');
|
||||
refreshSave();
|
||||
}
|
||||
|
||||
tabRecord.addEventListener('click', () => setMode('record'));
|
||||
tabUpload.addEventListener('click', () => setMode('upload'));
|
||||
|
||||
// --- Load the phrases ----------------------------------------------------
|
||||
fetch('/api/customvoice/phrases')
|
||||
.then(r => r.json())
|
||||
.then(data => {
|
||||
languages = data.languages || [];
|
||||
langSelect.innerHTML = '';
|
||||
for (const l of languages) {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = l.code;
|
||||
opt.textContent = l.label;
|
||||
langSelect.appendChild(opt);
|
||||
}
|
||||
if (languages.some(l => l.code === data.default)) langSelect.value = data.default;
|
||||
phraseBox.textContent = currentPhrase();
|
||||
})
|
||||
.catch(e => log('Failed to load phrases: ' + e.message, 'err'));
|
||||
|
||||
langSelect.addEventListener('change', () => { phraseBox.textContent = currentPhrase(); });
|
||||
|
||||
// --- Recording -----------------------------------------------------------
|
||||
function updateTimer() {
|
||||
elapsed = Math.min(AUTO_STOP_SECONDS, (performance.now() - startedAt) / 1000);
|
||||
timerEl.textContent = `${fmt(elapsed)} / ${fmt(MIN_SECONDS)}`;
|
||||
barFill.style.width = Math.min(100, (elapsed / MIN_SECONDS) * 100) + '%';
|
||||
if (elapsed >= MIN_SECONDS) {
|
||||
timerEl.classList.add('ready');
|
||||
bar.classList.add('ready');
|
||||
btnStop.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
btnRecord.addEventListener('click', async () => {
|
||||
log('');
|
||||
blob = null;
|
||||
refreshSave();
|
||||
preview.hidden = true;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
} catch (e) {
|
||||
log('Microphone access denied: ' + e.message, 'err');
|
||||
return;
|
||||
}
|
||||
const candidates = ['audio/webm;codecs=opus', 'audio/webm', 'audio/ogg;codecs=opus', 'audio/mp4'];
|
||||
const mime = candidates.find(m => window.MediaRecorder && MediaRecorder.isTypeSupported(m));
|
||||
try {
|
||||
recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
|
||||
} catch (e) {
|
||||
log('MediaRecorder unavailable: ' + e.message, 'err');
|
||||
stopStream();
|
||||
return;
|
||||
}
|
||||
|
||||
chunks = [];
|
||||
recorder.ondataavailable = (e) => { if (e.data && e.data.size) chunks.push(e.data); };
|
||||
recorder.onstop = () => {
|
||||
clearInterval(ticker);
|
||||
document.getElementById('recordRow').classList.remove('recording');
|
||||
blob = new Blob(chunks, { type: recorder.mimeType || 'audio/webm' });
|
||||
stopStream();
|
||||
if (elapsed < MIN_SECONDS) {
|
||||
log(`Recording too short (${elapsed.toFixed(1)} s) — please read for at least ${MIN_SECONDS} s.`, 'err');
|
||||
return;
|
||||
}
|
||||
preview.src = URL.createObjectURL(blob);
|
||||
preview.hidden = false;
|
||||
refreshSave();
|
||||
btnReset.disabled = false;
|
||||
log(`Recording ready (${elapsed.toFixed(1)} s). Listen, then use it.`, 'ok');
|
||||
};
|
||||
|
||||
elapsed = 0;
|
||||
startedAt = performance.now();
|
||||
timerEl.textContent = `${fmt(0)} / ${fmt(MIN_SECONDS)}`;
|
||||
timerEl.classList.remove('ready');
|
||||
bar.classList.remove('ready');
|
||||
barFill.style.width = '0%';
|
||||
btnStop.disabled = true;
|
||||
btnReset.disabled = true;
|
||||
document.getElementById('recordRow').classList.add('recording');
|
||||
timerEl.classList.add('rec');
|
||||
|
||||
recorder.start();
|
||||
ticker = setInterval(() => {
|
||||
updateTimer();
|
||||
if (elapsed >= AUTO_STOP_SECONDS) recorder.stop();
|
||||
}, 100);
|
||||
btnRecord.disabled = true;
|
||||
});
|
||||
|
||||
btnStop.addEventListener('click', () => { if (recorder && recorder.state !== 'inactive') recorder.stop(); });
|
||||
|
||||
function stopStream() {
|
||||
btnRecord.disabled = false;
|
||||
timerEl.classList.remove('rec');
|
||||
if (stream) { stream.getTracks().forEach(t => t.stop()); stream = null; }
|
||||
}
|
||||
|
||||
btnReset.addEventListener('click', () => {
|
||||
blob = null;
|
||||
preview.hidden = true;
|
||||
preview.removeAttribute('src');
|
||||
refreshSave();
|
||||
btnReset.disabled = true;
|
||||
elapsed = 0;
|
||||
timerEl.textContent = `${fmt(0)} / ${fmt(MIN_SECONDS)}`;
|
||||
timerEl.classList.remove('ready');
|
||||
bar.classList.remove('ready');
|
||||
barFill.style.width = '0%';
|
||||
log('');
|
||||
});
|
||||
|
||||
// --- Upload --------------------------------------------------------------
|
||||
fileInput.addEventListener('change', () => {
|
||||
uploadFile = fileInput.files && fileInput.files[0] ? fileInput.files[0] : null;
|
||||
if (uploadFile) {
|
||||
uploadPreview.src = URL.createObjectURL(uploadFile);
|
||||
uploadPreview.hidden = false;
|
||||
} else {
|
||||
uploadPreview.hidden = true;
|
||||
uploadPreview.removeAttribute('src');
|
||||
}
|
||||
refreshSave();
|
||||
});
|
||||
|
||||
uploadText.addEventListener('input', refreshSave);
|
||||
|
||||
// --- Save / clear --------------------------------------------------------
|
||||
btnSave.addEventListener('click', async () => {
|
||||
// Pick the active source: the recording (record mode) or the chosen file
|
||||
// plus its typed transcript (upload mode).
|
||||
let body, text, type;
|
||||
if (mode === 'record') {
|
||||
if (!blob) return;
|
||||
body = blob;
|
||||
type = blob.type || 'application/octet-stream';
|
||||
} else {
|
||||
text = uploadText.value.trim();
|
||||
if (!uploadFile || !text) return;
|
||||
body = uploadFile;
|
||||
type = uploadFile.type || 'application/octet-stream';
|
||||
}
|
||||
|
||||
btnSave.disabled = true;
|
||||
log('Uploading…');
|
||||
try {
|
||||
let url = '/api/customvoice?lang=' + encodeURIComponent(langSelect.value);
|
||||
if (text) url += '&text=' + encodeURIComponent(text);
|
||||
const r = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': type },
|
||||
body,
|
||||
});
|
||||
const data = await r.json().catch(() => ({}));
|
||||
if (!r.ok || !data.ok) throw new Error(data.error || ('HTTP ' + r.status));
|
||||
log('Custom voice installed — ' + data.audio, 'ok');
|
||||
} catch (e) {
|
||||
log('Failed: ' + e.message, 'err');
|
||||
btnSave.disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
btnClear.addEventListener('click', async () => {
|
||||
log('Resetting…');
|
||||
try {
|
||||
const r = await fetch('/api/customvoice/clear', { method: 'POST' });
|
||||
const data = await r.json().catch(() => ({}));
|
||||
if (!r.ok || !data.ok) throw new Error(data.error || ('HTTP ' + r.status));
|
||||
log('Default voice restored.', 'ok');
|
||||
} catch (e) {
|
||||
log('Failed: ' + e.message, 'err');
|
||||
}
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+192
-2
@@ -93,8 +93,29 @@
|
||||
.bubble.karaoke .kw { color: var(--muted); transition: color .18s ease; }
|
||||
.bubble.karaoke .kw.done { color: var(--text); }
|
||||
.bubble.karaoke .kw.now { color: var(--accent); text-shadow: 0 0 8px rgba(79,140,255,.55); }
|
||||
/* Same karaoke style on the user's own bubbles, tuned for the blue bubble. */
|
||||
.msg.user .bubble.karaoke .kw { color: rgba(230,237,247,.5); }
|
||||
.msg.user .bubble.karaoke .kw.done { color: #ffffff; }
|
||||
.msg.user .bubble.karaoke .kw.now { color: #ffffff; text-shadow: 0 0 9px rgba(255,255,255,.85); }
|
||||
.hint { color: var(--muted); font-size: 13px; text-align: center; margin: auto; }
|
||||
|
||||
/* Barge-in indicator: flashes red whenever the agent is interrupted. */
|
||||
.bargein {
|
||||
display: inline-flex; align-items: center; gap: 7px;
|
||||
font-size: 12px; color: var(--muted);
|
||||
border: 1px solid var(--border); border-radius: 999px;
|
||||
padding: 5px 11px; white-space: nowrap;
|
||||
transition: border-color .2s, box-shadow .2s, color .2s;
|
||||
}
|
||||
.bargein .bi-dot {
|
||||
width: 9px; height: 9px; border-radius: 50%; flex: none;
|
||||
background: var(--err); opacity: .25;
|
||||
transition: opacity .15s, box-shadow .15s, transform .15s;
|
||||
}
|
||||
.bargein b { color: var(--text); font-variant-numeric: tabular-nums; }
|
||||
.bargein.active { border-color: var(--err); color: var(--err); box-shadow: 0 0 12px -2px var(--err); }
|
||||
.bargein.active .bi-dot { opacity: 1; box-shadow: 0 0 9px var(--err); transform: scale(1.35); }
|
||||
|
||||
footer { border-top: 1px solid var(--border); padding: 12px 18px; }
|
||||
.meters { display: flex; align-items: center; gap: 14px; margin-bottom: 10px; }
|
||||
.meter { height: 8px; background: var(--bg); border-radius: 4px; overflow: hidden; flex: 1; }
|
||||
@@ -225,13 +246,82 @@
|
||||
color: var(--muted);
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
/* --- Shared side menu --- */
|
||||
.menu-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
border-radius: 8px;
|
||||
padding: 6px 11px;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
}
|
||||
.menu-btn:hover { background: var(--bg); }
|
||||
.sidemenu {
|
||||
position: fixed;
|
||||
top: 0; left: 0;
|
||||
height: 100%;
|
||||
width: 240px;
|
||||
background: var(--panel);
|
||||
border-right: 1px solid var(--border);
|
||||
transform: translateX(-100%);
|
||||
transition: transform .2s ease;
|
||||
z-index: 40;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 18px 14px;
|
||||
gap: 6px;
|
||||
}
|
||||
.sidemenu.open { transform: translateX(0); }
|
||||
.sidemenu h2 {
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .7px;
|
||||
color: var(--muted);
|
||||
margin: 0 0 10px;
|
||||
padding: 0 12px;
|
||||
}
|
||||
.sidemenu a {
|
||||
display: block;
|
||||
padding: 10px 12px;
|
||||
border-radius: 8px;
|
||||
color: var(--text);
|
||||
text-decoration: none;
|
||||
font-size: 14px;
|
||||
border: 1px solid transparent;
|
||||
}
|
||||
.sidemenu a:hover { background: var(--bg); }
|
||||
.sidemenu a.active { background: var(--accent-dim); border-color: var(--accent); }
|
||||
.backdrop {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0,0,0,.45);
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity .2s;
|
||||
z-index: 30;
|
||||
}
|
||||
.backdrop.open { opacity: 1; pointer-events: auto; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="backdrop" id="menuBackdrop"></div>
|
||||
<nav class="sidemenu" id="sidemenu">
|
||||
<h2>Menu</h2>
|
||||
<a href="/mivocero" class="active">🎙️ Mivocero</a>
|
||||
<a href="/customvoice">🎨 Custom Voice</a>
|
||||
</nav>
|
||||
|
||||
<div class="app">
|
||||
<header>
|
||||
<button class="menu-btn" id="menuBtn" title="Menu">☰</button>
|
||||
<h1>🎙️ Mivocero</h1>
|
||||
<span class="status"><span class="dot" id="connDot"></span><span id="connText">disconnected</span></span>
|
||||
<span class="bargein" id="bargeinInd" title="Agent interrupted by the user (VAD barge-in or the Stop button)">
|
||||
<span class="bi-dot"></span>Barge-in <b id="bargeinCount">0</b>
|
||||
</span>
|
||||
<button id="btnConnect" class="ok">Connect</button>
|
||||
<button id="btnInterrupt" class="danger" disabled>Stop</button>
|
||||
</header>
|
||||
@@ -307,11 +397,24 @@
|
||||
// {"type":"audio_source","state":"…", …} external audio source status
|
||||
// {"type":"cue","kind":"…","name":"…"} filler / start filler / backchannel emitted
|
||||
// {"type":"cues","lang":"…","config":{…}} live cue policy (sent at start)
|
||||
// {"type":"barge_in","source":"…"} agent interrupted (VAD onset)
|
||||
//
|
||||
// The session closes itself after IDLE_TIMEOUT_MS without any traffic in
|
||||
// either direction (see the idle guard below).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// --- Shared side menu (chat / custom voice navigation) ---
|
||||
(() => {
|
||||
const menu = document.getElementById('sidemenu');
|
||||
const backdrop = document.getElementById('menuBackdrop');
|
||||
const setOpen = (open) => {
|
||||
menu.classList.toggle('open', open);
|
||||
backdrop.classList.toggle('open', open);
|
||||
};
|
||||
document.getElementById('menuBtn').addEventListener('click', () => setOpen(true));
|
||||
backdrop.addEventListener('click', () => setOpen(false));
|
||||
})();
|
||||
|
||||
const SAMPLE_RATE_OUT = 44100; // server <-> client (matches the TTS rate)
|
||||
const SAMPLE_RATE_IN = 44100;
|
||||
const FRAME_MS = 20;
|
||||
@@ -347,6 +450,11 @@ const cueEls = {
|
||||
};
|
||||
const cueCounts = { filler: 0, start_filler: 0, backchannel: 0 };
|
||||
const cueTimers = { filler: 0, start_filler: 0, backchannel: 0 };
|
||||
// Barge-in indicator (header pill): flashes red on each interruption.
|
||||
const bargeinInd = $('bargeinInd');
|
||||
const bargeinCountEl = $('bargeinCount');
|
||||
let bargeinTotal = 0;
|
||||
let bargeinTimer = 0;
|
||||
|
||||
let ws = null;
|
||||
let audioCtx = null;
|
||||
@@ -498,6 +606,61 @@ function endAgentMessage() {
|
||||
agentText = '';
|
||||
}
|
||||
|
||||
// --- User-message karaoke --------------------------------------------------
|
||||
// The user's transcript reaches the client only once the utterance has ended
|
||||
// (the STT runs server-side on the whole segment), so there is no per-word
|
||||
// audio timing to follow. The words are instead lit up by a short left-to-right
|
||||
// sweep, giving the same karaoke feel as the agent's reply.
|
||||
const USER_REVEAL_MS_PER_WORD = 90;
|
||||
let userRevealRAF = 0;
|
||||
let userRevealWords = null;
|
||||
|
||||
function addUserMessage(text) {
|
||||
const bubble = addMessage('user', '');
|
||||
bubble.classList.add('karaoke');
|
||||
const words = [];
|
||||
for (const tok of text.split(/(\s+)/)) {
|
||||
if (tok === '') continue;
|
||||
if (/^\s+$/.test(tok)) {
|
||||
bubble.appendChild(document.createTextNode(tok));
|
||||
} else {
|
||||
const span = document.createElement('span');
|
||||
span.className = 'kw';
|
||||
span.textContent = tok;
|
||||
bubble.appendChild(span);
|
||||
words.push(span);
|
||||
}
|
||||
}
|
||||
revealWords(words);
|
||||
return bubble;
|
||||
}
|
||||
|
||||
function revealWords(words) {
|
||||
if (userRevealRAF) { cancelAnimationFrame(userRevealRAF); userRevealRAF = 0; }
|
||||
if (userRevealWords) {
|
||||
for (const w of userRevealWords) { w.classList.add('done'); w.classList.remove('now'); }
|
||||
userRevealWords = null;
|
||||
}
|
||||
if (!words.length) return;
|
||||
const start = performance.now();
|
||||
userRevealWords = words;
|
||||
const tick = (now) => {
|
||||
const p = Math.floor((now - start) / USER_REVEAL_MS_PER_WORD) + 1;
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
words[i].classList.toggle('done', i < p - 1);
|
||||
words[i].classList.toggle('now', i === p - 1);
|
||||
}
|
||||
if (p <= words.length) {
|
||||
userRevealRAF = requestAnimationFrame(tick);
|
||||
} else {
|
||||
for (const w of words) { w.classList.add('done'); w.classList.remove('now'); }
|
||||
userRevealWords = null;
|
||||
userRevealRAF = 0;
|
||||
}
|
||||
};
|
||||
userRevealRAF = requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
// --- Audio worklets --------------------------------------------------------
|
||||
|
||||
const CAPTURE_PROCESSOR = `
|
||||
@@ -680,6 +843,25 @@ function resetCues() {
|
||||
}
|
||||
}
|
||||
|
||||
// Barge-in indicator: bump the counter and flash the pill. Fired by the
|
||||
// server "barge_in" event (VAD onset) and by the local Stop button.
|
||||
function flashBargeIn(source) {
|
||||
bargeinTotal += 1;
|
||||
bargeinCountEl.textContent = bargeinTotal;
|
||||
if (source) bargeinInd.title = 'Last barge-in: ' + source;
|
||||
bargeinInd.classList.add('active');
|
||||
clearTimeout(bargeinTimer);
|
||||
bargeinTimer = setTimeout(() => bargeinInd.classList.remove('active'), 700);
|
||||
}
|
||||
|
||||
function resetBargeIn() {
|
||||
bargeinTotal = 0;
|
||||
clearTimeout(bargeinTimer);
|
||||
bargeinInd.classList.remove('active');
|
||||
bargeinInd.title = 'Agent interrupted by the user (VAD barge-in or the Stop button)';
|
||||
bargeinCountEl.textContent = '0';
|
||||
}
|
||||
|
||||
// --- Cue methods panel -----------------------------------------------------
|
||||
// The method (how each cue decides to fire) plus the live parameters reported
|
||||
// by the server in the "cues" event at session start. `params` turns the
|
||||
@@ -784,7 +966,7 @@ function handleServerText(text) {
|
||||
switch (event.type) {
|
||||
case 'user':
|
||||
endAgentMessage(); // a new turn starts
|
||||
addMessage('user', event.text);
|
||||
addUserMessage(event.text);
|
||||
break;
|
||||
case 'assistant_delta':
|
||||
appendAgentDelta(event.text);
|
||||
@@ -792,6 +974,12 @@ function handleServerText(text) {
|
||||
case 'assistant_end':
|
||||
agentEnded = true; // keep the bubble: karaoke finishes when audio drains
|
||||
break;
|
||||
case 'barge_in':
|
||||
// The server cut the reply (VAD onset): flash the indicator and stop the
|
||||
// karaoke sweep — the audio is gone, so the reply is left as spoken.
|
||||
flashBargeIn(event.source || 'vad');
|
||||
endAgentMessage();
|
||||
break;
|
||||
case 'error':
|
||||
endAgentMessage();
|
||||
addMessage('error', event.text, 'error');
|
||||
@@ -883,6 +1071,7 @@ async function connect() {
|
||||
setConn('on', 'connected');
|
||||
log('connected');
|
||||
resetCues();
|
||||
resetBargeIn();
|
||||
btnConnect.disabled = true;
|
||||
btnInterrupt.disabled = false;
|
||||
textInput.disabled = false;
|
||||
@@ -936,6 +1125,7 @@ btnConnect.onclick = () => (ws ? disconnect() : connect());
|
||||
btnInterrupt.onclick = () => {
|
||||
if (sendJson({ type: 'interrupt' })) {
|
||||
log('→ interrupt');
|
||||
flashBargeIn('client');
|
||||
endAgentMessage();
|
||||
}
|
||||
};
|
||||
@@ -966,7 +1156,7 @@ composer.onsubmit = (e) => {
|
||||
if (!text) return;
|
||||
if (sendJson({ type: 'user_text', text })) {
|
||||
endAgentMessage();
|
||||
addMessage('user', text);
|
||||
addUserMessage(text);
|
||||
textInput.value = '';
|
||||
log('→ user_text');
|
||||
}
|
||||
|
||||
@@ -7,3 +7,7 @@ pub const PLAYGROUND_HTML: &str = include_str!("playground.html");
|
||||
/// The `/mivocero` speech-to-speech chat client, embedded into the binary at
|
||||
/// compile time.
|
||||
pub const MIVOCERO_HTML: &str = include_str!("mivocero.html");
|
||||
|
||||
/// The `/customvoice` voice-cloning builder page, embedded into the binary at
|
||||
/// compile time.
|
||||
pub const CUSTOMVOICE_HTML: &str = include_str!("customvoice.html");
|
||||
|
||||
+43
-4
@@ -235,9 +235,13 @@ pub async fn run_session(socket: WebSocket, state: AppState) {
|
||||
for event in detector.push(&frame) {
|
||||
match event {
|
||||
VadEvent::SpeechOnset { prob } => {
|
||||
// The user started talking: cut the agent off unless
|
||||
// it is already silent.
|
||||
if settings.barge_in && tm_for_vad.is_speaking() {
|
||||
// The user started talking: cut the agent off — but
|
||||
// only once the agent is actually *audible*. While
|
||||
// the reply is still being synthesised or is sitting
|
||||
// in the pre-buffer nothing has reached the user's
|
||||
// ears yet, so a transient (or the tail of the
|
||||
// user's own utterance) must not cancel it.
|
||||
if settings.barge_in && tm_for_vad.is_audible() {
|
||||
info!(
|
||||
prob = format!("{prob:.3}"),
|
||||
"VAD barge-in — interrupting agent"
|
||||
@@ -246,6 +250,13 @@ pub async fn run_session(socket: WebSocket, state: AppState) {
|
||||
"barge_in",
|
||||
json!({ "source": "vad", "prob": prob }),
|
||||
);
|
||||
// Tell the client so it can flash a barge-in
|
||||
// indicator (the VAD onset is server-side, so
|
||||
// the browser otherwise has no way to know).
|
||||
let _ = out_tx_for_mic.send(
|
||||
json!({ "type": "barge_in", "source": "vad", "prob": prob })
|
||||
.to_string(),
|
||||
);
|
||||
tm_for_vad.interrupt();
|
||||
}
|
||||
}
|
||||
@@ -268,7 +279,35 @@ pub async fn run_session(socket: WebSocket, state: AppState) {
|
||||
json!({ "duration_ms": duration_ms, "avg_prob": avg_prob }),
|
||||
);
|
||||
backchannel.on_speech_end();
|
||||
if settings.segment_stt {
|
||||
// Utterance gate (see `VadSettings::min_utterance_ms`
|
||||
// / `min_avg_prob`): drop segments too short or too
|
||||
// soft to be real speech before spending an STT+LLM
|
||||
// turn on them. This is what filters the VAD
|
||||
// false-triggers (breaths, clicks, acoustic echo,
|
||||
// noise) that the STT would otherwise turn into
|
||||
// hallucinated transcripts ("Bye.", "She", empty).
|
||||
let too_short = settings.min_utterance_ms > 0
|
||||
&& duration_ms < settings.min_utterance_ms as u64;
|
||||
let too_soft =
|
||||
settings.min_avg_prob > 0.0 && avg_prob < settings.min_avg_prob;
|
||||
if too_short || too_soft {
|
||||
info!(
|
||||
duration_ms,
|
||||
avg_prob = format!("{avg_prob:.3}"),
|
||||
too_short,
|
||||
too_soft,
|
||||
"utterance dropped (below STT gate)"
|
||||
);
|
||||
audit_for_mic.event(
|
||||
"utterance_dropped",
|
||||
json!({
|
||||
"duration_ms": duration_ms,
|
||||
"avg_prob": avg_prob,
|
||||
"too_short": too_short,
|
||||
"too_soft": too_soft,
|
||||
}),
|
||||
);
|
||||
} else if settings.segment_stt {
|
||||
match &worker_for_mic {
|
||||
Some(worker) => {
|
||||
info!(
|
||||
|
||||
Reference in New Issue
Block a user