//! Custom-voice builder: turns a recording (or an uploaded audio file) into a //! Fish Audio voice-cloning reference. //! //! The `/customvoice` page records the user reading a phrase (at least 10 s //! long) **or** lets them upload an audio file and type the matching //! transcript. The bytes are POSTed here, transcoded to a mono PCM WAV in the //! directory that holds the configured reference — the same directory the Fish //! Audio backend already reads, so the new file is immediately usable — the //! transcript is written as a sibling `.txt` (the convention //! [`crate::config::Config::tts_reference`] falls back to), and the //! running TTS engine is pointed at the new file via //! [`crate::tts::TtsEngine::set_reference`]. //! //! Before installation the take goes through a **post-processing chain** //! (see [`postprocess`]): high-pass to drop rumble/DC, optional FFT denoise and //! de-essing, leading/trailing silence trim, and a **two-pass EBU R128 //! loudness normalization** (single linear gain, so takes never jump in //! volume). Any step failing falls back to the plain transcode, so a take is //! never lost to a filter quirk. //! //! The predefined phrases live server-side so a plain recording always carries //! a transcript matching the text the page asked the user to read; a //! caller-supplied transcript (the upload flow) overrides it. use crate::config::{Config, OUTPUT_SAMPLE_RATE}; use crate::ws::AppState; use axum::body::Bytes; use axum::extract::{Query, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Json; use serde::Deserialize; use serde_json::{json, Value}; use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::process::{Output, Stdio}; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::process::Command; use tracing::{error, info, warn}; /// A predefined reading phrase, keyed by ISO 639-1 code. Long enough /// (~10 s at a natural pace) to give the cloning model a usable sample. struct Phrase { code: &'static str, label: &'static str, phrase: &'static str, } /// Reading phrases for the languages supported by [`crate::lang`]. const PHRASES: &[Phrase] = &[ Phrase { code: "es", label: "Español", phrase: "Hola, esta es mi voz personal. Estoy leyendo esta frase para crear una voz \ personalizada para mí. Hablo de forma natural y clara, para que el sistema pueda \ aprender el timbre y el ritmo de mi voz.", }, Phrase { code: "en", label: "English", phrase: "Hello, this is my personal voice. I am reading this sentence to create a custom \ voice for me. I speak in a natural and clear way, so the system can learn the \ tone and the rhythm of my voice.", }, Phrase { code: "it", label: "Italiano", phrase: "Ciao, questa è la mia voce personale. Sto leggendo questa frase per creare una \ voce su misura per me. Parlo in modo naturale e chiaro, così il sistema può \ imparare il timbro e il ritmo della mia voce.", }, Phrase { code: "fr", label: "Français", phrase: "Bonjour, c'est ma voix personnelle. Je lis cette phrase pour créer une voix \ personnalisée pour moi. Je parle de manière naturelle et claire, afin que le \ système puisse apprendre le timbre et le rythme de ma voix.", }, Phrase { code: "de", label: "Deutsch", phrase: "Hallo, das ist meine persönliche Stimme. Ich lese diesen Satz, um eine eigene \ Stimme für mich zu erstellen. Ich spreche natürlich und deutlich, damit das \ System die Klangfarbe und den Rhythmus meiner Stimme lernen kann.", }, Phrase { code: "pt", label: "Português", phrase: "Olá, esta é a minha voz pessoal. Estou a ler esta frase para criar uma voz \ personalizada para mim. Falo de forma natural e clara, para que o sistema possa \ aprender o timbre e o ritmo da minha voz.", }, ]; /// `GET /api/customvoice/phrases` — the reading phrases and the configured /// default language, used to populate the page's language selector. #[utoipa::path( get, path = "/api/customvoice/phrases", tag = "customvoice", responses(( status = 200, description = "The reading phrases (one per supported language) and the configured default language", body = crate::apidoc::PhrasesResponse, )), )] pub async fn phrases(State(state): State) -> Json { let languages: Vec = PHRASES .iter() .map(|p| json!({ "code": p.code, "label": p.label, "phrase": p.phrase })) .collect(); Json(json!({ "default": state.config.lang_default, "languages": languages, })) } /// Query string of `POST /api/customvoice`. #[derive(Deserialize, utoipa::IntoParams)] #[into_params(parameter_in = Query)] pub struct SetParams { /// ISO 639-1 code of the language the user recorded in. pub lang: String, /// Optional transcript override: the text actually spoken in the audio. /// Used by the upload flow, where the user types (or pastes) it. When /// absent or blank, the language's predefined reading phrase is used. pub text: Option, } /// `POST /api/customvoice?lang=[&text=]` — body is the raw /// audio (a browser recording or an uploaded file; whatever container the /// client sent, ffmpeg probes the content). Transcodes it to a WAV reference, /// persists the transcript, and installs the voice for the running session. #[utoipa::path( post, path = "/api/customvoice", tag = "customvoice", params(SetParams), request_body( content = String, content_type = "application/octet-stream", description = "Raw audio bytes: a browser recording (Opus/WebM) or an uploaded file (wav, mp3, m4a, ogg, flac…). ffmpeg probes the container.", ), responses( (status = 200, description = "Reference transcoded, persisted and installed as the active voice", body = crate::apidoc::SetVoiceResponse), (status = 400, description = "Unsupported language or empty body", body = crate::apidoc::ErrorResponse), (status = 500, description = "Could not create the directory or store the upload", body = crate::apidoc::ErrorResponse), (status = 502, description = "ffmpeg transcode failed", body = crate::apidoc::ErrorResponse), ), )] pub async fn set_custom_voice( State(state): State, Query(params): Query, body: Bytes, ) -> Response { let lang = params.lang.trim().to_ascii_lowercase(); let Some(entry) = PHRASES.iter().find(|p| p.code == lang) else { return error_response(StatusCode::BAD_REQUEST, "unsupported language"); }; if body.is_empty() { return error_response(StatusCode::BAD_REQUEST, "empty recording"); } // Transcript attached to the reference: the caller-supplied text when it is // non-blank (the upload flow lets the user type it), otherwise the // language's predefined reading phrase. let transcript = params .text .as_deref() .map(str::trim) .filter(|t| !t.is_empty()) .unwrap_or(entry.phrase) .to_string(); // Save next to the configured reference: that directory is already readable // by the Fish Audio backend, so no extra path plumbing is needed. let dir = reference_dir(&state.config); if let Err(e) = tokio::fs::create_dir_all(&dir).await { error!(error = %e, dir = %dir.display(), "failed to create reference directory"); return error_response( StatusCode::INTERNAL_SERVER_ERROR, "cannot create reference directory", ); } let stamp = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis()) .unwrap_or(0); let base = format!("custom_{}_{}", entry.code, stamp); let raw_path = dir.join(format!("{base}.upload")); let wav_path = dir.join(format!("{base}.wav")); let txt_path = dir.join(format!("{base}.txt")); if let Err(e) = tokio::fs::write(&raw_path, &body).await { error!(error = %e, path = %raw_path.display(), "failed to store upload"); return error_response(StatusCode::INTERNAL_SERVER_ERROR, "cannot store upload"); } match render_reference(&state.config, &raw_path, &wav_path).await { Ok(()) => {} Err(msg) => { let _ = tokio::fs::remove_file(&raw_path).await; error!(error = %msg, "custom-voice transcode failed"); return error_response(StatusCode::BAD_GATEWAY, &msg); } } let _ = tokio::fs::remove_file(&raw_path).await; if let Err(e) = tokio::fs::write(&txt_path, &transcript).await { // Non-fatal: the in-memory reference below still carries the transcript. warn!(error = %e, path = %txt_path.display(), "failed to persist reference transcript"); } let audio_path = wav_path.to_string_lossy().into_owned(); state .tts .set_reference(Some(audio_path.clone()), Some(transcript.clone())); info!(lang = %entry.code, audio = %audio_path, "custom voice installed"); ( StatusCode::OK, Json(json!({ "ok": true, "lang": entry.code, "audio": audio_path, "phrase": transcript, })), ) .into_response() } /// `POST /api/customvoice/clear` — drop the custom voice and restore the /// reference configured at startup (the `TTS_REFERENCE_AUDIO` voice), or no /// reference at all when none is configured. #[utoipa::path( post, path = "/api/customvoice/clear", tag = "customvoice", responses(( status = 200, description = "The configured reference is restored (or cleared when none is configured)", body = crate::apidoc::ClearResponse, )), )] pub async fn clear_custom_voice(State(state): State) -> Json { match state.config.tts_reference() { Some((audio_path, text)) => { info!(audio = %audio_path, "custom voice cleared — restored configured reference"); state.tts.set_reference(Some(audio_path), Some(text)); } None => { info!("custom voice cleared — no reference configured"); state.tts.set_reference(None, None); } } Json(json!({ "ok": true })) } /// The directory holding voice-cloning references: the parent of the /// configured reference WAV, or `voice_profiles` when nothing is configured. fn reference_dir(config: &Config) -> PathBuf { config .tts_reference_audio .as_ref() .and_then(|p| p.parent()) .filter(|p| !p.as_os_str().is_empty()) .map(PathBuf::from) .unwrap_or_else(|| PathBuf::from("voice_profiles")) } /// Produce the final WAV reference: transcode the upload and, unless disabled, /// run the post-processing chain. Any post-processing failure degrades to the /// plain transcode rather than losing the take. async fn render_reference(config: &Config, input: &Path, output: &Path) -> Result<(), String> { if config.customvoice_postprocess { match postprocess(config, input, output).await { Ok(()) => return Ok(()), Err(e) => warn!( error = %e, "custom-voice post-processing failed — storing the plain transcode" ), } } transcode_to_wav(config, input, output).await } /// Two-pass EBU R128 normalization over the cleaned signal: measure, then apply /// the measurement as a **single linear gain**. One gain for the whole take is /// what keeps the level consistent instead of pumping frame by frame — the /// usual cause of uneven volume in a cloned reference. async fn postprocess(config: &Config, input: &Path, output: &Path) -> Result<(), String> { let chain = cleanup_chain(config.customvoice_denoise); let opts = format!( "I={}:TP={}:LRA=11", config.customvoice_target_lufs, config.customvoice_true_peak ); // Pass 1 — measure. loudnorm prints its JSON summary through the logger, so // this run keeps the default log level. let measure_args: Vec = vec![ "-hide_banner".into(), "-nostdin".into(), "-nostats".into(), "-i".into(), input.into(), "-af".into(), format!("{chain},loudnorm={opts}:print_format=json").into(), "-f".into(), "null".into(), "-".into(), ]; let measured = run_ffmpeg(config, &measure_args).await?; let measured_stderr = String::from_utf8_lossy(&measured.stderr); let m = parse_loudnorm(&measured_stderr) .ok_or_else(|| "could not read the loudness measurement".to_string())?; // Pass 2 — apply the measurement as a linear gain and write the reference. let apply = format!( "{chain},loudnorm={opts}:measured_I={}:measured_TP={}:measured_LRA={}:\ measured_thresh={}:offset={}:linear=true", m.input_i, m.input_tp, m.input_lra, m.input_thresh, m.target_offset ); let args: Vec = vec![ "-hide_banner".into(), "-nostdin".into(), "-loglevel".into(), "error".into(), "-y".into(), "-i".into(), input.into(), "-af".into(), apply.into(), "-ar".into(), OUTPUT_SAMPLE_RATE.to_string().into(), "-ac".into(), "1".into(), "-c:a".into(), "pcm_s16le".into(), output.into(), ]; run_ffmpeg(config, &args).await.map(|_| ()) } /// The cleanup filters shared by both loudnorm passes: downmix to mono, then /// rumble/DC removal, optional FFT denoise and de-essing, and a /// leading/trailing silence trim (the clip is reversed to trim the tail with /// the same filter). /// /// The mono downmix comes **first** on purpose: loudnorm measures whatever /// reaches it, so measuring a stereo input and only folding it to mono at the /// output stage would land the written file ~3 dB below the target (the energy /// of two channels collapses into one). fn cleanup_chain(denoise: bool) -> String { let trim = "silenceremove=start_periods=1:start_duration=0.1:\ start_threshold=-50dB:start_silence=0.05"; let mut filters: Vec<&str> = vec!["aformat=channel_layouts=mono", "highpass=f=80"]; if denoise { filters.push("afftdn=nr=12:nf=-40"); filters.push("deesser"); } filters.extend([trim, "areverse", trim, "areverse"]); filters.join(",") } /// Transcode `input` to a mono 16-bit PCM WAV at the mixer's sample rate. async fn transcode_to_wav(config: &Config, input: &Path, output: &Path) -> Result<(), String> { let args: Vec = vec![ "-hide_banner".into(), "-nostdin".into(), "-loglevel".into(), "error".into(), "-y".into(), "-i".into(), input.into(), "-ac".into(), "1".into(), "-ar".into(), OUTPUT_SAMPLE_RATE.to_string().into(), "-c:a".into(), "pcm_s16le".into(), output.into(), ]; run_ffmpeg(config, &args).await.map(|_| ()) } /// Run ffmpeg with `args`. Output is discarded; on a non-zero exit the last /// stderr line is surfaced as the error message. async fn run_ffmpeg(config: &Config, args: &[OsString]) -> Result { let out = Command::new(&config.media_ffmpeg_bin) .args(args) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::piped()) .output() .await .map_err(|e| format!("failed to spawn ffmpeg ({}): {e}", config.media_ffmpeg_bin))?; if !out.status.success() { let err = String::from_utf8_lossy(&out.stderr); let last = err.trim().lines().last().unwrap_or("").to_string(); return Err(if last.is_empty() { format!("ffmpeg exited with {}", out.status) } else { format!("ffmpeg: {last}") }); } Ok(out) } /// The loudness figures loudnorm reports in its JSON summary. struct Loudness { input_i: f64, input_tp: f64, input_lra: f64, input_thresh: f64, target_offset: f64, } /// Extract the loudnorm JSON summary from ffmpeg's log output. fn parse_loudnorm(stderr: &str) -> Option { let start = stderr.rfind('{')?; let end = start + stderr[start..].find('}')?; let value: Value = serde_json::from_str(&stderr[start..=end]).ok()?; let num = |key: &str| value.get(key)?.as_str()?.trim().parse::().ok(); Some(Loudness { input_i: num("input_i")?, input_tp: num("input_tp")?, input_lra: num("input_lra")?, input_thresh: num("input_thresh")?, target_offset: num("target_offset")?, }) } /// Build a JSON error response. fn error_response(status: StatusCode, msg: &str) -> Response { (status, Json(json!({ "ok": false, "error": msg }))).into_response() }