roomote_PRE/piperust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

step3

Browse Source
This commit is contained in:
valenti
2026-10-17 22:01:31 +01:00
parent bfed42c3b3
commit 79f927d968
53 changed files with 2158 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/customvoice.rs
+397
View File
@@ -0,0 +1,397 @@
//! Custom-voice builder: turns a recording (or an uploaded audio file) into a
//! Fish Audio voice-cloning reference.
//!
//! The `/customvoice` page records the user reading a phrase (at least 10 s
//! long) **or** lets them upload an audio file and type the matching
//! transcript. The bytes are POSTed here, transcoded to a mono PCM WAV in the
//! directory that holds the configured reference — the same directory the Fish
//! Audio backend already reads, so the new file is immediately usable — the
//! transcript is written as a sibling `.txt` (the convention
//! [`crate::config::Config::tts_reference`] falls back to), and the
//! running TTS engine is pointed at the new file via
//! [`crate::tts::TtsEngine::set_reference`].
//!
//! Before installation the take goes through a **post-processing chain**
//! (see [`postprocess`]): high-pass to drop rumble/DC, optional FFT denoise and
//! de-essing, leading/trailing silence trim, and a **two-pass EBU R128
//! loudness normalization** (single linear gain, so takes never jump in
//! volume). Any step failing falls back to the plain transcode, so a take is
//! never lost to a filter quirk.
//!
//! The predefined phrases live server-side so a plain recording always carries
//! a transcript matching the text the page asked the user to read; a
//! caller-supplied transcript (the upload flow) overrides it.
use crate::config::{Config, OUTPUT_SAMPLE_RATE};
use crate::ws::AppState;
use axum::body::Bytes;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::Json;
use serde::Deserialize;
use serde_json::{json, Value};
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::{Output, Stdio};
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::process::Command;
use tracing::{error, info, warn};
/// A predefined reading phrase, keyed by ISO 639-1 code. Long enough
/// (~10 s at a natural pace) to give the cloning model a usable sample.
struct Phrase {
code: &'static str,
label: &'static str,
phrase: &'static str,
}
/// Reading phrases for the languages supported by [`crate::lang`].
const PHRASES: &[Phrase] = &[
Phrase {
code: "es",
label: "Español",
phrase: "Hola, esta es mi voz personal. Estoy leyendo esta frase para crear una voz \
personalizada para mí. Hablo de forma natural y clara, para que el sistema pueda \
aprender el timbre y el ritmo de mi voz.",
},
Phrase {
code: "en",
label: "English",
phrase: "Hello, this is my personal voice. I am reading this sentence to create a custom \
voice for me. I speak in a natural and clear way, so the system can learn the \
tone and the rhythm of my voice.",
},
Phrase {
code: "it",
label: "Italiano",
phrase: "Ciao, questa è la mia voce personale. Sto leggendo questa frase per creare una \
voce su misura per me. Parlo in modo naturale e chiaro, così il sistema può \
imparare il timbro e il ritmo della mia voce.",
},
Phrase {
code: "fr",
label: "Français",
phrase: "Bonjour, c'est ma voix personnelle. Je lis cette phrase pour créer une voix \
personnalisée pour moi. Je parle de manière naturelle et claire, afin que le \
système puisse apprendre le timbre et le rythme de ma voix.",
},
Phrase {
code: "de",
label: "Deutsch",
phrase: "Hallo, das ist meine persönliche Stimme. Ich lese diesen Satz, um eine eigene \
Stimme für mich zu erstellen. Ich spreche natürlich und deutlich, damit das \
System die Klangfarbe und den Rhythmus meiner Stimme lernen kann.",
},
Phrase {
code: "pt",
label: "Português",
phrase: "Olá, esta é a minha voz pessoal. Estou a ler esta frase para criar uma voz \
personalizada para mim. Falo de forma natural e clara, para que o sistema possa \
aprender o timbre e o ritmo da minha voz.",
},
];
/// `GET /api/customvoice/phrases` — the reading phrases and the configured
/// default language, used to populate the page's language selector.
pub async fn phrases(State(state): State<AppState>) -> Json<Value> {
let languages: Vec<Value> = PHRASES
.iter()
.map(|p| json!({ "code": p.code, "label": p.label, "phrase": p.phrase }))
.collect();
Json(json!({
"default": state.config.lang_default,
"languages": languages,
}))
}
/// Query string of `POST /api/customvoice`.
#[derive(Deserialize)]
pub struct SetParams {
/// ISO 639-1 code of the language the user recorded in.
pub lang: String,
/// Optional transcript override: the text actually spoken in the audio.
/// Used by the upload flow, where the user types (or pastes) it. When
/// absent or blank, the language's predefined reading phrase is used.
pub text: Option<String>,
}
/// `POST /api/customvoice?lang=<code>[&text=<transcript>]` — body is the raw
/// audio (a browser recording or an uploaded file; whatever container the
/// client sent, ffmpeg probes the content). Transcodes it to a WAV reference,
/// persists the transcript, and installs the voice for the running session.
pub async fn set_custom_voice(
State(state): State<AppState>,
Query(params): Query<SetParams>,
body: Bytes,
) -> Response {
let lang = params.lang.trim().to_ascii_lowercase();
let Some(entry) = PHRASES.iter().find(|p| p.code == lang) else {
return error_response(StatusCode::BAD_REQUEST, "unsupported language");
};
if body.is_empty() {
return error_response(StatusCode::BAD_REQUEST, "empty recording");
}
// Transcript attached to the reference: the caller-supplied text when it is
// non-blank (the upload flow lets the user type it), otherwise the
// language's predefined reading phrase.
let transcript = params
.text
.as_deref()
.map(str::trim)
.filter(|t| !t.is_empty())
.unwrap_or(entry.phrase)
.to_string();
// Save next to the configured reference: that directory is already readable
// by the Fish Audio backend, so no extra path plumbing is needed.
let dir = reference_dir(&state.config);
if let Err(e) = tokio::fs::create_dir_all(&dir).await {
error!(error = %e, dir = %dir.display(), "failed to create reference directory");
return error_response(StatusCode::INTERNAL_SERVER_ERROR, "cannot create reference directory");
}
let stamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis())
.unwrap_or(0);
let base = format!("custom_{}_{}", entry.code, stamp);
let raw_path = dir.join(format!("{base}.upload"));
let wav_path = dir.join(format!("{base}.wav"));
let txt_path = dir.join(format!("{base}.txt"));
if let Err(e) = tokio::fs::write(&raw_path, &body).await {
error!(error = %e, path = %raw_path.display(), "failed to store upload");
return error_response(StatusCode::INTERNAL_SERVER_ERROR, "cannot store upload");
}
match render_reference(&state.config, &raw_path, &wav_path).await {
Ok(()) => {}
Err(msg) => {
let _ = tokio::fs::remove_file(&raw_path).await;
error!(error = %msg, "custom-voice transcode failed");
return error_response(StatusCode::BAD_GATEWAY, &msg);
}
}
let _ = tokio::fs::remove_file(&raw_path).await;
if let Err(e) = tokio::fs::write(&txt_path, &transcript).await {
// Non-fatal: the in-memory reference below still carries the transcript.
warn!(error = %e, path = %txt_path.display(), "failed to persist reference transcript");
}
let audio_path = wav_path.to_string_lossy().into_owned();
state
.tts
.set_reference(Some(audio_path.clone()), Some(transcript.clone()));
info!(lang = %entry.code, audio = %audio_path, "custom voice installed");
(
StatusCode::OK,
Json(json!({
"ok": true,
"lang": entry.code,
"audio": audio_path,
"phrase": transcript,
})),
)
.into_response()
}
/// `POST /api/customvoice/clear` — drop the custom voice and restore the
/// reference configured at startup (the `TTS_REFERENCE_AUDIO` voice), or no
/// reference at all when none is configured.
pub async fn clear_custom_voice(State(state): State<AppState>) -> Json<Value> {
match state.config.tts_reference() {
Some((audio_path, text)) => {
info!(audio = %audio_path, "custom voice cleared — restored configured reference");
state.tts.set_reference(Some(audio_path), Some(text));
}
None => {
info!("custom voice cleared — no reference configured");
state.tts.set_reference(None, None);
}
}
Json(json!({ "ok": true }))
}
/// The directory holding voice-cloning references: the parent of the
/// configured reference WAV, or `voice_profiles` when nothing is configured.
fn reference_dir(config: &Config) -> PathBuf {
config
.tts_reference_audio
.as_ref()
.and_then(|p| p.parent())
.filter(|p| !p.as_os_str().is_empty())
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("voice_profiles"))
}
/// Produce the final WAV reference: transcode the upload and, unless disabled,
/// run the post-processing chain. Any post-processing failure degrades to the
/// plain transcode rather than losing the take.
async fn render_reference(config: &Config, input: &Path, output: &Path) -> Result<(), String> {
if config.customvoice_postprocess {
match postprocess(config, input, output).await {
Ok(()) => return Ok(()),
Err(e) => warn!(
error = %e,
"custom-voice post-processing failed — storing the plain transcode"
),
}
}
transcode_to_wav(config, input, output).await
}
/// Two-pass EBU R128 normalization over the cleaned signal: measure, then apply
/// the measurement as a **single linear gain**. One gain for the whole take is
/// what keeps the level consistent instead of pumping frame by frame — the
/// usual cause of uneven volume in a cloned reference.
async fn postprocess(config: &Config, input: &Path, output: &Path) -> Result<(), String> {
let chain = cleanup_chain(config.customvoice_denoise);
let opts = format!(
"I={}:TP={}:LRA=11",
config.customvoice_target_lufs, config.customvoice_true_peak
);
// Pass 1 — measure. loudnorm prints its JSON summary through the logger, so
// this run keeps the default log level.
let measure_args: Vec<OsString> = vec![
"-hide_banner".into(),
"-nostdin".into(),
"-nostats".into(),
"-i".into(),
input.into(),
"-af".into(),
format!("{chain},loudnorm={opts}:print_format=json").into(),
"-f".into(),
"null".into(),
"-".into(),
];
let measured = run_ffmpeg(config, &measure_args).await?;
let measured_stderr = String::from_utf8_lossy(&measured.stderr);
let m = parse_loudnorm(&measured_stderr)
.ok_or_else(|| "could not read the loudness measurement".to_string())?;
// Pass 2 — apply the measurement as a linear gain and write the reference.
let apply = format!(
"{chain},loudnorm={opts}:measured_I={}:measured_TP={}:measured_LRA={}:\
measured_thresh={}:offset={}:linear=true",
m.input_i, m.input_tp, m.input_lra, m.input_thresh, m.target_offset
);
let args: Vec<OsString> = vec![
"-hide_banner".into(),
"-nostdin".into(),
"-loglevel".into(),
"error".into(),
"-y".into(),
"-i".into(),
input.into(),
"-af".into(),
apply.into(),
"-ar".into(),
OUTPUT_SAMPLE_RATE.to_string().into(),
"-ac".into(),
"1".into(),
"-c:a".into(),
"pcm_s16le".into(),
output.into(),
];
run_ffmpeg(config, &args).await.map(|_| ())
}
/// The cleanup filters shared by both loudnorm passes: downmix to mono, then
/// rumble/DC removal, optional FFT denoise and de-essing, and a
/// leading/trailing silence trim (the clip is reversed to trim the tail with
/// the same filter).
///
/// The mono downmix comes **first** on purpose: loudnorm measures whatever
/// reaches it, so measuring a stereo input and only folding it to mono at the
/// output stage would land the written file ~3 dB below the target (the energy
/// of two channels collapses into one).
fn cleanup_chain(denoise: bool) -> String {
let trim = "silenceremove=start_periods=1:start_duration=0.1:\
start_threshold=-50dB:start_silence=0.05";
let mut filters: Vec<&str> = vec!["aformat=channel_layouts=mono", "highpass=f=80"];
if denoise {
filters.push("afftdn=nr=12:nf=-40");
filters.push("deesser");
}
filters.extend([trim, "areverse", trim, "areverse"]);
filters.join(",")
}
/// Transcode `input` to a mono 16-bit PCM WAV at the mixer's sample rate.
async fn transcode_to_wav(config: &Config, input: &Path, output: &Path) -> Result<(), String> {
let args: Vec<OsString> = vec![
"-hide_banner".into(),
"-nostdin".into(),
"-loglevel".into(),
"error".into(),
"-y".into(),
"-i".into(),
input.into(),
"-ac".into(),
"1".into(),
"-ar".into(),
OUTPUT_SAMPLE_RATE.to_string().into(),
"-c:a".into(),
"pcm_s16le".into(),
output.into(),
];
run_ffmpeg(config, &args).await.map(|_| ())
}
/// Run ffmpeg with `args`. Output is discarded; on a non-zero exit the last
/// stderr line is surfaced as the error message.
async fn run_ffmpeg(config: &Config, args: &[OsString]) -> Result<Output, String> {
let out = Command::new(&config.media_ffmpeg_bin)
.args(args)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.output()
.await
.map_err(|e| format!("failed to spawn ffmpeg ({}): {e}", config.media_ffmpeg_bin))?;
if !out.status.success() {
let err = String::from_utf8_lossy(&out.stderr);
let last = err.trim().lines().last().unwrap_or("").to_string();
return Err(if last.is_empty() {
format!("ffmpeg exited with {}", out.status)
} else {
format!("ffmpeg: {last}")
});
}
Ok(out)
}
/// The loudness figures loudnorm reports in its JSON summary.
struct Loudness {
input_i: f64,
input_tp: f64,
input_lra: f64,
input_thresh: f64,
target_offset: f64,
}
/// Extract the loudnorm JSON summary from ffmpeg's log output.
fn parse_loudnorm(stderr: &str) -> Option<Loudness> {
let start = stderr.rfind('{')?;
let end = start + stderr[start..].find('}')?;
let value: Value = serde_json::from_str(&stderr[start..=end]).ok()?;
let num = |key: &str| value.get(key)?.as_str()?.trim().parse::<f64>().ok();
Some(Loudness {
input_i: num("input_i")?,
input_tp: num("input_tp")?,
input_lra: num("input_lra")?,
input_thresh: num("input_thresh")?,
target_offset: num("target_offset")?,
})
}
/// Build a JSON error response.
fn error_response(status: StatusCode, msg: &str) -> Response {
(status, Json(json!({ "ok": false, "error": msg }))).into_response()
}