roomote_PRE/piperust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

step2

Browse Source
This commit is contained in:
valenti
2026-10-17 15:49:20 +01:00
parent 430997bd2e
commit bfed42c3b3
124 changed files with 5646 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lang.rs
+433
View File
@@ -0,0 +1,433 @@
//! Multilingual clip banks: language detection and per-language phrase tables.
//!
//! The shipped clip banks (backchannel, end-of-turn filler and start filler,
//! built in [`crate::assets`]) are pre-synthesized in a single language — the
//! one configured by `LANG_DEFAULT` (Spanish by default), which lives in the
//! base directories (`BACKCHANNEL_DIR`, `FILLERS_DIR`, `START_FILLER_DIR`).
//! When the user is detected speaking a different language, the banks for that
//! language are generated on demand into a per-language subdirectory
//! (`<base>/<lang>/`, ISO 639-1) and the session switches to them. Every
//! language has its own directory, so regenerating one never removes the clips
//! of another.
//!
//! Detection is a dedicated LLM call (`LANG_DETECT_ENABLED`) returning the ISO
//! 639-1 code of the transcript. The phrase lists come from the built-in tables
//! below for the common languages, and from the LLM
//! (`LANG_LLM_FALLBACK_ENABLED`) for any other language.
use crate::llm::LlmClient;
use crate::orchestrator::FillerBank;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
/// Maximum number of phrases accepted from an LLM-generated list.
const LLM_TEXT_LIMIT: usize = 12;
/// Backchannel phrases per language, as `(category, phrase)` pairs.
///
/// The category drives the progression through a turn (an acknowledgment first,
/// agreement later — see [`crate::orchestrator::backchannel`]); the Drama-3
/// style tag steers the delivery and is passed straight to the TTS. Spanish
/// matches [`DEFAULT_BACKCHANNEL_TEXTS`](crate::orchestrator::backchannel::DEFAULT_BACKCHANNEL_TEXTS),
/// the language of the shipped clip bank.
pub const BACKCHANNEL_TEXTS: &[(&str, &[(&str, &str)])] = &[
(
"es",
&[
("ack", "[calm] Mm-hmm..."),
("ack", "[attentive] Ajá..."),
("agree", "[agreeing] Claro..."),
("ack", "[listening] Ya veo..."),
("ack", "[attentive] Sí, sí..."),
("ack", "[calm] Entiendo..."),
("agree", "[relaxed] Vale..."),
("ack", "[thoughtful] Ah, ya..."),
],
),
(
"en",
&[
("ack", "[calm] Mm-hmm..."),
("ack", "[attentive] Uh-huh..."),
("agree", "[agreeing] Right..."),
("ack", "[listening] I see..."),
("ack", "[attentive] Yeah, yeah..."),
("ack", "[calm] Got it..."),
("agree", "[relaxed] Okay..."),
("ack", "[thoughtful] Oh, I see..."),
],
),
(
"it",
&[
("ack", "[calm] Mmm..."),
("ack", "[attentive] Eh già..."),
("agree", "[agreeing] Certo..."),
("ack", "[listening] Capisco..."),
("ack", "[attentive] Sì, sì..."),
("ack", "[calm] Ho capito..."),
("agree", "[relaxed] Va bene..."),
("ack", "[thoughtful] Ah, ecco..."),
],
),
(
"fr",
&[
("ack", "[calm] Mm-hmm..."),
("ack", "[attentive] Hum, oui..."),
("agree", "[agreeing] Bien sûr..."),
("ack", "[listening] Je vois..."),
("ack", "[attentive] Oui, oui..."),
("ack", "[calm] D'accord..."),
("agree", "[relaxed] Très bien..."),
("ack", "[thoughtful] Ah, je vois..."),
],
),
(
"de",
&[
("ack", "[calm] Mhm..."),
("ack", "[attentive] Aha..."),
("agree", "[agreeing] Genau..."),
("ack", "[listening] Ich verstehe..."),
("ack", "[attentive] Ja, ja..."),
("ack", "[calm] Verstehe..."),
("agree", "[relaxed] In Ordnung..."),
("ack", "[thoughtful] Ach so..."),
],
),
(
"pt",
&[
("ack", "[calm] Mm-hmm..."),
("ack", "[attentive] Aham..."),
("agree", "[agreeing] Claro..."),
("ack", "[listening] Entendo..."),
("ack", "[attentive] Sim, sim..."),
("ack", "[calm] Entendi..."),
("agree", "[relaxed] Tá bom..."),
("ack", "[thoughtful] Ah, entendi..."),
],
),
];
/// End-of-turn filler interjections per language, played round-robin. Spanish
/// matches [`FILLER_CLIPS`](crate::assets::FILLER_CLIPS), the language of the
/// shipped filler bank.
pub const FILLER_TEXTS: &[(&str, &[&str])] = &[
("es", &["Uhm...", "Eh...", "Mmm...", "A ver...", "Bueno...", "Pues...", "Vaya..."]),
("en", &["Uhm...", "Hmm...", "Mmm...", "Well...", "Uh, let's see...", "Okay...", "Right..."]),
("it", &["Uhm...", "Eh...", "Mmm...", "Vediamo...", "Allora...", "Beh...", "Dunque..."]),
("fr", &["Hum...", "Euh...", "Mmm...", "Voyons...", "Alors...", "Ben...", "Bref..."]),
("de", &["Ähm...", "Hm...", "Mmm...", "Mal sehen...", "Also...", "Naja...", "Tja..."]),
("pt", &["Hã...", "É...", "Mmm...", "Vejamos...", "Então...", "Bom...", "Pois..."]),
];
/// Start-filler phrases per language: the turn-start acknowledgment played when
/// the reply audio is late (see
/// [`TurnManager::schedule_start_filler`](crate::orchestrator::turn_manager::TurnManager)).
/// Each phrase carries a Drama-3 style tag. Spanish matches the shipped default
/// `START_FILLER_TEXTS`, the language of the base clip bank.
pub const START_FILLER_TEXTS: &[(&str, &[&str])] = &[
(
"es",
&["[calm] Un momento...", "[thoughtful] A ver, veamos...", "[sigh] Vale, ya voy..."],
),
(
"en",
&["[calm] One moment...", "[thoughtful] Let me see...", "[sigh] Okay, coming up..."],
),
(
"it",
&["[calm] Un momento...", "[thoughtful] Vediamo un po'...", "[sigh] Va bene, arrivo..."],
),
(
"fr",
&["[calm] Un instant...", "[thoughtful] Voyons voir...", "[sigh] Bon, voilà..."],
),
(
"de",
&["[calm] Einen Moment...", "[thoughtful] Mal sehen...", "[sigh] Okay, gleich..."],
),
(
"pt",
&["[calm] Um momento...", "[thoughtful] Vejamos...", "[sigh] Tá bom, já vai..."],
),
];
/// Backchannel phrases of `lang` as `(category, phrase)` pairs, or `None` when
/// the language is not in the built-in table.
pub fn backchannel_texts(lang: &str) -> Option<Vec<(String, String)>> {
BACKCHANNEL_TEXTS
.iter()
.find(|(l, _)| *l == lang)
.map(|(_, list)| {
list.iter()
.map(|(c, p)| (c.to_string(), p.to_string()))
.collect()
})
}
/// End-of-turn filler phrases of `lang`, or `None` when the language is not in
/// the built-in table.
pub fn filler_texts(lang: &str) -> Option<Vec<String>> {
FILLER_TEXTS
.iter()
.find(|(l, _)| *l == lang)
.map(|(_, list)| list.iter().map(|s| s.to_string()).collect())
}
/// Start-filler phrases of `lang`, or `None` when the language is not in the
/// built-in table.
pub fn start_filler_texts(lang: &str) -> Option<Vec<String>> {
START_FILLER_TEXTS
.iter()
.find(|(l, _)| *l == lang)
.map(|(_, list)| list.iter().map(|s| s.to_string()).collect())
}
/// The clip banks of one language, published together so the filler path
/// (orchestrator) and the backchannel path (mic task) always agree on which
/// language is active.
#[derive(Clone)]
pub struct LangBanks {
/// ISO 639-1 code of this bank.
pub lang: String,
/// Backchannel clips for this language.
pub backchannel: Arc<FillerBank>,
/// Spoken phrases of [`Self::backchannel`], in clip order.
pub backchannel_texts: Vec<String>,
/// Category of each phrase in [`Self::backchannel_texts`] (parallel array).
pub backchannel_categories: Vec<String>,
/// End-of-turn filler clips for this language.
pub filler: Arc<FillerBank>,
/// Start-filler clips for this language.
pub start_filler: Arc<FillerBank>,
}
/// Per-session language state, shared between the turn manager (end-of-turn and
/// start filler) and the microphone task (backchannel).
pub struct LanguageState {
/// The banks of the configured default language (`LANG_DEFAULT`), i.e. the
/// base directories. Kept so switching back needs no regeneration.
base: Arc<LangBanks>,
/// Currently active banks.
current: parking_lot::RwLock<Arc<LangBanks>>,
/// Bumped on every switch so the mic task can notice a change with a single
/// atomic load per audio frame.
generation: AtomicU64,
}
impl LanguageState {
pub fn new(
lang: String,
backchannel: Arc<FillerBank>,
backchannel_texts: Vec<String>,
backchannel_categories: Vec<String>,
filler: Arc<FillerBank>,
start_filler: Arc<FillerBank>,
) -> Self {
let base = Arc::new(LangBanks {
lang,
backchannel,
backchannel_texts,
backchannel_categories,
filler,
start_filler,
});
Self {
current: parking_lot::RwLock::new(base.clone()),
base,
generation: AtomicU64::new(0),
}
}
/// The banks currently in use.
pub fn current(&self) -> Arc<LangBanks> {
self.current.read().clone()
}
/// The banks of the default language (base directories).
pub fn base(&self) -> Arc<LangBanks> {
self.base.clone()
}
/// Current generation; changes whenever the active language changes.
pub fn generation(&self) -> u64 {
self.generation.load(Ordering::Acquire)
}
/// Publish a new set of banks and bump the generation.
pub fn set(&self, banks: LangBanks) {
*self.current.write() = Arc::new(banks);
self.generation.fetch_add(1, Ordering::AcqRel);
}
}
/// System prompt of the language-identification call. Written to yield *only*
/// the ISO code, so the reply can be parsed without a JSON or function-call
/// round trip (this deployment runs no server-side tool parser).
const DETECT_SYSTEM_PROMPT: &str = "You are a language identification tool. Read the user's \
text and reply with only the ISO 639-1 two-letter code of the language it is written in \
(for example: en, es, it, fr, de, pt). No punctuation, no quotes, no explanation. Reply \
with und if you cannot tell.";
/// Ask the LLM for the ISO 639-1 code of `sample`.
///
/// Best-effort: `None` on any request failure or when the reply does not carry
/// a plausible code, in which case the currently active language simply stays.
pub async fn detect_language(llm: &LlmClient, sample: &str) -> Option<String> {
let cancel = CancellationToken::new();
let reply = match llm
.simple_chat(DETECT_SYSTEM_PROMPT, sample, 8, 0.0, &cancel)
.await
{
Ok(r) => r,
Err(e) => {
warn!(error = %e, "language detection call failed");
return None;
}
};
parse_lang_code(&reply)
}
/// The first run of ASCII letters in `raw`, lowercased, kept only when it looks
/// like an ISO 639-1 code (`"es."` → `"es"`, `"Spanish"` → `None`,
/// `"und"` → `None`).
fn parse_lang_code(raw: &str) -> Option<String> {
let mut run = String::new();
let mut started = false;
for c in raw.trim().chars() {
if c.is_ascii_alphabetic() {
started = true;
run.push(c.to_ascii_lowercase());
} else if started {
break;
}
}
match run.len() {
2 | 3 if run != "und" => Some(run),
_ => None,
}
}
const BACKCHANNEL_GEN_PROMPT: &str = "You write backchannel cues: the very short \
acknowledgments a listener says while someone else is speaking, like \"mm-hmm\", \
\"I see\" or \"right\". Write eight different cues in the language with ISO 639-1 code \
{lang}. Each cue is at most three words, ends with an ellipsis and is never a question. \
Reply with one cue per line and nothing else: no numbering, no quotes, no explanations.";
const FILLER_GEN_PROMPT: &str = "You write hesitation fillers: the short sounds a speaker \
makes to buy a moment while thinking, like \"uhm\", \"well\" or \"let me see\". Write \
seven different fillers in the language with ISO 639-1 code {lang}. Each filler is one \
or two words ending with an ellipsis. Reply with one filler per line and nothing else: \
no numbering, no quotes, no explanations.";
const START_FILLER_GEN_PROMPT: &str = "You write turn-start acknowledgments: the very short \
phrases a speaker says to buy a moment right before answering, like \"one moment\", \
\"let me see\" or \"okay, coming up\". Write three different phrases in the language \
with ISO 639-1 code {lang}. Each phrase is two to four words and ends with an ellipsis. \
Reply with one phrase per line and nothing else: no numbering, no quotes, no \
explanations.";
/// Generate a backchannel phrase list with the LLM (fallback for languages
/// without a built-in table). `None` when the call fails or yields nothing
/// usable.
pub async fn generate_backchannel_texts(llm: &LlmClient, lang: &str) -> Option<Vec<String>> {
let prompt = BACKCHANNEL_GEN_PROMPT.replace("{lang}", lang);
generate_texts(llm, &prompt, "backchannel").await
}
/// Generate an end-of-turn filler phrase list with the LLM.
pub async fn generate_filler_texts(llm: &LlmClient, lang: &str) -> Option<Vec<String>> {
let prompt = FILLER_GEN_PROMPT.replace("{lang}", lang);
generate_texts(llm, &prompt, "filler").await
}
/// Generate a start-filler phrase list with the LLM (fallback for languages
/// without a built-in table).
pub async fn generate_start_filler_texts(llm: &LlmClient, lang: &str) -> Option<Vec<String>> {
let prompt = START_FILLER_GEN_PROMPT.replace("{lang}", lang);
generate_texts(llm, &prompt, "start-filler").await
}
async fn generate_texts(llm: &LlmClient, prompt: &str, kind: &str) -> Option<Vec<String>> {
let cancel = CancellationToken::new();
let reply = match llm
.simple_chat(prompt, "Generate the list now.", 256, 0.8, &cancel)
.await
{
Ok(r) => r,
Err(e) => {
warn!(kind, error = %e, "phrase-list generation failed");
return None;
}
};
let texts = parse_lines(&reply, LLM_TEXT_LIMIT);
if texts.len() < 2 {
warn!(kind, lines = texts.len(), "phrase-list generation returned too few usable lines");
return None;
}
info!(kind, phrases = texts.len(), "generated phrase list via LLM");
Some(texts)
}
/// Split an LLM list reply into clean phrases: one per line, bullets/numbering
/// and surrounding quotes removed, over-long lines dropped.
fn parse_lines(raw: &str, max: usize) -> Vec<String> {
raw.lines()
.map(|l| {
l.trim()
.trim_start_matches(|c: char| {
c.is_ascii_digit() || matches!(c, '-' | '*' | '.' | ')' | ':' | ' ')
})
.trim_matches(|c| matches!(c, '"' | '\'' | '`'))
.trim()
})
.filter(|l| !l.is_empty() && l.len() <= 60)
.take(max)
.map(|l| l.to_string())
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_iso_code_from_various_replies() {
assert_eq!(parse_lang_code("es"), Some("es".to_string()));
assert_eq!(parse_lang_code("EN."), Some("en".to_string()));
assert_eq!(parse_lang_code(" it\n"), Some("it".to_string()));
assert_eq!(parse_lang_code("und"), None);
assert_eq!(parse_lang_code("Spanish"), None);
assert_eq!(parse_lang_code(""), None);
}
#[test]
fn strips_bullets_and_numbering_from_generated_lists() {
let raw = "1. Mmm...\n- Certo...\n\"Va bene...\"\n\n* Capisco...";
assert_eq!(
parse_lines(raw, 10),
vec![
"Mmm...".to_string(),
"Certo...".to_string(),
"Va bene...".to_string(),
"Capisco...".to_string(),
]
);
}
#[test]
fn builtin_tables_cover_the_base_languages() {
for lang in ["es", "en", "it", "fr", "de", "pt"] {
assert!(backchannel_texts(lang).is_some(), "missing backchannel table for {lang}");
assert!(filler_texts(lang).is_some(), "missing filler table for {lang}");
assert!(start_filler_texts(lang).is_some(), "missing start-filler table for {lang}");
}
assert!(backchannel_texts("xx").is_none());
assert!(start_filler_texts("xx").is_none());
}
}