//! Multilingual clip banks: language detection and per-language phrase tables. //! //! The shipped clip banks (backchannel, end-of-turn filler and start filler, //! built in [`crate::assets`]) are pre-synthesized in a single language — the //! one configured by `LANG_DEFAULT` (Spanish by default), which lives in the //! base directories (`BACKCHANNEL_DIR`, `FILLERS_DIR`, `START_FILLER_DIR`). //! When the user is detected speaking a different language, the banks for that //! language are generated on demand into a per-language subdirectory //! (`//`, ISO 639-1) and the session switches to them. Every //! language has its own directory, so regenerating one never removes the clips //! of another. //! //! Detection is a dedicated LLM call (`LANG_DETECT_ENABLED`) returning the ISO //! 639-1 code of the transcript. The phrase lists come from the built-in tables //! below for the common languages, and from the LLM //! (`LANG_LLM_FALLBACK_ENABLED`) for any other language. use crate::llm::LlmClient; use crate::orchestrator::FillerBank; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; /// Maximum number of phrases accepted from an LLM-generated list. const LLM_TEXT_LIMIT: usize = 12; /// Backchannel phrases per language, as `(category, phrase)` pairs. /// /// The category drives the progression through a turn (an acknowledgment first, /// agreement later — see [`crate::orchestrator::backchannel`]); the Drama-3 /// style tag steers the delivery and is passed straight to the TTS. Spanish /// matches [`DEFAULT_BACKCHANNEL_TEXTS`](crate::orchestrator::backchannel::DEFAULT_BACKCHANNEL_TEXTS), /// the language of the shipped clip bank. pub const BACKCHANNEL_TEXTS: &[(&str, &[(&str, &str)])] = &[ ( "es", &[ ("ack", "[calm] Mm-hmm..."), ("ack", "[attentive] Ajá..."), ("agree", "[agreeing] Claro..."), ("ack", "[listening] Ya veo..."), ("ack", "[attentive] Sí, sí..."), ("ack", "[calm] Entiendo..."), ("agree", "[relaxed] Vale..."), ("ack", "[thoughtful] Ah, ya..."), ], ), ( "en", &[ ("ack", "[calm] Mm-hmm..."), ("ack", "[attentive] Uh-huh..."), ("agree", "[agreeing] Right..."), ("ack", "[listening] I see..."), ("ack", "[attentive] Yeah, yeah..."), ("ack", "[calm] Got it..."), ("agree", "[relaxed] Okay..."), ("ack", "[thoughtful] Oh, I see..."), ], ), ( "it", &[ ("ack", "[calm] Mmm..."), ("ack", "[attentive] Eh già..."), ("agree", "[agreeing] Certo..."), ("ack", "[listening] Capisco..."), ("ack", "[attentive] Sì, sì..."), ("ack", "[calm] Ho capito..."), ("agree", "[relaxed] Va bene..."), ("ack", "[thoughtful] Ah, ecco..."), ], ), ( "fr", &[ ("ack", "[calm] Mm-hmm..."), ("ack", "[attentive] Hum, oui..."), ("agree", "[agreeing] Bien sûr..."), ("ack", "[listening] Je vois..."), ("ack", "[attentive] Oui, oui..."), ("ack", "[calm] D'accord..."), ("agree", "[relaxed] Très bien..."), ("ack", "[thoughtful] Ah, je vois..."), ], ), ( "de", &[ ("ack", "[calm] Mhm..."), ("ack", "[attentive] Aha..."), ("agree", "[agreeing] Genau..."), ("ack", "[listening] Ich verstehe..."), ("ack", "[attentive] Ja, ja..."), ("ack", "[calm] Verstehe..."), ("agree", "[relaxed] In Ordnung..."), ("ack", "[thoughtful] Ach so..."), ], ), ( "pt", &[ ("ack", "[calm] Mm-hmm..."), ("ack", "[attentive] Aham..."), ("agree", "[agreeing] Claro..."), ("ack", "[listening] Entendo..."), ("ack", "[attentive] Sim, sim..."), ("ack", "[calm] Entendi..."), ("agree", "[relaxed] Tá bom..."), ("ack", "[thoughtful] Ah, entendi..."), ], ), ]; /// End-of-turn filler interjections per language, played round-robin. Spanish /// matches [`FILLER_CLIPS`](crate::assets::FILLER_CLIPS), the language of the /// shipped filler bank. pub const FILLER_TEXTS: &[(&str, &[&str])] = &[ ("es", &["Uhm...", "Eh...", "Mmm...", "A ver...", "Bueno...", "Pues...", "Vaya..."]), ("en", &["Uhm...", "Hmm...", "Mmm...", "Well...", "Uh, let's see...", "Okay...", "Right..."]), ("it", &["Uhm...", "Eh...", "Mmm...", "Vediamo...", "Allora...", "Beh...", "Dunque..."]), ("fr", &["Hum...", "Euh...", "Mmm...", "Voyons...", "Alors...", "Ben...", "Bref..."]), ("de", &["Ähm...", "Hm...", "Mmm...", "Mal sehen...", "Also...", "Naja...", "Tja..."]), ("pt", &["Hã...", "É...", "Mmm...", "Vejamos...", "Então...", "Bom...", "Pois..."]), ]; /// Start-filler phrases per language: the turn-start acknowledgment played when /// the reply audio is late (see /// [`TurnManager::schedule_start_filler`](crate::orchestrator::turn_manager::TurnManager)). /// Each phrase carries a Drama-3 style tag. Spanish matches the shipped default /// `START_FILLER_TEXTS`, the language of the base clip bank. pub const START_FILLER_TEXTS: &[(&str, &[&str])] = &[ ( "es", &["[calm] Un momento...", "[thoughtful] A ver, veamos...", "[sigh] Vale, ya voy..."], ), ( "en", &["[calm] One moment...", "[thoughtful] Let me see...", "[sigh] Okay, coming up..."], ), ( "it", &["[calm] Un momento...", "[thoughtful] Vediamo un po'...", "[sigh] Va bene, arrivo..."], ), ( "fr", &["[calm] Un instant...", "[thoughtful] Voyons voir...", "[sigh] Bon, voilà..."], ), ( "de", &["[calm] Einen Moment...", "[thoughtful] Mal sehen...", "[sigh] Okay, gleich..."], ), ( "pt", &["[calm] Um momento...", "[thoughtful] Vejamos...", "[sigh] Tá bom, já vai..."], ), ]; /// Backchannel phrases of `lang` as `(category, phrase)` pairs, or `None` when /// the language is not in the built-in table. pub fn backchannel_texts(lang: &str) -> Option> { BACKCHANNEL_TEXTS .iter() .find(|(l, _)| *l == lang) .map(|(_, list)| { list.iter() .map(|(c, p)| (c.to_string(), p.to_string())) .collect() }) } /// End-of-turn filler phrases of `lang`, or `None` when the language is not in /// the built-in table. pub fn filler_texts(lang: &str) -> Option> { FILLER_TEXTS .iter() .find(|(l, _)| *l == lang) .map(|(_, list)| list.iter().map(|s| s.to_string()).collect()) } /// Start-filler phrases of `lang`, or `None` when the language is not in the /// built-in table. pub fn start_filler_texts(lang: &str) -> Option> { START_FILLER_TEXTS .iter() .find(|(l, _)| *l == lang) .map(|(_, list)| list.iter().map(|s| s.to_string()).collect()) } /// The clip banks of one language, published together so the filler path /// (orchestrator) and the backchannel path (mic task) always agree on which /// language is active. #[derive(Clone)] pub struct LangBanks { /// ISO 639-1 code of this bank. pub lang: String, /// Backchannel clips for this language. pub backchannel: Arc, /// Spoken phrases of [`Self::backchannel`], in clip order. pub backchannel_texts: Vec, /// Category of each phrase in [`Self::backchannel_texts`] (parallel array). pub backchannel_categories: Vec, /// End-of-turn filler clips for this language. pub filler: Arc, /// Start-filler clips for this language. pub start_filler: Arc, } /// Per-session language state, shared between the turn manager (end-of-turn and /// start filler) and the microphone task (backchannel). pub struct LanguageState { /// The banks of the configured default language (`LANG_DEFAULT`), i.e. the /// base directories. Kept so switching back needs no regeneration. base: Arc, /// Currently active banks. current: parking_lot::RwLock>, /// Bumped on every switch so the mic task can notice a change with a single /// atomic load per audio frame. generation: AtomicU64, } impl LanguageState { pub fn new( lang: String, backchannel: Arc, backchannel_texts: Vec, backchannel_categories: Vec, filler: Arc, start_filler: Arc, ) -> Self { let base = Arc::new(LangBanks { lang, backchannel, backchannel_texts, backchannel_categories, filler, start_filler, }); Self { current: parking_lot::RwLock::new(base.clone()), base, generation: AtomicU64::new(0), } } /// The banks currently in use. pub fn current(&self) -> Arc { self.current.read().clone() } /// The banks of the default language (base directories). pub fn base(&self) -> Arc { self.base.clone() } /// Current generation; changes whenever the active language changes. pub fn generation(&self) -> u64 { self.generation.load(Ordering::Acquire) } /// Publish a new set of banks and bump the generation. pub fn set(&self, banks: LangBanks) { *self.current.write() = Arc::new(banks); self.generation.fetch_add(1, Ordering::AcqRel); } } /// System prompt of the language-identification call. Written to yield *only* /// the ISO code, so the reply can be parsed without a JSON or function-call /// round trip (this deployment runs no server-side tool parser). const DETECT_SYSTEM_PROMPT: &str = "You are a language identification tool. Read the user's \ text and reply with only the ISO 639-1 two-letter code of the language it is written in \ (for example: en, es, it, fr, de, pt). No punctuation, no quotes, no explanation. Reply \ with und if you cannot tell."; /// Ask the LLM for the ISO 639-1 code of `sample`. /// /// Best-effort: `None` on any request failure or when the reply does not carry /// a plausible code, in which case the currently active language simply stays. pub async fn detect_language(llm: &LlmClient, sample: &str) -> Option { let cancel = CancellationToken::new(); let reply = match llm .simple_chat(DETECT_SYSTEM_PROMPT, sample, 8, 0.0, &cancel) .await { Ok(r) => r, Err(e) => { warn!(error = %e, "language detection call failed"); return None; } }; parse_lang_code(&reply) } /// The first run of ASCII letters in `raw`, lowercased, kept only when it looks /// like an ISO 639-1 code (`"es."` → `"es"`, `"Spanish"` → `None`, /// `"und"` → `None`). fn parse_lang_code(raw: &str) -> Option { let mut run = String::new(); let mut started = false; for c in raw.trim().chars() { if c.is_ascii_alphabetic() { started = true; run.push(c.to_ascii_lowercase()); } else if started { break; } } match run.len() { 2 | 3 if run != "und" => Some(run), _ => None, } } const BACKCHANNEL_GEN_PROMPT: &str = "You write backchannel cues: the very short \ acknowledgments a listener says while someone else is speaking, like \"mm-hmm\", \ \"I see\" or \"right\". Write eight different cues in the language with ISO 639-1 code \ {lang}. Each cue is at most three words, ends with an ellipsis and is never a question. \ Reply with one cue per line and nothing else: no numbering, no quotes, no explanations."; const FILLER_GEN_PROMPT: &str = "You write hesitation fillers: the short sounds a speaker \ makes to buy a moment while thinking, like \"uhm\", \"well\" or \"let me see\". Write \ seven different fillers in the language with ISO 639-1 code {lang}. Each filler is one \ or two words ending with an ellipsis. Reply with one filler per line and nothing else: \ no numbering, no quotes, no explanations."; const START_FILLER_GEN_PROMPT: &str = "You write turn-start acknowledgments: the very short \ phrases a speaker says to buy a moment right before answering, like \"one moment\", \ \"let me see\" or \"okay, coming up\". Write three different phrases in the language \ with ISO 639-1 code {lang}. Each phrase is two to four words and ends with an ellipsis. \ Reply with one phrase per line and nothing else: no numbering, no quotes, no \ explanations."; /// Generate a backchannel phrase list with the LLM (fallback for languages /// without a built-in table). `None` when the call fails or yields nothing /// usable. pub async fn generate_backchannel_texts(llm: &LlmClient, lang: &str) -> Option> { let prompt = BACKCHANNEL_GEN_PROMPT.replace("{lang}", lang); generate_texts(llm, &prompt, "backchannel").await } /// Generate an end-of-turn filler phrase list with the LLM. pub async fn generate_filler_texts(llm: &LlmClient, lang: &str) -> Option> { let prompt = FILLER_GEN_PROMPT.replace("{lang}", lang); generate_texts(llm, &prompt, "filler").await } /// Generate a start-filler phrase list with the LLM (fallback for languages /// without a built-in table). pub async fn generate_start_filler_texts(llm: &LlmClient, lang: &str) -> Option> { let prompt = START_FILLER_GEN_PROMPT.replace("{lang}", lang); generate_texts(llm, &prompt, "start-filler").await } async fn generate_texts(llm: &LlmClient, prompt: &str, kind: &str) -> Option> { let cancel = CancellationToken::new(); let reply = match llm .simple_chat(prompt, "Generate the list now.", 256, 0.8, &cancel) .await { Ok(r) => r, Err(e) => { warn!(kind, error = %e, "phrase-list generation failed"); return None; } }; let texts = parse_lines(&reply, LLM_TEXT_LIMIT); if texts.len() < 2 { warn!(kind, lines = texts.len(), "phrase-list generation returned too few usable lines"); return None; } info!(kind, phrases = texts.len(), "generated phrase list via LLM"); Some(texts) } /// Split an LLM list reply into clean phrases: one per line, bullets/numbering /// and surrounding quotes removed, over-long lines dropped. fn parse_lines(raw: &str, max: usize) -> Vec { raw.lines() .map(|l| { l.trim() .trim_start_matches(|c: char| { c.is_ascii_digit() || matches!(c, '-' | '*' | '.' | ')' | ':' | ' ') }) .trim_matches(|c| matches!(c, '"' | '\'' | '`')) .trim() }) .filter(|l| !l.is_empty() && l.len() <= 60) .take(max) .map(|l| l.to_string()) .collect() } #[cfg(test)] mod tests { use super::*; #[test] fn parses_iso_code_from_various_replies() { assert_eq!(parse_lang_code("es"), Some("es".to_string())); assert_eq!(parse_lang_code("EN."), Some("en".to_string())); assert_eq!(parse_lang_code(" it\n"), Some("it".to_string())); assert_eq!(parse_lang_code("und"), None); assert_eq!(parse_lang_code("Spanish"), None); assert_eq!(parse_lang_code(""), None); } #[test] fn strips_bullets_and_numbering_from_generated_lists() { let raw = "1. Mmm...\n- Certo...\n\"Va bene...\"\n\n* Capisco..."; assert_eq!( parse_lines(raw, 10), vec![ "Mmm...".to_string(), "Certo...".to_string(), "Va bene...".to_string(), "Capisco...".to_string(), ] ); } #[test] fn builtin_tables_cover_the_base_languages() { for lang in ["es", "en", "it", "fr", "de", "pt"] { assert!(backchannel_texts(lang).is_some(), "missing backchannel table for {lang}"); assert!(filler_texts(lang).is_some(), "missing filler table for {lang}"); assert!(start_filler_texts(lang).is_some(), "missing start-filler table for {lang}"); } assert!(backchannel_texts("xx").is_none()); assert!(start_filler_texts("xx").is_none()); } }