piperust/src/lang.rs

//! Multilingual clip banks: language detection and per-language phrase tables.
//!
//! The shipped clip banks (backchannel, end-of-turn filler and start filler,
//! built in [`crate::assets`]) are pre-synthesized in a single language — the
//! one configured by `LANG_DEFAULT` (Spanish by default), which lives in the
//! base directories (`BACKCHANNEL_DIR`, `FILLERS_DIR`, `START_FILLER_DIR`).
//! When the user is detected speaking a different language, the banks for that
//! language are generated on demand into a per-language subdirectory
//! (`<base>/<lang>/`, ISO 639-1) and the session switches to them. Every
//! language has its own directory, so regenerating one never removes the clips
//! of another.
//!
//! Detection is a dedicated LLM call (`LANG_DETECT_ENABLED`) returning the ISO
//! 639-1 code of the transcript. The phrase lists come from the built-in tables
//! below for the common languages, and from the LLM
//! (`LANG_LLM_FALLBACK_ENABLED`) for any other language.

use crate::llm::LlmClient;
use crate::orchestrator::FillerBank;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};

/// Maximum number of phrases accepted from an LLM-generated list.
const LLM_TEXT_LIMIT: usize = 12;

/// Backchannel phrases per language, as `(category, phrase)` pairs.
///
/// The category drives the progression through a turn (an acknowledgment first,
/// agreement later — see [`crate::orchestrator::backchannel`]); the Drama-3
/// style tag steers the delivery and is passed straight to the TTS. Spanish
/// matches [`DEFAULT_BACKCHANNEL_TEXTS`](crate::orchestrator::backchannel::DEFAULT_BACKCHANNEL_TEXTS),
/// the language of the shipped clip bank.
pub const BACKCHANNEL_TEXTS: &[(&str, &[(&str, &str)])] = &[
    (
        "es",
        &[
            ("ack", "[calm] Mm-hmm..."),
            ("ack", "[attentive] Ajá..."),
            ("agree", "[agreeing] Claro..."),
            ("ack", "[listening] Ya veo..."),
            ("ack", "[attentive] Sí, sí..."),
            ("ack", "[calm] Entiendo..."),
            ("agree", "[relaxed] Vale..."),
            ("ack", "[thoughtful] Ah, ya..."),
        ],
    ),
    (
        "en",
        &[
            ("ack", "[calm] Mm-hmm..."),
            ("ack", "[attentive] Uh-huh..."),
            ("agree", "[agreeing] Right..."),
            ("ack", "[listening] I see..."),
            ("ack", "[attentive] Yeah, yeah..."),
            ("ack", "[calm] Got it..."),
            ("agree", "[relaxed] Okay..."),
            ("ack", "[thoughtful] Oh, I see..."),
        ],
    ),
    (
        "it",
        &[
            ("ack", "[calm] Mmm..."),
            ("ack", "[attentive] Eh già..."),
            ("agree", "[agreeing] Certo..."),
            ("ack", "[listening] Capisco..."),
            ("ack", "[attentive] Sì, sì..."),
            ("ack", "[calm] Ho capito..."),
            ("agree", "[relaxed] Va bene..."),
            ("ack", "[thoughtful] Ah, ecco..."),
        ],
    ),
    (
        "fr",
        &[
            ("ack", "[calm] Mm-hmm..."),
            ("ack", "[attentive] Hum, oui..."),
            ("agree", "[agreeing] Bien sûr..."),
            ("ack", "[listening] Je vois..."),
            ("ack", "[attentive] Oui, oui..."),
            ("ack", "[calm] D'accord..."),
            ("agree", "[relaxed] Très bien..."),
            ("ack", "[thoughtful] Ah, je vois..."),
        ],
    ),
    (
        "de",
        &[
            ("ack", "[calm] Mhm..."),
            ("ack", "[attentive] Aha..."),
            ("agree", "[agreeing] Genau..."),
            ("ack", "[listening] Ich verstehe..."),
            ("ack", "[attentive] Ja, ja..."),
            ("ack", "[calm] Verstehe..."),
            ("agree", "[relaxed] In Ordnung..."),
            ("ack", "[thoughtful] Ach so..."),
        ],
    ),
    (
        "pt",
        &[
            ("ack", "[calm] Mm-hmm..."),
            ("ack", "[attentive] Aham..."),
            ("agree", "[agreeing] Claro..."),
            ("ack", "[listening] Entendo..."),
            ("ack", "[attentive] Sim, sim..."),
            ("ack", "[calm] Entendi..."),
            ("agree", "[relaxed] Tá bom..."),
            ("ack", "[thoughtful] Ah, entendi..."),
        ],
    ),
];

/// End-of-turn filler interjections per language, played round-robin. Spanish
/// matches [`FILLER_CLIPS`](crate::assets::FILLER_CLIPS), the language of the
/// shipped filler bank.
pub const FILLER_TEXTS: &[(&str, &[&str])] = &[
    ("es", &["Uhm...", "Eh...", "Mmm...", "A ver...", "Bueno...", "Pues...", "Vaya..."]),
    ("en", &["Uhm...", "Hmm...", "Mmm...", "Well...", "Uh, let's see...", "Okay...", "Right..."]),
    ("it", &["Uhm...", "Eh...", "Mmm...", "Vediamo...", "Allora...", "Beh...", "Dunque..."]),
    ("fr", &["Hum...", "Euh...", "Mmm...", "Voyons...", "Alors...", "Ben...", "Bref..."]),
    ("de", &["Ähm...", "Hm...", "Mmm...", "Mal sehen...", "Also...", "Naja...", "Tja..."]),
    ("pt", &["Hã...", "É...", "Mmm...", "Vejamos...", "Então...", "Bom...", "Pois..."]),
];

/// Start-filler phrases per language: the turn-start acknowledgment played when
/// the reply audio is late (see
/// [`TurnManager::schedule_start_filler`](crate::orchestrator::turn_manager::TurnManager)).
/// Each phrase carries a Drama-3 style tag. Spanish matches the shipped default
/// `START_FILLER_TEXTS`, the language of the base clip bank.
pub const START_FILLER_TEXTS: &[(&str, &[&str])] = &[
    (
        "es",
        &["[calm] Un momento...", "[thoughtful] A ver, veamos...", "[sigh] Vale, ya voy..."],
    ),
    (
        "en",
        &["[calm] One moment...", "[thoughtful] Let me see...", "[sigh] Okay, coming up..."],
    ),
    (
        "it",
        &["[calm] Un momento...", "[thoughtful] Vediamo un po'...", "[sigh] Va bene, arrivo..."],
    ),
    (
        "fr",
        &["[calm] Un instant...", "[thoughtful] Voyons voir...", "[sigh] Bon, voilà..."],
    ),
    (
        "de",
        &["[calm] Einen Moment...", "[thoughtful] Mal sehen...", "[sigh] Okay, gleich..."],
    ),
    (
        "pt",
        &["[calm] Um momento...", "[thoughtful] Vejamos...", "[sigh] Tá bom, já vai..."],
    ),
];

/// Backchannel phrases of `lang` as `(category, phrase)` pairs, or `None` when
/// the language is not in the built-in table.
pub fn backchannel_texts(lang: &str) -> Option<Vec<(String, String)>> {
    BACKCHANNEL_TEXTS
        .iter()
        .find(|(l, _)| *l == lang)
        .map(|(_, list)| {
            list.iter()
                .map(|(c, p)| (c.to_string(), p.to_string()))
                .collect()
        })
}

/// End-of-turn filler phrases of `lang`, or `None` when the language is not in
/// the built-in table.
pub fn filler_texts(lang: &str) -> Option<Vec<String>> {
    FILLER_TEXTS
        .iter()
        .find(|(l, _)| *l == lang)
        .map(|(_, list)| list.iter().map(|s| s.to_string()).collect())
}

/// Start-filler phrases of `lang`, or `None` when the language is not in the
/// built-in table.
pub fn start_filler_texts(lang: &str) -> Option<Vec<String>> {
    START_FILLER_TEXTS
        .iter()
        .find(|(l, _)| *l == lang)
        .map(|(_, list)| list.iter().map(|s| s.to_string()).collect())
}

/// The clip banks of one language, published together so the filler path
/// (orchestrator) and the backchannel path (mic task) always agree on which
/// language is active.
#[derive(Clone)]
pub struct LangBanks {
    /// ISO 639-1 code of this bank.
    pub lang: String,
    /// Backchannel clips for this language.
    pub backchannel: Arc<FillerBank>,
    /// Spoken phrases of [`Self::backchannel`], in clip order.
    pub backchannel_texts: Vec<String>,
    /// Category of each phrase in [`Self::backchannel_texts`] (parallel array).
    pub backchannel_categories: Vec<String>,
    /// End-of-turn filler clips for this language.
    pub filler: Arc<FillerBank>,
    /// Start-filler clips for this language.
    pub start_filler: Arc<FillerBank>,
}

/// Per-session language state, shared between the turn manager (end-of-turn and
/// start filler) and the microphone task (backchannel).
pub struct LanguageState {
    /// The banks of the configured default language (`LANG_DEFAULT`), i.e. the
    /// base directories. Kept so switching back needs no regeneration.
    base: Arc<LangBanks>,
    /// Currently active banks.
    current: parking_lot::RwLock<Arc<LangBanks>>,
    /// Bumped on every switch so the mic task can notice a change with a single
    /// atomic load per audio frame.
    generation: AtomicU64,
}

impl LanguageState {
    pub fn new(
        lang: String,
        backchannel: Arc<FillerBank>,
        backchannel_texts: Vec<String>,
        backchannel_categories: Vec<String>,
        filler: Arc<FillerBank>,
        start_filler: Arc<FillerBank>,
    ) -> Self {
        let base = Arc::new(LangBanks {
            lang,
            backchannel,
            backchannel_texts,
            backchannel_categories,
            filler,
            start_filler,
        });
        Self {
            current: parking_lot::RwLock::new(base.clone()),
            base,
            generation: AtomicU64::new(0),
        }
    }

    /// The banks currently in use.
    pub fn current(&self) -> Arc<LangBanks> {
        self.current.read().clone()
    }

    /// The banks of the default language (base directories).
    pub fn base(&self) -> Arc<LangBanks> {
        self.base.clone()
    }

    /// Current generation; changes whenever the active language changes.
    pub fn generation(&self) -> u64 {
        self.generation.load(Ordering::Acquire)
    }

    /// Publish a new set of banks and bump the generation.
    pub fn set(&self, banks: LangBanks) {
        *self.current.write() = Arc::new(banks);
        self.generation.fetch_add(1, Ordering::AcqRel);
    }
}

/// System prompt of the language-identification call. Written to yield *only*
/// the ISO code, so the reply can be parsed without a JSON or function-call
/// round trip (this deployment runs no server-side tool parser).
const DETECT_SYSTEM_PROMPT: &str = "You are a language identification tool. Read the user's \
     text and reply with only the ISO 639-1 two-letter code of the language it is written in \
     (for example: en, es, it, fr, de, pt). No punctuation, no quotes, no explanation. Reply \
     with und if you cannot tell.";

/// Ask the LLM for the ISO 639-1 code of `sample`.
///
/// Best-effort: `None` on any request failure or when the reply does not carry
/// a plausible code, in which case the currently active language simply stays.
pub async fn detect_language(llm: &LlmClient, sample: &str) -> Option<String> {
    let cancel = CancellationToken::new();
    let reply = match llm
        .simple_chat(DETECT_SYSTEM_PROMPT, sample, 8, 0.0, &cancel)
        .await
    {
        Ok(r) => r,
        Err(e) => {
            warn!(error = %e, "language detection call failed");
            return None;
        }
    };
    parse_lang_code(&reply)
}

/// The first run of ASCII letters in `raw`, lowercased, kept only when it looks
/// like an ISO 639-1 code (`"es."` → `"es"`, `"Spanish"` → `None`,
/// `"und"` → `None`).
fn parse_lang_code(raw: &str) -> Option<String> {
    let mut run = String::new();
    let mut started = false;
    for c in raw.trim().chars() {
        if c.is_ascii_alphabetic() {
            started = true;
            run.push(c.to_ascii_lowercase());
        } else if started {
            break;
        }
    }
    match run.len() {
        2 | 3 if run != "und" => Some(run),
        _ => None,
    }
}

const BACKCHANNEL_GEN_PROMPT: &str = "You write backchannel cues: the very short \
     acknowledgments a listener says while someone else is speaking, like \"mm-hmm\", \
     \"I see\" or \"right\". Write eight different cues in the language with ISO 639-1 code \
     {lang}. Each cue is at most three words, ends with an ellipsis and is never a question. \
     Reply with one cue per line and nothing else: no numbering, no quotes, no explanations.";

const FILLER_GEN_PROMPT: &str = "You write hesitation fillers: the short sounds a speaker \
     makes to buy a moment while thinking, like \"uhm\", \"well\" or \"let me see\". Write \
     seven different fillers in the language with ISO 639-1 code {lang}. Each filler is one \
     or two words ending with an ellipsis. Reply with one filler per line and nothing else: \
     no numbering, no quotes, no explanations.";

const START_FILLER_GEN_PROMPT: &str = "You write turn-start acknowledgments: the very short \
     phrases a speaker says to buy a moment right before answering, like \"one moment\", \
     \"let me see\" or \"okay, coming up\". Write three different phrases in the language \
     with ISO 639-1 code {lang}. Each phrase is two to four words and ends with an ellipsis. \
     Reply with one phrase per line and nothing else: no numbering, no quotes, no \
     explanations.";

/// Generate a backchannel phrase list with the LLM (fallback for languages
/// without a built-in table). `None` when the call fails or yields nothing
/// usable.
pub async fn generate_backchannel_texts(llm: &LlmClient, lang: &str) -> Option<Vec<String>> {
    let prompt = BACKCHANNEL_GEN_PROMPT.replace("{lang}", lang);
    generate_texts(llm, &prompt, "backchannel").await
}

/// Generate an end-of-turn filler phrase list with the LLM.
pub async fn generate_filler_texts(llm: &LlmClient, lang: &str) -> Option<Vec<String>> {
    let prompt = FILLER_GEN_PROMPT.replace("{lang}", lang);
    generate_texts(llm, &prompt, "filler").await
}

/// Generate a start-filler phrase list with the LLM (fallback for languages
/// without a built-in table).
pub async fn generate_start_filler_texts(llm: &LlmClient, lang: &str) -> Option<Vec<String>> {
    let prompt = START_FILLER_GEN_PROMPT.replace("{lang}", lang);
    generate_texts(llm, &prompt, "start-filler").await
}

async fn generate_texts(llm: &LlmClient, prompt: &str, kind: &str) -> Option<Vec<String>> {
    let cancel = CancellationToken::new();
    let reply = match llm
        .simple_chat(prompt, "Generate the list now.", 256, 0.8, &cancel)
        .await
    {
        Ok(r) => r,
        Err(e) => {
            warn!(kind, error = %e, "phrase-list generation failed");
            return None;
        }
    };
    let texts = parse_lines(&reply, LLM_TEXT_LIMIT);
    if texts.len() < 2 {
        warn!(kind, lines = texts.len(), "phrase-list generation returned too few usable lines");
        return None;
    }
    info!(kind, phrases = texts.len(), "generated phrase list via LLM");
    Some(texts)
}

/// Split an LLM list reply into clean phrases: one per line, bullets/numbering
/// and surrounding quotes removed, over-long lines dropped.
fn parse_lines(raw: &str, max: usize) -> Vec<String> {
    raw.lines()
        .map(|l| {
            l.trim()
                .trim_start_matches(|c: char| {
                    c.is_ascii_digit() || matches!(c, '-' | '*' | '.' | ')' | ':' | ' ')
                })
                .trim_matches(|c| matches!(c, '"' | '\'' | '`'))
                .trim()
        })
        .filter(|l| !l.is_empty() && l.len() <= 60)
        .take(max)
        .map(|l| l.to_string())
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parses_iso_code_from_various_replies() {
        assert_eq!(parse_lang_code("es"), Some("es".to_string()));
        assert_eq!(parse_lang_code("EN."), Some("en".to_string()));
        assert_eq!(parse_lang_code("  it\n"), Some("it".to_string()));
        assert_eq!(parse_lang_code("und"), None);
        assert_eq!(parse_lang_code("Spanish"), None);
        assert_eq!(parse_lang_code(""), None);
    }

    #[test]
    fn strips_bullets_and_numbering_from_generated_lists() {
        let raw = "1. Mmm...\n- Certo...\n\"Va bene...\"\n\n* Capisco...";
        assert_eq!(
            parse_lines(raw, 10),
            vec![
                "Mmm...".to_string(),
                "Certo...".to_string(),
                "Va bene...".to_string(),
                "Capisco...".to_string(),
            ]
        );
    }

    #[test]
    fn builtin_tables_cover_the_base_languages() {
        for lang in ["es", "en", "it", "fr", "de", "pt"] {
            assert!(backchannel_texts(lang).is_some(), "missing backchannel table for {lang}");
            assert!(filler_texts(lang).is_some(), "missing filler table for {lang}");
            assert!(start_filler_texts(lang).is_some(), "missing start-filler table for {lang}");
        }
        assert!(backchannel_texts("xx").is_none());
        assert!(start_filler_texts("xx").is_none());
    }
}