step2
This commit is contained in:
@@ -9,13 +9,17 @@
|
||||
//! - **Without LLM**: transcript → TTS (the original pass-through behaviour).
|
||||
|
||||
use crate::audio::{AudioMixer, SampleRingBuffer};
|
||||
use crate::audit::SessionAudit;
|
||||
use crate::config::Config;
|
||||
use crate::lang::{self, LangBanks, LanguageState};
|
||||
use crate::llm::{ChatMessage, LlmClient};
|
||||
use crate::stt::{Tokenizer, Transcript};
|
||||
use crate::tools::ToolCallStripper;
|
||||
use crate::tts::TtsEngine;
|
||||
use crate::{assets, orchestrator::backchannel::DEFAULT_CATEGORY};
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -62,12 +66,15 @@ fn pump(src: &SampleRingBuffer, dst: &SampleRingBuffer) -> usize {
|
||||
}
|
||||
|
||||
/// Preloaded PCM samples keyed by name (e.g. "uhm"). Used for the filler bank
|
||||
/// injected at end-of-utterance (played round-robin) and, from a different
|
||||
/// directory, for the backchannel clips played while the user speaks.
|
||||
/// injected at end-of-utterance (played round-robin), for the backchannel clips
|
||||
/// played while the user speaks, and for the start-filler bank (both played in
|
||||
/// random order).
|
||||
pub struct FillerBank {
|
||||
samples: HashMap<String, Vec<i16>>,
|
||||
/// Names in a stable order, for round-robin selection.
|
||||
names: Vec<String>,
|
||||
/// xorshift64 state for [`Self::random`] (seeded from the clock at load).
|
||||
rng: AtomicU64,
|
||||
}
|
||||
|
||||
impl FillerBank {
|
||||
@@ -107,17 +114,37 @@ impl FillerBank {
|
||||
|
||||
let mut names: Vec<String> = samples.keys().cloned().collect();
|
||||
names.sort();
|
||||
Self { samples, names }
|
||||
|
||||
// Seed the xorshift64 state from the wall clock (OR 1 so it is never
|
||||
// zero, which would make xorshift degenerate).
|
||||
let seed = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_nanos() as u64)
|
||||
.unwrap_or(0x9E37_79B9_7F4A_7C15)
|
||||
| 1;
|
||||
|
||||
Self {
|
||||
samples,
|
||||
names,
|
||||
rng: AtomicU64::new(seed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the samples for a clip by name. Kept for reference/debugging.
|
||||
#[allow(dead_code)]
|
||||
/// Get the samples for a clip by name. Used by the backchannel to play a
|
||||
/// specific (category-selected) clip instead of a uniform random one.
|
||||
pub fn get(&self, name: &str) -> Option<&Vec<i16>> {
|
||||
self.samples.get(name)
|
||||
}
|
||||
|
||||
/// Clip names in the stable (sorted) order used by [`Self::nth`]. The
|
||||
/// backchannel uses it to map the phrase list (and its categories) onto the
|
||||
/// synthesized clips, since clips are named `{prefix}{index}` in list order.
|
||||
pub fn names(&self) -> &[String] {
|
||||
&self.names
|
||||
}
|
||||
|
||||
/// Clip at position `index` (wrapping), with its name. Used for the
|
||||
/// round-robin filler and backchannel selection.
|
||||
/// round-robin end-of-utterance filler.
|
||||
pub fn nth(&self, index: usize) -> Option<(&str, &Vec<i16>)> {
|
||||
if self.names.is_empty() {
|
||||
return None;
|
||||
@@ -126,6 +153,23 @@ impl FillerBank {
|
||||
self.samples.get(name).map(|s| (name.as_str(), s))
|
||||
}
|
||||
|
||||
/// Clip chosen uniformly at random, with its name. Used by the start filler
|
||||
/// and the backchannel so the cue varies from turn to turn.
|
||||
pub fn random(&self) -> Option<(&str, &Vec<i16>)> {
|
||||
if self.names.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// xorshift64 step over the shared state.
|
||||
let mut x = self.rng.load(Ordering::Relaxed);
|
||||
x ^= x << 13;
|
||||
x ^= x >> 7;
|
||||
x ^= x << 17;
|
||||
self.rng.store(x, Ordering::Relaxed);
|
||||
|
||||
let name = &self.names[(x % self.names.len() as u64) as usize];
|
||||
self.samples.get(name).map(|s| (name.as_str(), s))
|
||||
}
|
||||
|
||||
/// Whether the bank holds no clips.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.names.is_empty()
|
||||
@@ -136,7 +180,10 @@ impl FillerBank {
|
||||
pub struct TurnManager {
|
||||
mixer: AudioMixer,
|
||||
tts: Arc<dyn TtsEngine>,
|
||||
fillers: Arc<FillerBank>,
|
||||
/// Active-language clip banks, shared with the mic task (backchannel). The
|
||||
/// end-of-turn and start fillers are read from here so all three cues
|
||||
/// follow the same language — see [`crate::lang`].
|
||||
langs: Arc<LanguageState>,
|
||||
/// Optional tokenizer to decode STT token IDs into text. If `None`, a
|
||||
/// placeholder string is sent to the TTS (useful for testing the pipeline
|
||||
/// without a real tokenizer).
|
||||
@@ -154,22 +201,31 @@ pub struct TurnManager {
|
||||
/// Outbound text-event channel (JSON strings) consumed by the session
|
||||
/// sender task and forwarded to the browser (chat box).
|
||||
out_tx: Option<mpsc::UnboundedSender<String>>,
|
||||
/// User turns seen so far, used to schedule the periodic language check.
|
||||
lang_turns: AtomicU64,
|
||||
/// Whether a language-detection call is currently in flight (one at a time).
|
||||
lang_detecting: Arc<AtomicBool>,
|
||||
/// Per-session audit sink: the language switches are recorded here (every
|
||||
/// other event reaches the log through the outbound tap in the session
|
||||
/// sender — see [`crate::audit`]).
|
||||
audit: Arc<SessionAudit>,
|
||||
}
|
||||
|
||||
impl TurnManager {
|
||||
pub fn new(
|
||||
mixer: AudioMixer,
|
||||
tts: Arc<dyn TtsEngine>,
|
||||
fillers: Arc<FillerBank>,
|
||||
langs: Arc<LanguageState>,
|
||||
tokenizer: Option<Tokenizer>,
|
||||
llm: Option<Arc<LlmClient>>,
|
||||
config: Arc<Config>,
|
||||
out_tx: Option<mpsc::UnboundedSender<String>>,
|
||||
audit: Arc<SessionAudit>,
|
||||
) -> Self {
|
||||
Self {
|
||||
mixer,
|
||||
tts,
|
||||
fillers,
|
||||
langs,
|
||||
tokenizer,
|
||||
cancel: Arc::new(parking_lot::Mutex::new(None)),
|
||||
filler_cursor: AtomicUsize::new(0),
|
||||
@@ -177,13 +233,76 @@ impl TurnManager {
|
||||
config,
|
||||
history: parking_lot::Mutex::new(Vec::new()),
|
||||
out_tx,
|
||||
lang_turns: AtomicU64::new(0),
|
||||
lang_detecting: Arc::new(AtomicBool::new(false)),
|
||||
audit,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pick the next filler clip round-robin from the bank.
|
||||
/// Pick the next filler clip round-robin from the bank of the active
|
||||
/// language.
|
||||
fn next_filler(&self) -> Option<(String, Vec<i16>)> {
|
||||
let banks = self.langs.current();
|
||||
let i = self.filler_cursor.fetch_add(1, Ordering::Relaxed);
|
||||
self.fillers.nth(i).map(|(name, s)| (name.to_string(), s.clone()))
|
||||
banks
|
||||
.filler
|
||||
.nth(i)
|
||||
.map(|(name, s)| (name.to_string(), s.clone()))
|
||||
}
|
||||
|
||||
/// Schedule the *start filler* acknowledgment on the mixer's dedicated
|
||||
/// start-filler channel.
|
||||
///
|
||||
/// Like [`Self::schedule_filler`] this is a latency mask, but with its own
|
||||
/// (shorter) delay, `START_FILLER_GO_DELAY`: the cue is only queued when the
|
||||
/// first response audio is still **late** after that window — if the first
|
||||
/// audio chunk has already reached the mixer (even if it is still inside the
|
||||
/// pre-buffer and playback has not started), the agent is already talking
|
||||
/// and the cue is dropped. One phrase is picked **at random** from the bank
|
||||
/// (built from `START_FILLER_TEXTS`), so the cue varies turn to turn.
|
||||
/// Disabled by `START_FILLER_ENABLED=false`; a barge-in (`cancel`) or a new
|
||||
/// turn drops the pending cue, and `mixer.flush` clears it once queued.
|
||||
fn schedule_start_filler(&self, cancel: &CancellationToken) {
|
||||
if !self.config.start_filler_enabled {
|
||||
return;
|
||||
}
|
||||
// Own the clip: the spawned task must be `'static` (same as the
|
||||
// round-robin filler above).
|
||||
let banks = self.langs.current();
|
||||
let Some((name, samples)) = banks
|
||||
.start_filler
|
||||
.random()
|
||||
.map(|(n, s)| (n.to_string(), s.clone()))
|
||||
else {
|
||||
debug!("start filler enabled but no clip loaded — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
let delay_ms = self.config.start_filler_go_delay_ms;
|
||||
let mixer = self.mixer.clone();
|
||||
let cancel = cancel.clone();
|
||||
let out_tx = self.out_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {}
|
||||
// Barge-in / new turn: never play a stale acknowledgment.
|
||||
_ = cancel.cancelled() => return,
|
||||
}
|
||||
if mixer.tts_audio_arrived() {
|
||||
debug!(
|
||||
filler = %name,
|
||||
delay_ms, "first response audio already arrived — start filler skipped"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let queued = mixer.start_filler_queue.push(&samples);
|
||||
debug!(filler = %name, samples = queued, delay_ms, "queued start filler");
|
||||
if let Some(tx) = &out_tx {
|
||||
let _ = tx.send(
|
||||
json!({ "type": "cue", "kind": "start_filler", "name": name }).to_string(),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Schedule the single end-of-turn filler acknowledgment.
|
||||
@@ -207,6 +326,7 @@ impl TurnManager {
|
||||
let delay_ms = self.config.filler_delay_ms;
|
||||
let mixer = self.mixer.clone();
|
||||
let cancel = cancel.clone();
|
||||
let out_tx = self.out_tx.clone();
|
||||
let samples_len = samples.len();
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
@@ -223,6 +343,11 @@ impl TurnManager {
|
||||
}
|
||||
mixer.filler_queue.push(&samples);
|
||||
debug!(filler = %name, samples = samples_len, delay_ms, "pushed filler (delayed)");
|
||||
if let Some(tx) = &out_tx {
|
||||
let _ = tx.send(
|
||||
json!({ "type": "cue", "kind": "filler", "name": name }).to_string(),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -274,12 +399,14 @@ impl TurnManager {
|
||||
/// 2. Cancel any in-flight reply (barge-in within the turn).
|
||||
/// 3. Run the LLM (if configured) or send the text straight to the TTS.
|
||||
///
|
||||
/// The filler is not queued here: it is scheduled once the response is
|
||||
/// dispatched, so it only plays if the audio is late (see
|
||||
/// [`Self::schedule_filler`]).
|
||||
/// The latency masks are not queued here: both the start filler and the
|
||||
/// end-of-turn filler are *scheduled* once the response is dispatched, so
|
||||
/// they only play if the audio is late — see [`Self::schedule_filler`] and
|
||||
/// [`Self::schedule_start_filler`].
|
||||
async fn handle_user_turn(&self, text: String) {
|
||||
self.emit(json!({ "type": "user", "text": text }));
|
||||
|
||||
self.maybe_detect_language(&text);
|
||||
self.cancel_current_tts();
|
||||
|
||||
match &self.llm {
|
||||
@@ -288,11 +415,58 @@ impl TurnManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kick off a language check for `text` when the detection schedule allows
|
||||
/// it. Runs in the background so the reply is never delayed: the switch (if
|
||||
/// any) takes effect from the next turn. See [`crate::lang`].
|
||||
///
|
||||
/// The first turn is always checked; afterwards a sample is sent only every
|
||||
/// `LANG_REDETECT_EVERY` turns, so a language change mid-conversation is
|
||||
/// picked up without paying for a detection call on every turn.
|
||||
fn maybe_detect_language(&self, text: &str) {
|
||||
if !self.config.lang_detect_enabled {
|
||||
return;
|
||||
}
|
||||
let Some(llm) = self.llm.clone() else {
|
||||
return;
|
||||
};
|
||||
// Only the backchannel, the end-of-turn filler and the start filler are
|
||||
// language-aware; with all of them disabled there is nothing to switch.
|
||||
if !self.config.backchannel.enabled
|
||||
&& !self.config.filler_enabled
|
||||
&& !self.config.start_filler_enabled
|
||||
{
|
||||
return;
|
||||
}
|
||||
if text.chars().count() < self.config.lang_min_chars {
|
||||
return;
|
||||
}
|
||||
let turn = self.lang_turns.fetch_add(1, Ordering::Relaxed);
|
||||
if turn % self.config.lang_redetect_every != 0 {
|
||||
return;
|
||||
}
|
||||
// One detection at a time: a slow LLM call must not pile up.
|
||||
if self.lang_detecting.swap(true, Ordering::AcqRel) {
|
||||
return;
|
||||
}
|
||||
let langs = self.langs.clone();
|
||||
let tts = self.tts.clone();
|
||||
let config = self.config.clone();
|
||||
let detecting = self.lang_detecting.clone();
|
||||
let audit = self.audit.clone();
|
||||
let text = text.to_string();
|
||||
tokio::spawn(async move {
|
||||
switch_language_if_needed(langs, llm, tts, config, &text, audit).await;
|
||||
detecting.store(false, Ordering::Release);
|
||||
});
|
||||
}
|
||||
|
||||
/// Run one full LLM turn: stream the reply, chunk it into sentences and
|
||||
/// synthesize each sentence as soon as it is complete.
|
||||
async fn run_llm_turn(&self, llm: Arc<LlmClient>, user_text: String) {
|
||||
// Build the message list from the rolling history.
|
||||
let messages = {
|
||||
// Build the message list from the rolling history. It grows across
|
||||
// tool rounds (assistant call + tool results) but only the *spoken*
|
||||
// text is written back to the rolling history.
|
||||
let mut messages = {
|
||||
let mut hist = self.history.lock();
|
||||
hist.push(ChatMessage::user(&user_text));
|
||||
let max = self.config.llm_history_messages.max(2);
|
||||
@@ -313,6 +487,7 @@ impl TurnManager {
|
||||
*self.cancel.lock() = Some(cancel.clone());
|
||||
self.mixer.mark_stream_start();
|
||||
self.schedule_filler(&cancel);
|
||||
self.schedule_start_filler(&cancel);
|
||||
debug!(
|
||||
endpoint = llm.endpoint(),
|
||||
model = llm.model(),
|
||||
@@ -414,63 +589,129 @@ impl TurnManager {
|
||||
}
|
||||
});
|
||||
|
||||
// LLM producer: streams clean text deltas into a channel.
|
||||
let (delta_tx, mut delta_rx) = mpsc::unbounded_channel::<String>();
|
||||
let stream_mode = self.config.llm_stream;
|
||||
let llm_cancel = cancel.clone();
|
||||
let llm_task = tokio::spawn(async move {
|
||||
if stream_mode {
|
||||
llm.chat_stream(&messages, &llm_cancel, delta_tx).await
|
||||
} else {
|
||||
let res = llm.chat(&messages, &llm_cancel).await;
|
||||
if let Ok(text) = &res {
|
||||
let _ = delta_tx.send(text.clone());
|
||||
}
|
||||
res
|
||||
}
|
||||
});
|
||||
// Tool-calling loop. Each round streams one reply; tool call blocks are
|
||||
// stripped from the spoken/displayed text (the model emits them as
|
||||
// text — see `crate::tools`), the calls run over HTTP, and the results
|
||||
// are handed back for the next round. Bounded by
|
||||
// `LLM_TOOL_MAX_ITERATIONS`. With no tools configured the loop runs
|
||||
// exactly once.
|
||||
let registry = llm.tools().cloned();
|
||||
let max_rounds = self.config.llm_tool_max_iterations.max(1);
|
||||
let mut spoken = String::new();
|
||||
let mut round = 0usize;
|
||||
|
||||
loop {
|
||||
// LLM producer: streams clean text deltas into a channel.
|
||||
let (delta_tx, mut delta_rx) = mpsc::unbounded_channel::<String>();
|
||||
let stream_mode = self.config.llm_stream;
|
||||
let llm_cancel = cancel.clone();
|
||||
let llm_round = llm.clone();
|
||||
let round_messages = messages.clone();
|
||||
let llm_task = tokio::spawn(async move {
|
||||
if stream_mode {
|
||||
llm_round.chat_stream(&round_messages, &llm_cancel, delta_tx).await
|
||||
} else {
|
||||
let res = llm_round.chat(&round_messages, &llm_cancel).await;
|
||||
if let Ok(text) = &res {
|
||||
let _ = delta_tx.send(text.clone());
|
||||
}
|
||||
res
|
||||
}
|
||||
});
|
||||
|
||||
// Consume deltas: emit `assistant_delta` events and feed the
|
||||
// splitter, so the TTS starts as soon as the first few words are
|
||||
// available. The stripper holds back tool-call blocks so they are
|
||||
// never spoken nor shown.
|
||||
let mut splitter = SentenceSplitter::new(
|
||||
self.config.llm_stream_first_chunk_chars,
|
||||
self.config.llm_stream_chunk_chars,
|
||||
);
|
||||
let mut stripper = ToolCallStripper::new();
|
||||
let mut raw = String::new();
|
||||
while let Some(delta) = delta_rx.recv().await {
|
||||
if cancel.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
raw.push_str(&delta);
|
||||
let visible = stripper.push(&delta);
|
||||
if visible.is_empty() {
|
||||
continue;
|
||||
}
|
||||
self.emit(json!({ "type": "assistant_delta", "text": visible }));
|
||||
spoken.push_str(&visible);
|
||||
for sentence in splitter.push(&visible) {
|
||||
if sentence_tx.send(sentence).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !cancel.is_cancelled() {
|
||||
let tail = stripper.finish();
|
||||
if !tail.is_empty() {
|
||||
self.emit(json!({ "type": "assistant_delta", "text": tail }));
|
||||
for sentence in splitter.push(&tail) {
|
||||
if sentence_tx.send(sentence).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
spoken.push_str(&tail);
|
||||
}
|
||||
if let Some(rest) = splitter.finish() {
|
||||
let _ = sentence_tx.send(rest);
|
||||
}
|
||||
}
|
||||
|
||||
match llm_task.await {
|
||||
Ok(Ok(_)) => {}
|
||||
Ok(Err(e)) => {
|
||||
error!(error = %e, "LLM request failed");
|
||||
self.emit(json!({ "type": "error", "text": e.to_string() }));
|
||||
}
|
||||
Err(e) => {
|
||||
error!(error = %e, "LLM task join failed");
|
||||
self.emit(json!({ "type": "error", "text": format!("LLM task failed: {e}") }));
|
||||
}
|
||||
}
|
||||
|
||||
// Consume deltas: emit `assistant_delta` events and feed the splitter,
|
||||
// so the TTS starts as soon as the first few words are available.
|
||||
let mut splitter = SentenceSplitter::new(
|
||||
self.config.llm_stream_first_chunk_chars,
|
||||
self.config.llm_stream_chunk_chars,
|
||||
);
|
||||
let mut full = String::new();
|
||||
while let Some(delta) = delta_rx.recv().await {
|
||||
if cancel.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
self.emit(json!({ "type": "assistant_delta", "text": delta }));
|
||||
full.push_str(&delta);
|
||||
for sentence in splitter.push(&delta) {
|
||||
if sentence_tx.send(sentence).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !cancel.is_cancelled() {
|
||||
if let Some(rest) = splitter.finish() {
|
||||
let _ = sentence_tx.send(rest);
|
||||
|
||||
let calls = stripper.take_calls();
|
||||
if calls.is_empty() {
|
||||
break;
|
||||
}
|
||||
let Some(registry) = registry.as_ref() else {
|
||||
warn!("model emitted a tool call but no tools are configured — ignoring");
|
||||
break;
|
||||
};
|
||||
if round + 1 >= max_rounds {
|
||||
warn!(max_rounds, "tool-call iteration limit reached — stopping");
|
||||
break;
|
||||
}
|
||||
round += 1;
|
||||
|
||||
// Record the model's own call followed by each result, so the next
|
||||
// round sees the exchange (standard OpenAI tool round-trip).
|
||||
messages.push(ChatMessage::assistant(raw.clone()));
|
||||
for call in &calls {
|
||||
info!(tool = %call.name, round, "executing tool call");
|
||||
self.emit(json!({
|
||||
"type": "tool_call",
|
||||
"name": &call.name,
|
||||
"arguments": &call.arguments,
|
||||
}));
|
||||
let result = registry.execute(call).await;
|
||||
messages.push(ChatMessage::tool(result));
|
||||
}
|
||||
}
|
||||
|
||||
drop(sentence_tx);
|
||||
let _ = tts_task.await;
|
||||
|
||||
match llm_task.await {
|
||||
Ok(Ok(_)) => {}
|
||||
Ok(Err(e)) => {
|
||||
error!(error = %e, "LLM request failed");
|
||||
self.emit(json!({ "type": "error", "text": e.to_string() }));
|
||||
}
|
||||
Err(e) => {
|
||||
error!(error = %e, "LLM task join failed");
|
||||
self.emit(json!({ "type": "error", "text": format!("LLM task failed: {e}") }));
|
||||
}
|
||||
}
|
||||
|
||||
if !cancel.is_cancelled() && !full.trim().is_empty() {
|
||||
self.history.lock().push(ChatMessage::assistant(full.clone()));
|
||||
if !cancel.is_cancelled() && !spoken.trim().is_empty() {
|
||||
self.history.lock().push(ChatMessage::assistant(spoken.clone()));
|
||||
self.emit(json!({ "type": "assistant_end" }));
|
||||
}
|
||||
|
||||
@@ -500,6 +741,12 @@ impl TurnManager {
|
||||
self.tts.set_reference(audio_path, text);
|
||||
}
|
||||
|
||||
/// Set the gain of the mixer's external audio channel (URL / radio /
|
||||
/// YouTube background source). See [`crate::media`].
|
||||
pub fn set_external_gain(&self, gain: f32) {
|
||||
self.mixer.set_external_gain(gain);
|
||||
}
|
||||
|
||||
/// Synthesize `text` directly with the TTS engine, bypassing the LLM.
|
||||
/// Used by the playground's "Speak" box to test the TTS pipeline.
|
||||
pub async fn speak(&self, text: &str) {
|
||||
@@ -529,6 +776,7 @@ impl TurnManager {
|
||||
let text_owned = text.to_string();
|
||||
self.mixer.mark_stream_start();
|
||||
self.schedule_filler(&cancel);
|
||||
self.schedule_start_filler(&cancel);
|
||||
debug!(
|
||||
engine = tts.name(),
|
||||
sample_rate = tts.sample_rate(),
|
||||
@@ -560,6 +808,133 @@ impl TurnManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect the language of `text` and, when it differs from the active one,
|
||||
/// build and publish the clip banks for it.
|
||||
///
|
||||
/// The phrase list comes from the built-in tables ([`crate::lang`]) for the
|
||||
/// common languages, or from the LLM when `LANG_LLM_FALLBACK_ENABLED` is set.
|
||||
/// Clips are synthesized only when missing — [`assets::ensure_lang_backchannel_clips`]
|
||||
/// keeps a per-language index — and live in their own subdirectory, so no other
|
||||
/// language is touched. Switching back to the default language restores the
|
||||
/// base banks and needs no synthesis at all.
|
||||
async fn switch_language_if_needed(
|
||||
langs: Arc<LanguageState>,
|
||||
llm: Arc<LlmClient>,
|
||||
tts: Arc<dyn TtsEngine>,
|
||||
config: Arc<Config>,
|
||||
text: &str,
|
||||
audit: Arc<SessionAudit>,
|
||||
) {
|
||||
let Some(lang) = lang::detect_language(&llm, text).await else {
|
||||
return;
|
||||
};
|
||||
let current = langs.current();
|
||||
if lang == current.lang {
|
||||
return;
|
||||
}
|
||||
if lang == config.lang_default {
|
||||
info!(lang = %lang, "user language back to the default — restoring the base clip banks");
|
||||
audit.event(
|
||||
"language_switch",
|
||||
json!({ "from": current.lang, "to": lang, "generated": false }),
|
||||
);
|
||||
langs.set((*langs.base()).clone());
|
||||
return;
|
||||
}
|
||||
|
||||
// Phrase lists: built-in table first, LLM fallback otherwise.
|
||||
let backchannel = match lang::backchannel_texts(&lang) {
|
||||
Some(pairs) => pairs,
|
||||
None if config.lang_llm_fallback_enabled => {
|
||||
match lang::generate_backchannel_texts(&llm, &lang).await {
|
||||
Some(phrases) => phrases
|
||||
.into_iter()
|
||||
.map(|p| (DEFAULT_CATEGORY.to_string(), p))
|
||||
.collect(),
|
||||
None => return,
|
||||
}
|
||||
}
|
||||
None => return,
|
||||
};
|
||||
let filler = match lang::filler_texts(&lang) {
|
||||
Some(texts) => texts,
|
||||
None if config.lang_llm_fallback_enabled => {
|
||||
match lang::generate_filler_texts(&llm, &lang).await {
|
||||
Some(texts) => texts,
|
||||
None => return,
|
||||
}
|
||||
}
|
||||
None => return,
|
||||
};
|
||||
// Start filler is optional: when the language has no phrase list (and the
|
||||
// LLM fallback is unavailable) the cue simply stays silent for it, without
|
||||
// preventing the switch of the banks above.
|
||||
let start_filler = if !config.start_filler_enabled {
|
||||
None
|
||||
} else {
|
||||
match lang::start_filler_texts(&lang) {
|
||||
Some(texts) => Some(texts),
|
||||
None if config.lang_llm_fallback_enabled => {
|
||||
lang::generate_start_filler_texts(&llm, &lang).await
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
};
|
||||
|
||||
info!(
|
||||
from = %current.lang,
|
||||
to = %lang,
|
||||
backchannel = backchannel.len(),
|
||||
filler = filler.len(),
|
||||
start_filler = start_filler.as_ref().map(|t| t.len()).unwrap_or(0),
|
||||
"user language changed — generating the clip banks for it"
|
||||
);
|
||||
|
||||
let phrases: Vec<String> = backchannel.iter().map(|(_, p)| p.clone()).collect();
|
||||
let categories: Vec<String> = backchannel.iter().map(|(c, _)| c.clone()).collect();
|
||||
|
||||
// Idempotent: an unchanged phrase list is skipped via the per-directory
|
||||
// `texts.txt` index, so an already-generated language is just reloaded.
|
||||
assets::ensure_lang_backchannel_clips(&config.backchannel.dir, &lang, &phrases, tts.as_ref())
|
||||
.await;
|
||||
assets::ensure_lang_filler_clips(&config.fillers_dir, &lang, &filler, tts.as_ref()).await;
|
||||
if let Some(texts) = &start_filler {
|
||||
assets::ensure_lang_start_filler_clips(
|
||||
&config.start_filler_dir,
|
||||
&lang,
|
||||
texts,
|
||||
tts.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let backchannel_bank = Arc::new(FillerBank::load(
|
||||
&assets::lang_dir(&config.backchannel.dir, &lang),
|
||||
"backchannel",
|
||||
));
|
||||
let filler_bank = Arc::new(FillerBank::load(
|
||||
&assets::lang_dir(&config.fillers_dir, &lang),
|
||||
"filler",
|
||||
));
|
||||
let start_filler_bank = Arc::new(FillerBank::load(
|
||||
&assets::lang_dir(&config.start_filler_dir, &lang),
|
||||
"start-filler",
|
||||
));
|
||||
|
||||
langs.set(LangBanks {
|
||||
lang,
|
||||
backchannel: backchannel_bank,
|
||||
backchannel_texts: phrases,
|
||||
backchannel_categories: categories,
|
||||
filler: filler_bank,
|
||||
start_filler: start_filler_bank,
|
||||
});
|
||||
audit.event(
|
||||
"language_switch",
|
||||
json!({ "from": current.lang, "to": langs.current().lang, "generated": true }),
|
||||
);
|
||||
}
|
||||
|
||||
/// Spawn an orchestrator loop that consumes transcripts from `rx` and drives
|
||||
/// the turn manager.
|
||||
pub fn spawn_orchestrator(
|
||||
|
||||
Reference in New Issue
Block a user