roomote_PRE/piperust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

step2

Browse Source
This commit is contained in:
valenti
2026-10-17 15:49:20 +01:00
parent 430997bd2e
commit bfed42c3b3
124 changed files with 5646 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/orchestrator/backchannel.rs
+417 -26
View File
@@ -4,8 +4,15 @@
//! which is played once the user stops, at end-of-utterance. A backchannel is a
//! short "mm-hmm" / "ajá" mixed in at a reduced gain to signal that the agent is
//! listening, without taking the turn. It is emitted only while the agent itself
//! is silent (otherwise the user is being interrupted) and never more than
//! `max_per_turn` times.
//! is silent (otherwise the user is being interrupted).
//!
//! To sound natural rather than metronomic, a cue is played only when the user
//! is in a **micro-pause** ([`VadDetector::silence_ms`](crate::vad::VadDetector::silence_ms)),
//! at a **random depth** inside that pause, its timing is randomized, a
//! probabilistic gate skips some qualifying pauses,
//! recently used clips are not repeated, the volume drifts slightly, and the
//! phrases are grouped in **categories** that follow a progression through the
//! turn (an acknowledgment first, agreement later).
//!
//! Every behavioural knob is configured from the environment
//! ([`BackchannelSettings`]), so the feature can be tuned or disabled without a
@@ -13,9 +20,14 @@
use crate::config::{env_bool, env_f32, env_u32, env_u64};
use crate::orchestrator::FillerBank;
use std::collections::VecDeque;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tracing::info;
/// Category applied to phrases written without an explicit `category::` prefix.
pub(crate) const DEFAULT_CATEGORY: &str = "ack";
/// Runtime configuration for the backchannel subsystem, read from `.env`.
#[derive(Debug, Clone)]
@@ -24,31 +36,110 @@ pub struct BackchannelSettings {
pub enabled: bool,
/// Directory of `*.raw` PCM clips (`BACKCHANNEL_DIR`).
pub dir: PathBuf,
/// Phrases spoken by the backchannel bank (`BACKCHANNEL_TEXTS`), one per
/// line, with the optional `category::` prefix already stripped. One clip is
/// synthesized per phrase at startup. Drama-3 style tags (`[calm]`, …) are
/// passed straight through to the TTS.
pub texts: Vec<String>,
/// Category of each phrase in [`Self::texts`] (parallel array). Lines
/// written as `category::phrase` set it; plain lines default to `ack`.
pub categories: Vec<String>,
/// Mix gain applied to backchannel clips (`BACKCHANNEL_GAIN`). Kept low so
/// the cue stays in the background.
pub gain: f32,
/// Continuous speech required before the first backchannel
/// Confirmed speech required before the first cue may be considered
/// (`BACKCHANNEL_MIN_SPEECH_MS`).
pub min_speech_ms: u64,
/// Minimum spacing between two backchannels (`BACKCHANNEL_INTERVAL_MS`).
/// Base spacing between two cues of the same turn
/// (`BACKCHANNEL_INTERVAL_MS`); randomized by ±`jitter_ms`/2.
pub interval_ms: u64,
/// Maximum backchannels per user turn (`BACKCHANNEL_MAX_PER_TURN`).
/// Timing jitter (`BACKCHANNEL_JITTER_MS`): delays the first cue by
/// `0..jitter_ms` after `min_speech_ms` and spreads the spacing.
pub jitter_ms: u64,
/// Lower bound of the micro-pause window (`BACKCHANNEL_PAUSE_MIN_MS`): the
/// earliest depth, inside a pause, at which the user is likely done with the
/// current word. The actual reaction depth is drawn at random in
/// `[pause_min_ms, pause_max_ms]`.
pub pause_min_ms: u64,
/// Upper bound of the micro-pause window (`BACKCHANNEL_PAUSE_MAX_MS`): the
/// latest depth a cue may react at. Above this the utterance is about to
/// close, so a cue would land on the reply.
pub pause_max_ms: u64,
/// Probability of cueing when a micro-pause qualifies (`BACKCHANNEL_PROB`).
pub prob: f32,
/// Random per-cue volume scaling as a ± fraction
/// (`BACKCHANNEL_VOLUME_JITTER`).
pub volume_jitter: f32,
/// Maximum cues per user turn (`BACKCHANNEL_MAX_PER_TURN`).
pub max_per_turn: u32,
}
impl Default for BackchannelSettings {
fn default() -> Self {
let (texts, categories) = parse_texts(&DEFAULT_BACKCHANNEL_TEXTS.join("\n"));
Self {
enabled: true,
dir: PathBuf::from("assets/backchannel"),
texts,
categories,
gain: 0.35,
min_speech_ms: 1_500,
interval_ms: 3_000,
max_per_turn: 2,
jitter_ms: 1_200,
pause_min_ms: 96,
pause_max_ms: 260,
prob: 0.65,
volume_jitter: 0.2,
max_per_turn: 3,
}
}
}
/// Default backchannel phrases (Spanish, matching the shipped clip set), with
/// Drama-3 style tags and a category prefix. Used when `BACKCHANNEL_TEXTS` is
/// unset.
pub const DEFAULT_BACKCHANNEL_TEXTS: &[&str] = &[
"ack::[calm] Mm-hmm...",
"ack::[attentive] Ajá...",
"agree::[agreeing] Claro...",
"ack::[listening] Ya veo...",
"ack::[attentive] Sí, sí...",
"ack::[calm] Entiendo...",
"agree::[relaxed] Vale...",
"ack::[thoughtful] Ah, ya...",
];
/// Split a raw `*_TEXTS` value into `(phrases, categories)`.
///
/// One phrase per line; `|` is also accepted as a separator so the list can
/// live on a single line. A line may carry a `category::` prefix (e.g.
/// `agree::[agreeing] Claro...`) used by the scheduler to vary the kind of
/// acknowledgment through the turn; without it the phrase is an `ack`.
fn parse_texts(raw: &str) -> (Vec<String>, Vec<String>) {
let mut texts = Vec::new();
let mut categories = Vec::new();
for line in raw.split(['\n', '|']) {
let line = line.trim();
if line.is_empty() {
continue;
}
let (category, phrase) = match line.split_once("::") {
Some((c, p)) => (c.trim(), p.trim()),
None => (DEFAULT_CATEGORY, line),
};
if phrase.is_empty() {
continue;
}
texts.push(phrase.to_string());
categories.push(if category.is_empty() {
DEFAULT_CATEGORY.to_string()
} else {
category.to_string()
});
}
(texts, categories)
}
impl BackchannelSettings {
/// Build the settings from environment variables, falling back to defaults.
pub fn from_env() -> Self {
@@ -58,19 +149,99 @@ impl BackchannelSettings {
s.dir = PathBuf::from(p);
}
}
if let Ok(t) = std::env::var("BACKCHANNEL_TEXTS") {
let (texts, categories) = parse_texts(&t);
if !texts.is_empty() {
s.texts = texts;
s.categories = categories;
}
}
s.enabled = env_bool("BACKCHANNEL_ENABLED", s.enabled);
s.gain = env_f32("BACKCHANNEL_GAIN", s.gain);
s.min_speech_ms = env_u64("BACKCHANNEL_MIN_SPEECH_MS", s.min_speech_ms);
s.interval_ms = env_u64("BACKCHANNEL_INTERVAL_MS", s.interval_ms);
s.jitter_ms = env_u64("BACKCHANNEL_JITTER_MS", s.jitter_ms);
s.pause_min_ms = env_u64("BACKCHANNEL_PAUSE_MIN_MS", s.pause_min_ms);
s.pause_max_ms = env_u64("BACKCHANNEL_PAUSE_MAX_MS", s.pause_max_ms);
s.prob = env_f32("BACKCHANNEL_PROB", s.prob);
s.volume_jitter = env_f32("BACKCHANNEL_VOLUME_JITTER", s.volume_jitter);
s.max_per_turn = env_u32("BACKCHANNEL_MAX_PER_TURN", s.max_per_turn);
s.gain = s.gain.clamp(0.0, 1.0);
// Never cue before the turn has meaningfully started.
s.interval_ms = s.interval_ms.max(s.min_speech_ms);
s.prob = s.prob.clamp(0.0, 1.0);
s.volume_jitter = s.volume_jitter.clamp(0.0, 1.0);
// Keep the spacing and the micro-pause window coherent.
s.interval_ms = s.interval_ms.max(500);
if s.pause_max_ms < s.pause_min_ms {
s.pause_max_ms = s.pause_min_ms;
}
s
}
}
/// Tiny xorshift64 generator for timing jitter, the probabilistic gate and the
/// volume drift. The backchannel path is not hot enough to warrant more.
struct XorShift64(u64);
impl XorShift64 {
fn new(seed: u64) -> Self {
// OR 1 so the state is never zero (which would make xorshift degenerate).
Self(seed | 1)
}
fn next_u64(&mut self) -> u64 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
self.0 = x;
x
}
/// Uniform in `[0, 1)`.
fn frac(&mut self) -> f32 {
(self.next_u64() >> 40) as f32 / (1u64 << 24) as f32
}
/// Uniform integer in `[lo, hi]` inclusive (`lo` when `hi <= lo`).
fn range(&mut self, lo: u64, hi: u64) -> u64 {
if hi <= lo {
lo
} else {
lo + self.next_u64() % (hi - lo + 1)
}
}
}
/// One synthesized clip, with the category of its source phrase.
struct ClipRef {
name: String,
category: String,
}
/// Map a phrase list (and its categories) onto the clip names of `bank`.
///
/// Clips are named `{prefix}{index}` in list order and `FillerBank` keeps that
/// order, so the i-th clip matches the i-th phrase — and thus the i-th
/// category. A bank that does not match the list (custom clips) leaves the
/// mapping short; the scheduler then falls back to a plain random pick.
fn map_clips(bank: &FillerBank, texts: &[String], categories: &[String]) -> Vec<ClipRef> {
let names = bank.names();
texts
.iter()
.enumerate()
.filter_map(|(i, _)| {
names.get(i).map(|name| ClipRef {
name: name.clone(),
category: categories
.get(i)
.cloned()
.unwrap_or_else(|| DEFAULT_CATEGORY.to_string()),
})
})
.collect()
}
/// Per-session backchannel decision logic.
///
/// Fed by the VAD: [`on_speech_start`](Self::on_speech_start) when a user turn
@@ -79,33 +250,112 @@ impl BackchannelSettings {
pub struct BackchannelScheduler {
bank: Arc<FillerBank>,
settings: BackchannelSettings,
/// Round-robin cursor over the clip bank.
next_index: usize,
/// Clip name + category per bank entry, in the order of `settings.texts`.
clips: Vec<ClipRef>,
/// VAD end-of-utterance silence window (`VAD_MIN_SILENCE_MS`), used to cap
/// the micro-pause window so a cue never lands on the pause that closes the
/// turn.
vad_min_silence_ms: u32,
/// When the current user turn started (only set while speaking).
turn_start: Option<Instant>,
/// When the last backchannel was emitted in this turn.
/// When the last cue was emitted in this turn.
last_emitted: Option<Instant>,
/// Backchannels emitted in the current turn.
/// Cues emitted in the current turn.
emitted: u32,
/// Earliest instant the first cue of this turn may be played (randomized).
first_cue_at: Option<Instant>,
/// Randomized gap to wait after the previous cue before the next one.
next_gap_ms: u64,
/// Whether the current micro-pause was already evaluated (cue or skip).
pause_settled: bool,
/// Random depth (ms) at which to react inside the current micro-pause, so
/// cues do not all land at exactly `pause_min_ms`. Drawn once per pause.
pause_target_ms: Option<u64>,
/// Categories already used in this turn, for a natural progression.
used_categories: Vec<String>,
/// Category of the last cue (avoid chaining two cues of the same kind).
last_category: Option<String>,
/// Names of the most recently used clips (anti-repetition), kept across
/// turns so the same cue never plays twice in a row.
recent: VecDeque<String>,
/// RNG for jitter, the probabilistic gate and the volume drift.
rng: XorShift64,
}
impl BackchannelScheduler {
pub fn new(bank: Arc<FillerBank>, settings: BackchannelSettings) -> Self {
/// Create a scheduler for one session. `vad_min_silence_ms` is the VAD EOU
/// window (`VAD_MIN_SILENCE_MS`); it bounds the micro-pause window.
pub fn new(
bank: Arc<FillerBank>,
settings: BackchannelSettings,
vad_min_silence_ms: u32,
) -> Self {
let clips = map_clips(&bank, &settings.texts, &settings.categories);
let seed = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0x9E37_79B9_7F4A_7C15);
// One line so the effective policy (and the parsed categories) is
// visible in the log — the knobs are numerous and tuned by ear.
let categories: Vec<&str> = clips.iter().map(|c| c.category.as_str()).collect();
info!(
clips = clips.len(),
categories = ?categories,
pause_ms = format!("{}..{}", settings.pause_min_ms, settings.pause_max_ms),
first_cue_ms = settings.min_speech_ms,
interval_ms = settings.interval_ms,
jitter_ms = settings.jitter_ms,
prob = settings.prob,
volume_jitter = settings.volume_jitter,
max_per_turn = settings.max_per_turn,
"backchannel scheduler ready"
);
Self {
bank,
settings,
next_index: 0,
clips,
vad_min_silence_ms,
turn_start: None,
last_emitted: None,
emitted: 0,
first_cue_at: None,
next_gap_ms: 0,
pause_settled: false,
pause_target_ms: None,
used_categories: Vec::new(),
last_category: None,
recent: VecDeque::new(),
rng: XorShift64::new(seed),
}
}
/// Point the scheduler at another language's bank: replace the clip list and
/// the phrase/category mapping. Called between turns when the detected user
/// language changes (see [`crate::lang`]). The per-turn counters are left as
/// they are — the next VAD speech start resets them anyway.
pub fn retarget(&mut self, bank: Arc<FillerBank>, texts: &[String], categories: &[String]) {
self.clips = map_clips(&bank, texts, categories);
self.bank = bank;
}
/// Begin a new user turn (VAD speech start).
pub fn on_speech_start(&mut self) {
self.turn_start = Some(Instant::now());
let now = Instant::now();
self.turn_start = Some(now);
self.last_emitted = None;
self.emitted = 0;
self.pause_settled = false;
self.pause_target_ms = None;
self.used_categories.clear();
self.last_category = None;
// Randomize when the first cue becomes eligible: base delay plus a
// random 0..jitter offset, so turns don't all cue at the same instant.
let jitter = self.rng.range(0, self.settings.jitter_ms);
self.first_cue_at = Some(now + Duration::from_millis(self.settings.min_speech_ms + jitter));
self.next_gap_ms = self.settings.interval_ms;
}
/// Close the current user turn (VAD speech end).
@@ -113,35 +363,176 @@ impl BackchannelScheduler {
self.turn_start = None;
self.last_emitted = None;
self.emitted = 0;
self.first_cue_at = None;
self.pause_settled = false;
self.pause_target_ms = None;
self.used_categories.clear();
self.last_category = None;
}
/// Emit a backchannel into `mixer` if the policy allows it right now.
///
/// Must be called while the user is speaking and the agent is silent.
/// Returns the clip name when one was emitted.
pub fn maybe_emit(&mut self, mixer: &crate::audio::AudioMixer) -> Option<String> {
/// `pause_ms` is the length of the micro-pause the user is currently in
/// ([`VadDetector::silence_ms`](crate::vad::VadDetector::silence_ms), 0
/// while voice is ongoing). Returns the clip `(name, category)` when one
/// was emitted.
pub fn maybe_emit(
&mut self,
mixer: &crate::audio::AudioMixer,
pause_ms: u64,
) -> Option<(String, String)> {
if !self.settings.enabled {
return None;
}
let start = self.turn_start?;
if self.emitted >= self.settings.max_per_turn {
if self.turn_start.is_none() || self.emitted >= self.settings.max_per_turn {
return None;
}
if start.elapsed() < Duration::from_millis(self.settings.min_speech_ms) {
// Cue only inside a micro-pause: never mid-word, and never on the pause
// that is long enough to close the utterance (that one belongs to the
// reply). A zero-length pause means voice is ongoing: re-arm so the
// *next* micro-pause is evaluated.
if pause_ms == 0 {
self.pause_settled = false;
self.pause_target_ms = None;
return None;
}
let max = self.pause_max();
if pause_ms > max {
// The pause ran past the window (it is about to close the turn):
// abandon it without firing.
self.pause_settled = true;
return None;
}
if self.pause_settled {
return None;
}
// At the start of a new micro-pause pick the depth to react at
// uniformly inside the window, so the cue does not always land at
// exactly `pause_min_ms` — the reaction point varies pause to pause.
let target = match self.pause_target_ms {
Some(t) => t,
None => {
let lo = self.settings.pause_min_ms.min(max);
let t = self.rng.range(lo, max);
self.pause_target_ms = Some(t);
t
}
};
if pause_ms < target {
return None;
}
// Randomized turn timing: hold off until the first-cue instant.
if let Some(first) = self.first_cue_at {
if Instant::now() < first {
return None;
}
}
// Randomized spacing since the previous cue.
if let Some(last) = self.last_emitted {
if last.elapsed() < Duration::from_millis(self.settings.interval_ms) {
if last.elapsed() < Duration::from_millis(self.next_gap_ms) {
return None;
}
}
let (name, samples) = self.bank.nth(self.next_index)?;
let name = name.to_string();
mixer.backchannel_queue.push(samples);
self.next_index = self.next_index.wrapping_add(1);
// Probabilistic gate: stay silent for some qualifying pauses, so the
// cadence matches how a listener nods along instead of a metronome.
if self.rng.frac() > self.settings.prob {
self.pause_settled = true;
return None;
}
let (name, category, samples) = self.select()?;
let samples = self.apply_volume(&samples);
mixer.backchannel_queue.push(&samples);
self.pause_settled = true;
self.last_emitted = Some(Instant::now());
self.emitted += 1;
Some(name)
self.used_categories.push(category.clone());
self.last_category = Some(category.clone());
self.recent.push_back(name.clone());
while self.recent.len() > 2 {
self.recent.pop_front();
}
// Randomize the next gap around the configured interval.
let half = self.settings.jitter_ms / 2;
let lo = self.settings.interval_ms.saturating_sub(half).max(500);
let hi = self.settings.interval_ms + half;
self.next_gap_ms = self.rng.range(lo, hi);
Some((name, category))
}
/// Upper bound of the micro-pause window: the configured maximum, capped so
/// it stays below the VAD end-of-utterance window (minus one analysis
/// window) — otherwise the cue would land on the pause that ends the turn.
fn pause_max(&self) -> u64 {
let vad_cap = (self.vad_min_silence_ms.saturating_sub(32) as u64).max(1);
self.settings.pause_max_ms.min(vad_cap)
}
/// Pick a clip honouring the category progression and anti-repetition, then
/// falling back to a plain random clip when the bank does not match the
/// phrase list. Returns `(name, category, samples)`.
fn select(&mut self) -> Option<(String, String, Vec<i16>)> {
if let Some(idx) = self.select_index() {
let clip = &self.clips[idx];
if let Some(samples) = self.bank.get(&clip.name) {
return Some((clip.name.clone(), clip.category.clone(), samples.clone()));
}
}
self.bank
.random()
.map(|(n, s)| (n.to_string(), DEFAULT_CATEGORY.to_string(), s.clone()))
}
/// Index of the clip to play: prefer a category other than the last one,
/// then categories not yet used this turn (a progression from "listening"
/// to "agreeing"), then clips not used recently, and pick uniformly at
/// random among whatever remains.
fn select_index(&mut self) -> Option<usize> {
if self.clips.is_empty() {
return None;
}
let all: Vec<usize> = (0..self.clips.len()).collect();
let not_last: Vec<usize> = all
.iter()
.copied()
.filter(|&i| Some(&self.clips[i].category) != self.last_category.as_ref())
.collect();
let pool = if not_last.is_empty() { all } else { not_last };
let fresh: Vec<usize> = pool
.iter()
.copied()
.filter(|&i| !self.used_categories.contains(&self.clips[i].category))
.collect();
let pool = if fresh.is_empty() { pool } else { fresh };
let unrepeated: Vec<usize> = pool
.iter()
.copied()
.filter(|&i| !self.recent.contains(&self.clips[i].name))
.collect();
let pool = if unrepeated.is_empty() { pool } else { unrepeated };
Some(pool[(self.rng.next_u64() % pool.len() as u64) as usize])
}
/// Scale a clip by a random ±`volume_jitter` factor so the cues do not all
/// sit at exactly the same level.
fn apply_volume(&mut self, samples: &[i16]) -> Vec<i16> {
let jitter = self.settings.volume_jitter;
if jitter <= 0.0 {
return samples.to_vec();
}
let scale = 1.0 + (self.rng.frac() * 2.0 - 1.0) * jitter;
samples
.iter()
.map(|&s| ((s as f32) * scale).clamp(i16::MIN as f32, i16::MAX as f32) as i16)
.collect()
}
}
src/orchestrator/turn_manager.rs
+440 -65
View File
@@ -9,13 +9,17 @@
//! - **Without LLM**: transcript → TTS (the original pass-through behaviour).
use crate::audio::{AudioMixer, SampleRingBuffer};
use crate::audit::SessionAudit;
use crate::config::Config;
use crate::lang::{self, LangBanks, LanguageState};
use crate::llm::{ChatMessage, LlmClient};
use crate::stt::{Tokenizer, Transcript};
use crate::tools::ToolCallStripper;
use crate::tts::TtsEngine;
use crate::{assets, orchestrator::backchannel::DEFAULT_CATEGORY};
use serde_json::json;
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
@@ -62,12 +66,15 @@ fn pump(src: &SampleRingBuffer, dst: &SampleRingBuffer) -> usize {
}
/// Preloaded PCM samples keyed by name (e.g. "uhm"). Used for the filler bank
/// injected at end-of-utterance (played round-robin) and, from a different
/// directory, for the backchannel clips played while the user speaks.
/// injected at end-of-utterance (played round-robin), for the backchannel clips
/// played while the user speaks, and for the start-filler bank (both played in
/// random order).
pub struct FillerBank {
samples: HashMap<String, Vec<i16>>,
/// Names in a stable order, for round-robin selection.
names: Vec<String>,
/// xorshift64 state for [`Self::random`] (seeded from the clock at load).
rng: AtomicU64,
}
impl FillerBank {
@@ -107,17 +114,37 @@ impl FillerBank {
let mut names: Vec<String> = samples.keys().cloned().collect();
names.sort();
Self { samples, names }
// Seed the xorshift64 state from the wall clock (OR 1 so it is never
// zero, which would make xorshift degenerate).
let seed = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0x9E37_79B9_7F4A_7C15)
| 1;
Self {
samples,
names,
rng: AtomicU64::new(seed),
}
}
/// Get the samples for a clip by name. Kept for reference/debugging.
#[allow(dead_code)]
/// Get the samples for a clip by name. Used by the backchannel to play a
/// specific (category-selected) clip instead of a uniform random one.
pub fn get(&self, name: &str) -> Option<&Vec<i16>> {
self.samples.get(name)
}
/// Clip names in the stable (sorted) order used by [`Self::nth`]. The
/// backchannel uses it to map the phrase list (and its categories) onto the
/// synthesized clips, since clips are named `{prefix}{index}` in list order.
pub fn names(&self) -> &[String] {
&self.names
}
/// Clip at position `index` (wrapping), with its name. Used for the
/// round-robin filler and backchannel selection.
/// round-robin end-of-utterance filler.
pub fn nth(&self, index: usize) -> Option<(&str, &Vec<i16>)> {
if self.names.is_empty() {
return None;
@@ -126,6 +153,23 @@ impl FillerBank {
self.samples.get(name).map(|s| (name.as_str(), s))
}
/// Clip chosen uniformly at random, with its name. Used by the start filler
/// and the backchannel so the cue varies from turn to turn.
pub fn random(&self) -> Option<(&str, &Vec<i16>)> {
if self.names.is_empty() {
return None;
}
// xorshift64 step over the shared state.
let mut x = self.rng.load(Ordering::Relaxed);
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
self.rng.store(x, Ordering::Relaxed);
let name = &self.names[(x % self.names.len() as u64) as usize];
self.samples.get(name).map(|s| (name.as_str(), s))
}
/// Whether the bank holds no clips.
pub fn is_empty(&self) -> bool {
self.names.is_empty()
@@ -136,7 +180,10 @@ impl FillerBank {
pub struct TurnManager {
mixer: AudioMixer,
tts: Arc<dyn TtsEngine>,
fillers: Arc<FillerBank>,
/// Active-language clip banks, shared with the mic task (backchannel). The
/// end-of-turn and start fillers are read from here so all three cues
/// follow the same language — see [`crate::lang`].
langs: Arc<LanguageState>,
/// Optional tokenizer to decode STT token IDs into text. If `None`, a
/// placeholder string is sent to the TTS (useful for testing the pipeline
/// without a real tokenizer).
@@ -154,22 +201,31 @@ pub struct TurnManager {
/// Outbound text-event channel (JSON strings) consumed by the session
/// sender task and forwarded to the browser (chat box).
out_tx: Option<mpsc::UnboundedSender<String>>,
/// User turns seen so far, used to schedule the periodic language check.
lang_turns: AtomicU64,
/// Whether a language-detection call is currently in flight (one at a time).
lang_detecting: Arc<AtomicBool>,
/// Per-session audit sink: the language switches are recorded here (every
/// other event reaches the log through the outbound tap in the session
/// sender — see [`crate::audit`]).
audit: Arc<SessionAudit>,
}
impl TurnManager {
pub fn new(
mixer: AudioMixer,
tts: Arc<dyn TtsEngine>,
fillers: Arc<FillerBank>,
langs: Arc<LanguageState>,
tokenizer: Option<Tokenizer>,
llm: Option<Arc<LlmClient>>,
config: Arc<Config>,
out_tx: Option<mpsc::UnboundedSender<String>>,
audit: Arc<SessionAudit>,
) -> Self {
Self {
mixer,
tts,
fillers,
langs,
tokenizer,
cancel: Arc::new(parking_lot::Mutex::new(None)),
filler_cursor: AtomicUsize::new(0),
@@ -177,13 +233,76 @@ impl TurnManager {
config,
history: parking_lot::Mutex::new(Vec::new()),
out_tx,
lang_turns: AtomicU64::new(0),
lang_detecting: Arc::new(AtomicBool::new(false)),
audit,
}
}
/// Pick the next filler clip round-robin from the bank.
/// Pick the next filler clip round-robin from the bank of the active
/// language.
fn next_filler(&self) -> Option<(String, Vec<i16>)> {
let banks = self.langs.current();
let i = self.filler_cursor.fetch_add(1, Ordering::Relaxed);
self.fillers.nth(i).map(|(name, s)| (name.to_string(), s.clone()))
banks
.filler
.nth(i)
.map(|(name, s)| (name.to_string(), s.clone()))
}
/// Schedule the *start filler* acknowledgment on the mixer's dedicated
/// start-filler channel.
///
/// Like [`Self::schedule_filler`] this is a latency mask, but with its own
/// (shorter) delay, `START_FILLER_GO_DELAY`: the cue is only queued when the
/// first response audio is still **late** after that window — if the first
/// audio chunk has already reached the mixer (even if it is still inside the
/// pre-buffer and playback has not started), the agent is already talking
/// and the cue is dropped. One phrase is picked **at random** from the bank
/// (built from `START_FILLER_TEXTS`), so the cue varies turn to turn.
/// Disabled by `START_FILLER_ENABLED=false`; a barge-in (`cancel`) or a new
/// turn drops the pending cue, and `mixer.flush` clears it once queued.
fn schedule_start_filler(&self, cancel: &CancellationToken) {
if !self.config.start_filler_enabled {
return;
}
// Own the clip: the spawned task must be `'static` (same as the
// round-robin filler above).
let banks = self.langs.current();
let Some((name, samples)) = banks
.start_filler
.random()
.map(|(n, s)| (n.to_string(), s.clone()))
else {
debug!("start filler enabled but no clip loaded — skipping");
return;
};
let delay_ms = self.config.start_filler_go_delay_ms;
let mixer = self.mixer.clone();
let cancel = cancel.clone();
let out_tx = self.out_tx.clone();
tokio::spawn(async move {
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {}
// Barge-in / new turn: never play a stale acknowledgment.
_ = cancel.cancelled() => return,
}
if mixer.tts_audio_arrived() {
debug!(
filler = %name,
delay_ms, "first response audio already arrived — start filler skipped"
);
return;
}
let queued = mixer.start_filler_queue.push(&samples);
debug!(filler = %name, samples = queued, delay_ms, "queued start filler");
if let Some(tx) = &out_tx {
let _ = tx.send(
json!({ "type": "cue", "kind": "start_filler", "name": name }).to_string(),
);
}
});
}
/// Schedule the single end-of-turn filler acknowledgment.
@@ -207,6 +326,7 @@ impl TurnManager {
let delay_ms = self.config.filler_delay_ms;
let mixer = self.mixer.clone();
let cancel = cancel.clone();
let out_tx = self.out_tx.clone();
let samples_len = samples.len();
tokio::spawn(async move {
tokio::select! {
@@ -223,6 +343,11 @@ impl TurnManager {
}
mixer.filler_queue.push(&samples);
debug!(filler = %name, samples = samples_len, delay_ms, "pushed filler (delayed)");
if let Some(tx) = &out_tx {
let _ = tx.send(
json!({ "type": "cue", "kind": "filler", "name": name }).to_string(),
);
}
});
}
@@ -274,12 +399,14 @@ impl TurnManager {
/// 2. Cancel any in-flight reply (barge-in within the turn).
/// 3. Run the LLM (if configured) or send the text straight to the TTS.
///
/// The filler is not queued here: it is scheduled once the response is
/// dispatched, so it only plays if the audio is late (see
/// [`Self::schedule_filler`]).
/// The latency masks are not queued here: both the start filler and the
/// end-of-turn filler are *scheduled* once the response is dispatched, so
/// they only play if the audio is late — see [`Self::schedule_filler`] and
/// [`Self::schedule_start_filler`].
async fn handle_user_turn(&self, text: String) {
self.emit(json!({ "type": "user", "text": text }));
self.maybe_detect_language(&text);
self.cancel_current_tts();
match &self.llm {
@@ -288,11 +415,58 @@ impl TurnManager {
}
}
/// Kick off a language check for `text` when the detection schedule allows
/// it. Runs in the background so the reply is never delayed: the switch (if
/// any) takes effect from the next turn. See [`crate::lang`].
///
/// The first turn is always checked; afterwards a sample is sent only every
/// `LANG_REDETECT_EVERY` turns, so a language change mid-conversation is
/// picked up without paying for a detection call on every turn.
fn maybe_detect_language(&self, text: &str) {
if !self.config.lang_detect_enabled {
return;
}
let Some(llm) = self.llm.clone() else {
return;
};
// Only the backchannel, the end-of-turn filler and the start filler are
// language-aware; with all of them disabled there is nothing to switch.
if !self.config.backchannel.enabled
&& !self.config.filler_enabled
&& !self.config.start_filler_enabled
{
return;
}
if text.chars().count() < self.config.lang_min_chars {
return;
}
let turn = self.lang_turns.fetch_add(1, Ordering::Relaxed);
if turn % self.config.lang_redetect_every != 0 {
return;
}
// One detection at a time: a slow LLM call must not pile up.
if self.lang_detecting.swap(true, Ordering::AcqRel) {
return;
}
let langs = self.langs.clone();
let tts = self.tts.clone();
let config = self.config.clone();
let detecting = self.lang_detecting.clone();
let audit = self.audit.clone();
let text = text.to_string();
tokio::spawn(async move {
switch_language_if_needed(langs, llm, tts, config, &text, audit).await;
detecting.store(false, Ordering::Release);
});
}
/// Run one full LLM turn: stream the reply, chunk it into sentences and
/// synthesize each sentence as soon as it is complete.
async fn run_llm_turn(&self, llm: Arc<LlmClient>, user_text: String) {
// Build the message list from the rolling history.
let messages = {
// Build the message list from the rolling history. It grows across
// tool rounds (assistant call + tool results) but only the *spoken*
// text is written back to the rolling history.
let mut messages = {
let mut hist = self.history.lock();
hist.push(ChatMessage::user(&user_text));
let max = self.config.llm_history_messages.max(2);
@@ -313,6 +487,7 @@ impl TurnManager {
*self.cancel.lock() = Some(cancel.clone());
self.mixer.mark_stream_start();
self.schedule_filler(&cancel);
self.schedule_start_filler(&cancel);
debug!(
endpoint = llm.endpoint(),
model = llm.model(),
@@ -414,63 +589,129 @@ impl TurnManager {
}
});
// LLM producer: streams clean text deltas into a channel.
let (delta_tx, mut delta_rx) = mpsc::unbounded_channel::<String>();
let stream_mode = self.config.llm_stream;
let llm_cancel = cancel.clone();
let llm_task = tokio::spawn(async move {
if stream_mode {
llm.chat_stream(&messages, &llm_cancel, delta_tx).await
} else {
let res = llm.chat(&messages, &llm_cancel).await;
if let Ok(text) = &res {
let _ = delta_tx.send(text.clone());
}
res
}
});
// Tool-calling loop. Each round streams one reply; tool call blocks are
// stripped from the spoken/displayed text (the model emits them as
// text — see `crate::tools`), the calls run over HTTP, and the results
// are handed back for the next round. Bounded by
// `LLM_TOOL_MAX_ITERATIONS`. With no tools configured the loop runs
// exactly once.
let registry = llm.tools().cloned();
let max_rounds = self.config.llm_tool_max_iterations.max(1);
let mut spoken = String::new();
let mut round = 0usize;
loop {
// LLM producer: streams clean text deltas into a channel.
let (delta_tx, mut delta_rx) = mpsc::unbounded_channel::<String>();
let stream_mode = self.config.llm_stream;
let llm_cancel = cancel.clone();
let llm_round = llm.clone();
let round_messages = messages.clone();
let llm_task = tokio::spawn(async move {
if stream_mode {
llm_round.chat_stream(&round_messages, &llm_cancel, delta_tx).await
} else {
let res = llm_round.chat(&round_messages, &llm_cancel).await;
if let Ok(text) = &res {
let _ = delta_tx.send(text.clone());
}
res
}
});
// Consume deltas: emit `assistant_delta` events and feed the
// splitter, so the TTS starts as soon as the first few words are
// available. The stripper holds back tool-call blocks so they are
// never spoken nor shown.
let mut splitter = SentenceSplitter::new(
self.config.llm_stream_first_chunk_chars,
self.config.llm_stream_chunk_chars,
);
let mut stripper = ToolCallStripper::new();
let mut raw = String::new();
while let Some(delta) = delta_rx.recv().await {
if cancel.is_cancelled() {
break;
}
raw.push_str(&delta);
let visible = stripper.push(&delta);
if visible.is_empty() {
continue;
}
self.emit(json!({ "type": "assistant_delta", "text": visible }));
spoken.push_str(&visible);
for sentence in splitter.push(&visible) {
if sentence_tx.send(sentence).is_err() {
break;
}
}
}
if !cancel.is_cancelled() {
let tail = stripper.finish();
if !tail.is_empty() {
self.emit(json!({ "type": "assistant_delta", "text": tail }));
for sentence in splitter.push(&tail) {
if sentence_tx.send(sentence).is_err() {
break;
}
}
spoken.push_str(&tail);
}
if let Some(rest) = splitter.finish() {
let _ = sentence_tx.send(rest);
}
}
match llm_task.await {
Ok(Ok(_)) => {}
Ok(Err(e)) => {
error!(error = %e, "LLM request failed");
self.emit(json!({ "type": "error", "text": e.to_string() }));
}
Err(e) => {
error!(error = %e, "LLM task join failed");
self.emit(json!({ "type": "error", "text": format!("LLM task failed: {e}") }));
}
}
// Consume deltas: emit `assistant_delta` events and feed the splitter,
// so the TTS starts as soon as the first few words are available.
let mut splitter = SentenceSplitter::new(
self.config.llm_stream_first_chunk_chars,
self.config.llm_stream_chunk_chars,
);
let mut full = String::new();
while let Some(delta) = delta_rx.recv().await {
if cancel.is_cancelled() {
break;
}
self.emit(json!({ "type": "assistant_delta", "text": delta }));
full.push_str(&delta);
for sentence in splitter.push(&delta) {
if sentence_tx.send(sentence).is_err() {
break;
}
}
}
if !cancel.is_cancelled() {
if let Some(rest) = splitter.finish() {
let _ = sentence_tx.send(rest);
let calls = stripper.take_calls();
if calls.is_empty() {
break;
}
let Some(registry) = registry.as_ref() else {
warn!("model emitted a tool call but no tools are configured — ignoring");
break;
};
if round + 1 >= max_rounds {
warn!(max_rounds, "tool-call iteration limit reached — stopping");
break;
}
round += 1;
// Record the model's own call followed by each result, so the next
// round sees the exchange (standard OpenAI tool round-trip).
messages.push(ChatMessage::assistant(raw.clone()));
for call in &calls {
info!(tool = %call.name, round, "executing tool call");
self.emit(json!({
"type": "tool_call",
"name": &call.name,
"arguments": &call.arguments,
}));
let result = registry.execute(call).await;
messages.push(ChatMessage::tool(result));
}
}
drop(sentence_tx);
let _ = tts_task.await;
match llm_task.await {
Ok(Ok(_)) => {}
Ok(Err(e)) => {
error!(error = %e, "LLM request failed");
self.emit(json!({ "type": "error", "text": e.to_string() }));
}
Err(e) => {
error!(error = %e, "LLM task join failed");
self.emit(json!({ "type": "error", "text": format!("LLM task failed: {e}") }));
}
}
if !cancel.is_cancelled() && !full.trim().is_empty() {
self.history.lock().push(ChatMessage::assistant(full.clone()));
if !cancel.is_cancelled() && !spoken.trim().is_empty() {
self.history.lock().push(ChatMessage::assistant(spoken.clone()));
self.emit(json!({ "type": "assistant_end" }));
}
@@ -500,6 +741,12 @@ impl TurnManager {
self.tts.set_reference(audio_path, text);
}
/// Set the gain of the mixer's external audio channel (URL / radio /
/// YouTube background source). See [`crate::media`].
pub fn set_external_gain(&self, gain: f32) {
self.mixer.set_external_gain(gain);
}
/// Synthesize `text` directly with the TTS engine, bypassing the LLM.
/// Used by the playground's "Speak" box to test the TTS pipeline.
pub async fn speak(&self, text: &str) {
@@ -529,6 +776,7 @@ impl TurnManager {
let text_owned = text.to_string();
self.mixer.mark_stream_start();
self.schedule_filler(&cancel);
self.schedule_start_filler(&cancel);
debug!(
engine = tts.name(),
sample_rate = tts.sample_rate(),
@@ -560,6 +808,133 @@ impl TurnManager {
}
}
/// Detect the language of `text` and, when it differs from the active one,
/// build and publish the clip banks for it.
///
/// The phrase list comes from the built-in tables ([`crate::lang`]) for the
/// common languages, or from the LLM when `LANG_LLM_FALLBACK_ENABLED` is set.
/// Clips are synthesized only when missing — [`assets::ensure_lang_backchannel_clips`]
/// keeps a per-language index — and live in their own subdirectory, so no other
/// language is touched. Switching back to the default language restores the
/// base banks and needs no synthesis at all.
async fn switch_language_if_needed(
langs: Arc<LanguageState>,
llm: Arc<LlmClient>,
tts: Arc<dyn TtsEngine>,
config: Arc<Config>,
text: &str,
audit: Arc<SessionAudit>,
) {
let Some(lang) = lang::detect_language(&llm, text).await else {
return;
};
let current = langs.current();
if lang == current.lang {
return;
}
if lang == config.lang_default {
info!(lang = %lang, "user language back to the default — restoring the base clip banks");
audit.event(
"language_switch",
json!({ "from": current.lang, "to": lang, "generated": false }),
);
langs.set((*langs.base()).clone());
return;
}
// Phrase lists: built-in table first, LLM fallback otherwise.
let backchannel = match lang::backchannel_texts(&lang) {
Some(pairs) => pairs,
None if config.lang_llm_fallback_enabled => {
match lang::generate_backchannel_texts(&llm, &lang).await {
Some(phrases) => phrases
.into_iter()
.map(|p| (DEFAULT_CATEGORY.to_string(), p))
.collect(),
None => return,
}
}
None => return,
};
let filler = match lang::filler_texts(&lang) {
Some(texts) => texts,
None if config.lang_llm_fallback_enabled => {
match lang::generate_filler_texts(&llm, &lang).await {
Some(texts) => texts,
None => return,
}
}
None => return,
};
// Start filler is optional: when the language has no phrase list (and the
// LLM fallback is unavailable) the cue simply stays silent for it, without
// preventing the switch of the banks above.
let start_filler = if !config.start_filler_enabled {
None
} else {
match lang::start_filler_texts(&lang) {
Some(texts) => Some(texts),
None if config.lang_llm_fallback_enabled => {
lang::generate_start_filler_texts(&llm, &lang).await
}
None => None,
}
};
info!(
from = %current.lang,
to = %lang,
backchannel = backchannel.len(),
filler = filler.len(),
start_filler = start_filler.as_ref().map(|t| t.len()).unwrap_or(0),
"user language changed — generating the clip banks for it"
);
let phrases: Vec<String> = backchannel.iter().map(|(_, p)| p.clone()).collect();
let categories: Vec<String> = backchannel.iter().map(|(c, _)| c.clone()).collect();
// Idempotent: an unchanged phrase list is skipped via the per-directory
// `texts.txt` index, so an already-generated language is just reloaded.
assets::ensure_lang_backchannel_clips(&config.backchannel.dir, &lang, &phrases, tts.as_ref())
.await;
assets::ensure_lang_filler_clips(&config.fillers_dir, &lang, &filler, tts.as_ref()).await;
if let Some(texts) = &start_filler {
assets::ensure_lang_start_filler_clips(
&config.start_filler_dir,
&lang,
texts,
tts.as_ref(),
)
.await;
}
let backchannel_bank = Arc::new(FillerBank::load(
&assets::lang_dir(&config.backchannel.dir, &lang),
"backchannel",
));
let filler_bank = Arc::new(FillerBank::load(
&assets::lang_dir(&config.fillers_dir, &lang),
"filler",
));
let start_filler_bank = Arc::new(FillerBank::load(
&assets::lang_dir(&config.start_filler_dir, &lang),
"start-filler",
));
langs.set(LangBanks {
lang,
backchannel: backchannel_bank,
backchannel_texts: phrases,
backchannel_categories: categories,
filler: filler_bank,
start_filler: start_filler_bank,
});
audit.event(
"language_switch",
json!({ "from": current.lang, "to": langs.current().lang, "generated": true }),
);
}
/// Spawn an orchestrator loop that consumes transcripts from `rx` and drives
/// the turn manager.
pub fn spawn_orchestrator(