//! Backchannel: brief acknowledgments emitted *while the user is speaking*. //! //! This is distinct from a filler (see [`TurnManager`](crate::orchestrator::TurnManager)), //! which is played once the user stops, at end-of-utterance. A backchannel is a //! short "mm-hmm" / "ajá" mixed in at a reduced gain to signal that the agent is //! listening, without taking the turn. It is emitted only while the agent itself //! is silent (otherwise the user is being interrupted). //! //! To sound natural rather than metronomic, a cue is played only when the user //! is in a **micro-pause** ([`VadDetector::silence_ms`](crate::vad::VadDetector::silence_ms)), //! at a **random depth** inside that pause, its timing is randomized, a //! probabilistic gate skips some qualifying pauses, //! recently used clips are not repeated, the volume drifts slightly, and the //! phrases are grouped in **categories** that follow a progression through the //! turn (an acknowledgment first, agreement later). //! //! Every behavioural knob is configured from the environment //! ([`BackchannelSettings`]), so the feature can be tuned or disabled without a //! rebuild. use crate::config::{env_bool, env_f32, env_u32, env_u64}; use crate::orchestrator::FillerBank; use std::collections::VecDeque; use std::path::PathBuf; use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::info; /// Category applied to phrases written without an explicit `category::` prefix. pub(crate) const DEFAULT_CATEGORY: &str = "ack"; /// Runtime configuration for the backchannel subsystem, read from `.env`. #[derive(Debug, Clone)] pub struct BackchannelSettings { /// Master switch (`BACKCHANNEL_ENABLED`). pub enabled: bool, /// Directory of `*.raw` PCM clips (`BACKCHANNEL_DIR`). pub dir: PathBuf, /// Phrases spoken by the backchannel bank (`BACKCHANNEL_TEXTS`), one per /// line, with the optional `category::` prefix already stripped. One clip is /// synthesized per phrase at startup. Drama-3 style tags (`[calm]`, …) are /// passed straight through to the TTS. pub texts: Vec, /// Category of each phrase in [`Self::texts`] (parallel array). Lines /// written as `category::phrase` set it; plain lines default to `ack`. pub categories: Vec, /// Mix gain applied to backchannel clips (`BACKCHANNEL_GAIN`). Kept low so /// the cue stays in the background. pub gain: f32, /// Confirmed speech required before the first cue may be considered /// (`BACKCHANNEL_MIN_SPEECH_MS`). pub min_speech_ms: u64, /// Base spacing between two cues of the same turn /// (`BACKCHANNEL_INTERVAL_MS`); randomized by ±`jitter_ms`/2. pub interval_ms: u64, /// Timing jitter (`BACKCHANNEL_JITTER_MS`): delays the first cue by /// `0..jitter_ms` after `min_speech_ms` and spreads the spacing. pub jitter_ms: u64, /// Lower bound of the micro-pause window (`BACKCHANNEL_PAUSE_MIN_MS`): the /// earliest depth, inside a pause, at which the user is likely done with the /// current word. The actual reaction depth is drawn at random in /// `[pause_min_ms, pause_max_ms]`. pub pause_min_ms: u64, /// Upper bound of the micro-pause window (`BACKCHANNEL_PAUSE_MAX_MS`): the /// latest depth a cue may react at. Above this the utterance is about to /// close, so a cue would land on the reply. pub pause_max_ms: u64, /// Probability of cueing when a micro-pause qualifies (`BACKCHANNEL_PROB`). pub prob: f32, /// Random per-cue volume scaling as a ± fraction /// (`BACKCHANNEL_VOLUME_JITTER`). pub volume_jitter: f32, /// Maximum cues per user turn (`BACKCHANNEL_MAX_PER_TURN`). pub max_per_turn: u32, } impl Default for BackchannelSettings { fn default() -> Self { let (texts, categories) = parse_texts(&DEFAULT_BACKCHANNEL_TEXTS.join("\n")); Self { enabled: true, dir: PathBuf::from("assets/backchannel"), texts, categories, gain: 0.35, min_speech_ms: 1_500, interval_ms: 3_000, jitter_ms: 1_200, pause_min_ms: 96, pause_max_ms: 260, prob: 0.65, volume_jitter: 0.2, max_per_turn: 3, } } } /// Default backchannel phrases (Spanish, matching the shipped clip set), with /// Drama-3 style tags and a category prefix. Used when `BACKCHANNEL_TEXTS` is /// unset. pub const DEFAULT_BACKCHANNEL_TEXTS: &[&str] = &[ "ack::[calm] Mm-hmm...", "ack::[attentive] Ajá...", "agree::[agreeing] Claro...", "ack::[listening] Ya veo...", "ack::[attentive] Sí, sí...", "ack::[calm] Entiendo...", "agree::[relaxed] Vale...", "ack::[thoughtful] Ah, ya...", ]; /// Split a raw `*_TEXTS` value into `(phrases, categories)`. /// /// One phrase per line; `|` is also accepted as a separator so the list can /// live on a single line. A line may carry a `category::` prefix (e.g. /// `agree::[agreeing] Claro...`) used by the scheduler to vary the kind of /// acknowledgment through the turn; without it the phrase is an `ack`. fn parse_texts(raw: &str) -> (Vec, Vec) { let mut texts = Vec::new(); let mut categories = Vec::new(); for line in raw.split(['\n', '|']) { let line = line.trim(); if line.is_empty() { continue; } let (category, phrase) = match line.split_once("::") { Some((c, p)) => (c.trim(), p.trim()), None => (DEFAULT_CATEGORY, line), }; if phrase.is_empty() { continue; } texts.push(phrase.to_string()); categories.push(if category.is_empty() { DEFAULT_CATEGORY.to_string() } else { category.to_string() }); } (texts, categories) } impl BackchannelSettings { /// Build the settings from environment variables, falling back to defaults. pub fn from_env() -> Self { let mut s = Self::default(); if let Ok(p) = std::env::var("BACKCHANNEL_DIR") { if !p.is_empty() { s.dir = PathBuf::from(p); } } if let Ok(t) = std::env::var("BACKCHANNEL_TEXTS") { let (texts, categories) = parse_texts(&t); if !texts.is_empty() { s.texts = texts; s.categories = categories; } } s.enabled = env_bool("BACKCHANNEL_ENABLED", s.enabled); s.gain = env_f32("BACKCHANNEL_GAIN", s.gain); s.min_speech_ms = env_u64("BACKCHANNEL_MIN_SPEECH_MS", s.min_speech_ms); s.interval_ms = env_u64("BACKCHANNEL_INTERVAL_MS", s.interval_ms); s.jitter_ms = env_u64("BACKCHANNEL_JITTER_MS", s.jitter_ms); s.pause_min_ms = env_u64("BACKCHANNEL_PAUSE_MIN_MS", s.pause_min_ms); s.pause_max_ms = env_u64("BACKCHANNEL_PAUSE_MAX_MS", s.pause_max_ms); s.prob = env_f32("BACKCHANNEL_PROB", s.prob); s.volume_jitter = env_f32("BACKCHANNEL_VOLUME_JITTER", s.volume_jitter); s.max_per_turn = env_u32("BACKCHANNEL_MAX_PER_TURN", s.max_per_turn); s.gain = s.gain.clamp(0.0, 1.0); s.prob = s.prob.clamp(0.0, 1.0); s.volume_jitter = s.volume_jitter.clamp(0.0, 1.0); // Keep the spacing and the micro-pause window coherent. s.interval_ms = s.interval_ms.max(500); if s.pause_max_ms < s.pause_min_ms { s.pause_max_ms = s.pause_min_ms; } s } } /// Tiny xorshift64 generator for timing jitter, the probabilistic gate and the /// volume drift. The backchannel path is not hot enough to warrant more. struct XorShift64(u64); impl XorShift64 { fn new(seed: u64) -> Self { // OR 1 so the state is never zero (which would make xorshift degenerate). Self(seed | 1) } fn next_u64(&mut self) -> u64 { let mut x = self.0; x ^= x << 13; x ^= x >> 7; x ^= x << 17; self.0 = x; x } /// Uniform in `[0, 1)`. fn frac(&mut self) -> f32 { (self.next_u64() >> 40) as f32 / (1u64 << 24) as f32 } /// Uniform integer in `[lo, hi]` inclusive (`lo` when `hi <= lo`). fn range(&mut self, lo: u64, hi: u64) -> u64 { if hi <= lo { lo } else { lo + self.next_u64() % (hi - lo + 1) } } } /// One synthesized clip, with the category of its source phrase. struct ClipRef { name: String, category: String, } /// Map a phrase list (and its categories) onto the clip names of `bank`. /// /// Clips are named `{prefix}{index}` in list order and `FillerBank` keeps that /// order, so the i-th clip matches the i-th phrase — and thus the i-th /// category. A bank that does not match the list (custom clips) leaves the /// mapping short; the scheduler then falls back to a plain random pick. fn map_clips(bank: &FillerBank, texts: &[String], categories: &[String]) -> Vec { let names = bank.names(); texts .iter() .enumerate() .filter_map(|(i, _)| { names.get(i).map(|name| ClipRef { name: name.clone(), category: categories .get(i) .cloned() .unwrap_or_else(|| DEFAULT_CATEGORY.to_string()), }) }) .collect() } /// Per-session backchannel decision logic. /// /// Fed by the VAD: [`on_speech_start`](Self::on_speech_start) when a user turn /// begins, [`on_speech_end`](Self::on_speech_end) when it closes, and /// [`maybe_emit`](Self::maybe_emit) on every frame while the user is speaking. pub struct BackchannelScheduler { bank: Arc, settings: BackchannelSettings, /// Clip name + category per bank entry, in the order of `settings.texts`. clips: Vec, /// VAD end-of-utterance silence window (`VAD_MIN_SILENCE_MS`), used to cap /// the micro-pause window so a cue never lands on the pause that closes the /// turn. vad_min_silence_ms: u32, /// When the current user turn started (only set while speaking). turn_start: Option, /// When the last cue was emitted in this turn. last_emitted: Option, /// Cues emitted in the current turn. emitted: u32, /// Earliest instant the first cue of this turn may be played (randomized). first_cue_at: Option, /// Randomized gap to wait after the previous cue before the next one. next_gap_ms: u64, /// Whether the current micro-pause was already evaluated (cue or skip). pause_settled: bool, /// Random depth (ms) at which to react inside the current micro-pause, so /// cues do not all land at exactly `pause_min_ms`. Drawn once per pause. pause_target_ms: Option, /// Categories already used in this turn, for a natural progression. used_categories: Vec, /// Category of the last cue (avoid chaining two cues of the same kind). last_category: Option, /// Names of the most recently used clips (anti-repetition), kept across /// turns so the same cue never plays twice in a row. recent: VecDeque, /// RNG for jitter, the probabilistic gate and the volume drift. rng: XorShift64, } impl BackchannelScheduler { /// Create a scheduler for one session. `vad_min_silence_ms` is the VAD EOU /// window (`VAD_MIN_SILENCE_MS`); it bounds the micro-pause window. pub fn new( bank: Arc, settings: BackchannelSettings, vad_min_silence_ms: u32, ) -> Self { let clips = map_clips(&bank, &settings.texts, &settings.categories); let seed = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos() as u64) .unwrap_or(0x9E37_79B9_7F4A_7C15); // One line so the effective policy (and the parsed categories) is // visible in the log — the knobs are numerous and tuned by ear. let categories: Vec<&str> = clips.iter().map(|c| c.category.as_str()).collect(); info!( clips = clips.len(), categories = ?categories, pause_ms = format!("{}..{}", settings.pause_min_ms, settings.pause_max_ms), first_cue_ms = settings.min_speech_ms, interval_ms = settings.interval_ms, jitter_ms = settings.jitter_ms, prob = settings.prob, volume_jitter = settings.volume_jitter, max_per_turn = settings.max_per_turn, "backchannel scheduler ready" ); Self { bank, settings, clips, vad_min_silence_ms, turn_start: None, last_emitted: None, emitted: 0, first_cue_at: None, next_gap_ms: 0, pause_settled: false, pause_target_ms: None, used_categories: Vec::new(), last_category: None, recent: VecDeque::new(), rng: XorShift64::new(seed), } } /// Point the scheduler at another language's bank: replace the clip list and /// the phrase/category mapping. Called between turns when the detected user /// language changes (see [`crate::lang`]). The per-turn counters are left as /// they are — the next VAD speech start resets them anyway. pub fn retarget(&mut self, bank: Arc, texts: &[String], categories: &[String]) { self.clips = map_clips(&bank, texts, categories); self.bank = bank; } /// Begin a new user turn (VAD speech start). pub fn on_speech_start(&mut self) { let now = Instant::now(); self.turn_start = Some(now); self.last_emitted = None; self.emitted = 0; self.pause_settled = false; self.pause_target_ms = None; self.used_categories.clear(); self.last_category = None; // Randomize when the first cue becomes eligible: base delay plus a // random 0..jitter offset, so turns don't all cue at the same instant. let jitter = self.rng.range(0, self.settings.jitter_ms); self.first_cue_at = Some(now + Duration::from_millis(self.settings.min_speech_ms + jitter)); self.next_gap_ms = self.settings.interval_ms; } /// Close the current user turn (VAD speech end). pub fn on_speech_end(&mut self) { self.turn_start = None; self.last_emitted = None; self.emitted = 0; self.first_cue_at = None; self.pause_settled = false; self.pause_target_ms = None; self.used_categories.clear(); self.last_category = None; } /// Emit a backchannel into `mixer` if the policy allows it right now. /// /// Must be called while the user is speaking and the agent is silent. /// `pause_ms` is the length of the micro-pause the user is currently in /// ([`VadDetector::silence_ms`](crate::vad::VadDetector::silence_ms), 0 /// while voice is ongoing). Returns the clip `(name, category)` when one /// was emitted. pub fn maybe_emit( &mut self, mixer: &crate::audio::AudioMixer, pause_ms: u64, ) -> Option<(String, String)> { if !self.settings.enabled { return None; } if self.turn_start.is_none() || self.emitted >= self.settings.max_per_turn { return None; } // Cue only inside a micro-pause: never mid-word, and never on the pause // that is long enough to close the utterance (that one belongs to the // reply). A zero-length pause means voice is ongoing: re-arm so the // *next* micro-pause is evaluated. if pause_ms == 0 { self.pause_settled = false; self.pause_target_ms = None; return None; } let max = self.pause_max(); if pause_ms > max { // The pause ran past the window (it is about to close the turn): // abandon it without firing. self.pause_settled = true; return None; } if self.pause_settled { return None; } // At the start of a new micro-pause pick the depth to react at // uniformly inside the window, so the cue does not always land at // exactly `pause_min_ms` — the reaction point varies pause to pause. let target = match self.pause_target_ms { Some(t) => t, None => { let lo = self.settings.pause_min_ms.min(max); let t = self.rng.range(lo, max); self.pause_target_ms = Some(t); t } }; if pause_ms < target { return None; } // Randomized turn timing: hold off until the first-cue instant. if let Some(first) = self.first_cue_at { if Instant::now() < first { return None; } } // Randomized spacing since the previous cue. if let Some(last) = self.last_emitted { if last.elapsed() < Duration::from_millis(self.next_gap_ms) { return None; } } // Probabilistic gate: stay silent for some qualifying pauses, so the // cadence matches how a listener nods along instead of a metronome. if self.rng.frac() > self.settings.prob { self.pause_settled = true; return None; } let (name, category, samples) = self.select()?; let samples = self.apply_volume(&samples); mixer.backchannel_queue.push(&samples); self.pause_settled = true; self.last_emitted = Some(Instant::now()); self.emitted += 1; self.used_categories.push(category.clone()); self.last_category = Some(category.clone()); self.recent.push_back(name.clone()); while self.recent.len() > 2 { self.recent.pop_front(); } // Randomize the next gap around the configured interval. let half = self.settings.jitter_ms / 2; let lo = self.settings.interval_ms.saturating_sub(half).max(500); let hi = self.settings.interval_ms + half; self.next_gap_ms = self.rng.range(lo, hi); Some((name, category)) } /// Upper bound of the micro-pause window: the configured maximum, capped so /// it stays below the VAD end-of-utterance window (minus one analysis /// window) — otherwise the cue would land on the pause that ends the turn. fn pause_max(&self) -> u64 { let vad_cap = (self.vad_min_silence_ms.saturating_sub(32) as u64).max(1); self.settings.pause_max_ms.min(vad_cap) } /// Pick a clip honouring the category progression and anti-repetition, then /// falling back to a plain random clip when the bank does not match the /// phrase list. Returns `(name, category, samples)`. fn select(&mut self) -> Option<(String, String, Vec)> { if let Some(idx) = self.select_index() { let clip = &self.clips[idx]; if let Some(samples) = self.bank.get(&clip.name) { return Some((clip.name.clone(), clip.category.clone(), samples.clone())); } } self.bank .random() .map(|(n, s)| (n.to_string(), DEFAULT_CATEGORY.to_string(), s.clone())) } /// Index of the clip to play: prefer a category other than the last one, /// then categories not yet used this turn (a progression from "listening" /// to "agreeing"), then clips not used recently, and pick uniformly at /// random among whatever remains. fn select_index(&mut self) -> Option { if self.clips.is_empty() { return None; } let all: Vec = (0..self.clips.len()).collect(); let not_last: Vec = all .iter() .copied() .filter(|&i| Some(&self.clips[i].category) != self.last_category.as_ref()) .collect(); let pool = if not_last.is_empty() { all } else { not_last }; let fresh: Vec = pool .iter() .copied() .filter(|&i| !self.used_categories.contains(&self.clips[i].category)) .collect(); let pool = if fresh.is_empty() { pool } else { fresh }; let unrepeated: Vec = pool .iter() .copied() .filter(|&i| !self.recent.contains(&self.clips[i].name)) .collect(); let pool = if unrepeated.is_empty() { pool } else { unrepeated }; Some(pool[(self.rng.next_u64() % pool.len() as u64) as usize]) } /// Scale a clip by a random ±`volume_jitter` factor so the cues do not all /// sit at exactly the same level. fn apply_volume(&mut self, samples: &[i16]) -> Vec { let jitter = self.settings.volume_jitter; if jitter <= 0.0 { return samples.to_vec(); } let scale = 1.0 + (self.rng.frac() * 2.0 - 1.0) * jitter; samples .iter() .map(|&s| ((s as f32) * scale).clamp(i16::MIN as f32, i16::MAX as f32) as i16) .collect() } }