first

src/vad/mod.rs

@@ -0,0 +1,110 @@
+//! Voice Activity Detection (Silero VAD v5 via ONNX Runtime).
+//!
+//! The detector runs on the 16 kHz mono stream produced by the session layer
+//! (the same stream fed to the STT engine) and emits three kinds of events:
+//!
+//! - [`VadEvent::SpeechOnset`] — fast onset, used to interrupt the agent
+//!   (barge-in) as soon as the user starts talking.
+//! - [`VadEvent::SpeechStart`] — confirmed utterance start.
+//! - [`VadEvent::SpeechEnd`] — utterance finished, carrying the captured audio
+//!   (with a pre-roll pad) that is handed to the STT engine.
+//!
+//! Every behavioural knob is configured from the environment (see
+//! [`VadSettings`]), so the feature can be tuned or disabled without a rebuild.
+
+pub mod silero;
+
+pub use silero::{SileroVad, VadDetector, VadEvent};
+
+use crate::config::{env_bool, env_f32, env_u32};
+use std::path::PathBuf;
+
+/// Runtime configuration for the VAD subsystem, read from `.env`.
+#[derive(Debug, Clone)]
+pub struct VadSettings {
+    /// Master switch (`VAD_ENABLED`). When false the legacy RMS segmentation
+    /// inside the STT worker is used instead.
+    pub enabled: bool,
+    /// Path to `silero_vad.onnx` (`VAD_MODEL_PATH`).
+    pub model_path: PathBuf,
+    /// Speech probability above which a window counts as voice
+    /// (`VAD_THRESHOLD`, default 0.5).
+    pub threshold: f32,
+    /// Probability below which a window counts as silence
+    /// (`VAD_NEG_THRESHOLD`). Must be lower than `threshold`; the gap is the
+    /// hysteresis that prevents rapid onset/offset flapping.
+    pub neg_threshold: f32,
+    /// Speech required to confirm an utterance (`VAD_MIN_SPEECH_MS`).
+    pub min_speech_ms: u32,
+    /// Silence required to close an utterance — the EOU window
+    /// (`VAD_MIN_SILENCE_MS`).
+    pub min_silence_ms: u32,
+    /// Audio kept before the detected onset so the first phoneme is not
+    /// clipped (`VAD_SPEECH_PAD_MS`).
+    pub speech_pad_ms: u32,
+    /// Speech required to fire the barge-in onset (`VAD_BARGE_IN_MIN_SPEECH_MS`).
+    pub barge_in_min_speech_ms: u32,
+    /// Feed the STT engine with VAD-segmented utterances instead of letting it
+    /// run its own RMS segmentation (`VAD_SEGMENT_STT`).
+    pub segment_stt: bool,
+    /// Interrupt the agent when the user starts speaking over it
+    /// (`VAD_BARGE_IN`).
+    pub barge_in: bool,
+    /// Log the speech probability of every analysis window (`VAD_DEBUG`).
+    pub debug: bool,
+}
+
+impl Default for VadSettings {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            model_path: PathBuf::from("models/silero_vad.onnx"),
+            threshold: 0.5,
+            neg_threshold: 0.35,
+            min_speech_ms: 250,
+            min_silence_ms: 300,
+            speech_pad_ms: 30,
+            barge_in_min_speech_ms: 150,
+            segment_stt: true,
+            barge_in: true,
+            debug: false,
+        }
+    }
+}
+
+impl VadSettings {
+    /// Build the settings from environment variables, falling back to defaults.
+    pub fn from_env() -> Self {
+        let mut s = Self::default();
+        if let Ok(p) = std::env::var("VAD_MODEL_PATH") {
+            if !p.is_empty() {
+                s.model_path = PathBuf::from(p);
+            }
+        }
+        s.enabled = env_bool("VAD_ENABLED", s.enabled);
+        s.segment_stt = env_bool("VAD_SEGMENT_STT", s.segment_stt);
+        s.barge_in = env_bool("VAD_BARGE_IN", s.barge_in);
+        s.debug = env_bool("VAD_DEBUG", s.debug);
+        s.threshold = env_f32("VAD_THRESHOLD", s.threshold);
+        s.neg_threshold = env_f32("VAD_NEG_THRESHOLD", s.neg_threshold);
+        s.min_speech_ms = env_u32("VAD_MIN_SPEECH_MS", s.min_speech_ms);
+        s.min_silence_ms = env_u32("VAD_MIN_SILENCE_MS", s.min_silence_ms);
+        s.speech_pad_ms = env_u32("VAD_SPEECH_PAD_MS", s.speech_pad_ms);
+        s.barge_in_min_speech_ms = env_u32("VAD_BARGE_IN_MIN_SPEECH_MS", s.barge_in_min_speech_ms);
+
+        // Hysteresis must stay below the speech threshold, and the fast
+        // barge-in onset can never be slower than the utterance onset.
+        s.neg_threshold = s.neg_threshold.min(s.threshold);
+        s.barge_in_min_speech_ms = s.barge_in_min_speech_ms.min(s.min_speech_ms);
+        s
+    }
+}
+
+/// Samples retained ahead of a detected onset (pre-roll + triggering speech).
+pub(crate) fn preroll_samples(settings: &VadSettings) -> usize {
+    let ms = settings
+        .speech_pad_ms
+        .saturating_add(settings.min_speech_ms)
+        .saturating_add(silero::WINDOW_MS);
+    (crate::config::STT_SAMPLE_RATE as u64 * ms as u64 / 1000) as usize
+}

src/vad/silero.rs

@@ -0,0 +1,399 @@
+//! Silero VAD v5 ONNX wrapper and the onset/offset state machine.
+//!
+//! The model consumes fixed 32 ms windows (512 samples at 16 kHz) preceded by
+//! a 64-sample context carried over from the previous window, plus a recurrent
+//! state tensor. It returns the speech probability of the window.
+
+use crate::config::STT_SAMPLE_RATE;
+use crate::vad::{preroll_samples, VadSettings};
+use anyhow::{bail, Context, Result};
+use ndarray::{Array1, ArrayD, IxDyn};
+use ort::environment::Environment;
+use ort::session::builder::GraphOptimizationLevel;
+use ort::session::{Session, SessionInputValue};
+use ort::value::{Value, ValueType};
+use parking_lot::Mutex;
+use std::collections::VecDeque;
+use std::path::Path;
+use std::sync::Arc;
+use tracing::{debug, error, info};
+
+/// Analysis window of the Silero v5 model at 16 kHz (512 samples = 32 ms).
+pub const WINDOW_SAMPLES: usize = 512;
+/// Duration of one analysis window in milliseconds.
+pub(crate) const WINDOW_MS: u32 = 32;
+/// Samples of the previous window prepended to the current one (v5 context).
+const CONTEXT_SAMPLES: usize = 64;
+/// Fallback recurrent state shape `[layers, batch, hidden]`.
+const STATE_SHAPE: [usize; 3] = [2, 1, 128];
+
+const IN_INPUT: &str = "input";
+const IN_STATE: &str = "state";
+const IN_SR: &str = "sr";
+const OUT_PROB: &str = "output";
+const OUT_STATE: &str = "stateN";
+
+struct Inner {
+    /// Keeps the ONNX Runtime environment alive for the session.
+    _environment: Environment,
+    session: Session,
+    /// `state` shape as declared by the graph (dynamic axes resolved to 1).
+    state_shape: Vec<usize>,
+    /// Rank of the `sr` input (0 = scalar, as exported by Silero v5).
+    sr_rank: usize,
+}
+
+/// Shared handle to the Silero VAD ONNX model. Cloning is cheap (`Arc`), the
+/// per-session recurrent state lives in [`VadDetector`].
+#[derive(Clone)]
+pub struct SileroVad {
+    inner: Arc<Mutex<Inner>>,
+}
+
+impl SileroVad {
+    /// Load `silero_vad.onnx` from `path`.
+    pub fn load(path: &Path) -> Result<Self> {
+        info!(model = %path.display(), "loading Silero VAD model");
+
+        let environment = crate::onnx::environment()?;
+        let session = Session::builder(&environment)
+            .map_err(|e| anyhow::anyhow!("failed to create ONNX session builder: {e}"))?
+            .with_optimization_level(GraphOptimizationLevel::Level3)
+            .map_err(|e| anyhow::anyhow!("failed to set ONNX optimization level: {e}"))?
+            .commit_from_file(path)
+            .with_context(|| format!("failed to load ONNX model {}", path.display()))?;
+
+        // Introspect the recurrent-state and sample-rate inputs so we build
+        // tensors matching the exported graph instead of hardcoding shapes.
+        let state_shape = declared_shape(&session, IN_STATE).unwrap_or_else(|_| STATE_SHAPE.to_vec());
+        let sr_rank = declared_shape(&session, IN_SR)
+            .map(|s| s.len())
+            .unwrap_or(0);
+
+        info!(
+            model = %path.display(),
+            window_samples = WINDOW_SAMPLES,
+            window_ms = WINDOW_MS,
+            state_shape = ?state_shape,
+            sr_rank,
+            "Silero VAD model loaded"
+        );
+
+        Ok(Self {
+            inner: Arc::new(Mutex::new(Inner {
+                _environment: environment,
+                session,
+                state_shape,
+                sr_rank,
+            })),
+        })
+    }
+
+    /// Number of `f32` elements of the recurrent state, as declared by the
+    /// graph. Detectors must allocate their state buffer with this size.
+    pub fn state_len(&self) -> usize {
+        self.inner.lock().state_shape.iter().product()
+    }
+
+    /// Speech probability of one window. `state` is updated in place and
+    /// `context` must hold the last [`CONTEXT_SAMPLES`] samples of the previous
+    /// window (zeros on the first call).
+    fn probability(&self, state: &mut [f32], context: &[f32], window: &[f32]) -> Result<f32> {
+        if window.len() != WINDOW_SAMPLES {
+            bail!("VAD window must be {WINDOW_SAMPLES} samples, got {}", window.len());
+        }
+
+        let mut input = Vec::with_capacity(CONTEXT_SAMPLES + WINDOW_SAMPLES);
+        input.extend_from_slice(context);
+        input.extend_from_slice(window);
+        let input_len = input.len();
+
+        let input_t = Value::from_array(ArrayD::from_shape_vec(IxDyn(&[1, input_len]), input)?)
+            .context("failed to build VAD input tensor")?;
+
+        let mut guard = self.inner.lock();
+        let inner = &mut *guard;
+
+        let state_t = Value::from_array(ArrayD::from_shape_vec(
+            IxDyn(&inner.state_shape),
+            state.to_vec(),
+        )?)
+        .context("failed to build VAD state tensor")?;
+
+        let sr = STT_SAMPLE_RATE as i64;
+        let sr_t = if inner.sr_rank == 0 {
+            Value::from_array(ArrayD::from_shape_vec(IxDyn(&[]), vec![sr])?)
+        } else {
+            Value::from_array(Array1::from_vec(vec![sr]))
+        }
+        .context("failed to build VAD sample-rate tensor")?;
+
+        let outputs = inner
+            .session
+            .run(vec![
+                (IN_INPUT.to_string(), SessionInputValue::from(input_t)),
+                (IN_STATE.to_string(), SessionInputValue::from(state_t)),
+                (IN_SR.to_string(), SessionInputValue::from(sr_t)),
+            ])
+            .context("Silero VAD inference failed")?;
+
+        let prob = outputs
+            .get(OUT_PROB)
+            .context("VAD model produced no 'output'")?
+            .try_extract_tensor::<f32>()
+            .context("failed to extract VAD probability")?
+            .1
+            .first()
+            .copied()
+            .unwrap_or(0.0);
+
+        let (_, new_state) = outputs
+            .get(OUT_STATE)
+            .context("VAD model produced no 'stateN'")?
+            .try_extract_tensor::<f32>()
+            .context("failed to extract VAD state")?;
+        if new_state.len() == state.len() {
+            state.copy_from_slice(new_state);
+        }
+
+        Ok(prob.clamp(0.0, 1.0))
+    }
+}
+
+/// Read the declared shape of a tensor input, resolving dynamic axes to 1.
+fn declared_shape(session: &Session, name: &str) -> Result<Vec<usize>> {
+    let input = session
+        .inputs()
+        .iter()
+        .find(|i| i.name() == name)
+        .with_context(|| format!("model has no input named '{name}'"))?;
+    match input.dtype() {
+        ValueType::Tensor { shape, .. } => Ok(shape
+            .iter()
+            .map(|&d| if d < 0 { 1 } else { d as usize })
+            .collect()),
+        other => bail!("input '{name}' is not a tensor: {other:?}"),
+    }
+}
+
+/// Events emitted by [`VadDetector::push`].
+#[derive(Debug, Clone)]
+pub enum VadEvent {
+    /// The user just started speaking (after `barge_in_min_speech_ms`). Used to
+    /// interrupt the agent.
+    SpeechOnset { prob: f32 },
+    /// Utterance start confirmed (after `min_speech_ms`).
+    SpeechStart { prob: f32 },
+    /// Utterance closed (after `min_silence_ms` of silence). `audio` is the
+    /// captured 16 kHz mono PCM, including the pre-roll pad.
+    SpeechEnd {
+        audio: Vec<i16>,
+        duration_ms: u64,
+        avg_prob: f32,
+    },
+}
+
+/// Per-session VAD: feeds the model and turns probabilities into speech
+/// onset/offset events.
+pub struct VadDetector {
+    model: SileroVad,
+    settings: VadSettings,
+    /// Recurrent model state (`state_shape` elements).
+    state: Vec<f32>,
+    /// Last [`CONTEXT_SAMPLES`] samples of the previous window.
+    context: Vec<f32>,
+    /// Samples received but not yet consumed by a full window.
+    pending: Vec<i16>,
+    /// Raw pre-roll ring, so the first phonemes are not clipped at onset.
+    preroll: VecDeque<i16>,
+    preroll_capacity: usize,
+    /// Audio of the utterance being captured.
+    utterance: Vec<i16>,
+    /// Whether the detector is inside a confirmed utterance.
+    in_speech: bool,
+    /// Fast onset already reported for the current speech segment.
+    onset_reported: bool,
+    /// Utterance start already reported for the current speech segment.
+    start_reported: bool,
+    /// Consecutive speech milliseconds while idle.
+    speech_run_ms: u32,
+    /// Consecutive silence milliseconds while in speech.
+    silence_run_ms: u32,
+    /// Running probability average over the current utterance.
+    prob_sum: f32,
+    speech_windows: u64,
+    /// Windows analysed since the detector was created (for logging).
+    window_count: u64,
+}
+
+impl VadDetector {
+    /// Create a detector for one session.
+    pub fn new(model: SileroVad, settings: VadSettings) -> Self {
+        let preroll_capacity = preroll_samples(&settings).max(WINDOW_SAMPLES);
+        let state_len = model.state_len().max(STATE_SHAPE.iter().product());
+        Self {
+            model,
+            settings,
+            state: vec![0.0; state_len],
+            context: vec![0.0; CONTEXT_SAMPLES],
+            pending: Vec::new(),
+            preroll: VecDeque::with_capacity(preroll_capacity),
+            preroll_capacity,
+            utterance: Vec::new(),
+            in_speech: false,
+            onset_reported: false,
+            start_reported: false,
+            speech_run_ms: 0,
+            silence_run_ms: 0,
+            prob_sum: 0.0,
+            speech_windows: 0,
+            window_count: 0,
+        }
+    }
+
+    /// Whether the detector is currently inside a confirmed utterance. Used by
+    /// the backchannel scheduler to know when the user is still speaking.
+    pub fn in_speech(&self) -> bool {
+        self.in_speech
+    }
+
+    /// Feed 16 kHz mono PCM and collect the events it triggers.
+    ///
+    /// Any number of samples may be pushed; internally they are buffered into
+    /// fixed 512-sample windows, so callers can pass arbitrary frame sizes.
+    pub fn push(&mut self, samples: &[i16]) -> Vec<VadEvent> {
+        let mut events = Vec::new();
+        if samples.is_empty() {
+            return events;
+        }
+
+        // Keep the pre-roll ring up to date over the raw stream.
+        for &s in samples {
+            if self.preroll.len() >= self.preroll_capacity {
+                self.preroll.pop_front();
+            }
+            self.preroll.push_back(s);
+        }
+
+        self.pending.extend_from_slice(samples);
+        while self.pending.len() >= WINDOW_SAMPLES {
+            let window: Vec<i16> = self.pending.drain(..WINDOW_SAMPLES).collect();
+            let prob = match self.analyse(&window) {
+                Ok(p) => p,
+                Err(e) => {
+                    error!(error = %e, "VAD inference failed — treating window as silence");
+                    0.0
+                }
+            };
+            self.window_count += 1;
+            self.step(prob, &window, &mut events);
+        }
+
+        events
+    }
+
+    /// Run the model on one window and refresh the recurrent context.
+    fn analyse(&mut self, window: &[i16]) -> Result<f32> {
+        let f32_window: Vec<f32> = window.iter().map(|&s| s as f32 / 32768.0).collect();
+        let model = self.model.clone();
+        let prob = model.probability(&mut self.state, &self.context, &f32_window)?;
+        self.context.clear();
+        self.context
+            .extend_from_slice(&f32_window[f32_window.len() - CONTEXT_SAMPLES..]);
+        Ok(prob)
+    }
+
+    /// Advance the onset/offset state machine by one window.
+    fn step(&mut self, prob: f32, window: &[i16], events: &mut Vec<VadEvent>) {
+        if self.settings.debug {
+            debug!(
+                window = self.window_count,
+                prob = format!("{prob:.3}"),
+                in_speech = self.in_speech,
+                "VAD window"
+            );
+        }
+
+        if !self.in_speech {
+            if prob >= self.settings.threshold {
+                self.speech_run_ms = self.speech_run_ms.saturating_add(WINDOW_MS);
+            } else {
+                self.speech_run_ms = 0;
+            }
+
+            if !self.onset_reported && self.speech_run_ms >= self.settings.barge_in_min_speech_ms {
+                self.onset_reported = true;
+                debug!(
+                    speech_ms = self.speech_run_ms,
+                    prob = format!("{prob:.3}"),
+                    "VAD speech onset"
+                );
+                events.push(VadEvent::SpeechOnset { prob });
+            }
+
+            if !self.start_reported && self.speech_run_ms >= self.settings.min_speech_ms {
+                self.start_reported = true;
+                self.in_speech = true;
+                // The pre-roll ring already holds this window (it is refreshed
+                // before the pending buffer is drained in `push`), so seed the
+                // utterance from it instead of appending again.
+                self.utterance.clear();
+                self.utterance.extend(self.preroll.iter().copied());
+                self.prob_sum = prob;
+                self.speech_windows = 1;
+                self.silence_run_ms = 0;
+                debug!(
+                    speech_ms = self.speech_run_ms,
+                    prob = format!("{prob:.3}"),
+                    preroll_samples = self.utterance.len(),
+                    "VAD speech start"
+                );
+                events.push(VadEvent::SpeechStart { prob });
+            }
+            return;
+        }
+
+        self.utterance.extend_from_slice(window);
+        self.prob_sum += prob;
+        self.speech_windows += 1;
+
+        if prob < self.settings.neg_threshold {
+            self.silence_run_ms = self.silence_run_ms.saturating_add(WINDOW_MS);
+        } else {
+            self.silence_run_ms = 0;
+        }
+
+        if self.silence_run_ms >= self.settings.min_silence_ms {
+            let audio = std::mem::take(&mut self.utterance);
+            let avg_prob = if self.speech_windows > 0 {
+                self.prob_sum / self.speech_windows as f32
+            } else {
+                0.0
+            };
+            let duration_ms = (audio.len() as u64 * 1000) / STT_SAMPLE_RATE as u64;
+            debug!(
+                duration_ms,
+                samples = audio.len(),
+                avg_prob = format!("{avg_prob:.3}"),
+                "VAD speech end"
+            );
+            events.push(VadEvent::SpeechEnd {
+                audio,
+                duration_ms,
+                avg_prob,
+            });
+            self.reset_segment();
+        }
+    }
+
+    /// Clear the per-segment state after an utterance is closed.
+    fn reset_segment(&mut self) {
+        self.in_speech = false;
+        self.onset_reported = false;
+        self.start_reported = false;
+        self.speech_run_ms = 0;
+        self.silence_run_ms = 0;
+        self.prob_sum = 0.0;
+        self.speech_windows = 0;
+    }
+}