This commit is contained in:
valenti
2026-10-17 22:01:31 +01:00
parent bfed42c3b3
commit 79f927d968
53 changed files with 2158 additions and 78 deletions
+52 -6
View File
@@ -371,13 +371,19 @@ impl TurnManager {
// external tokenizer only if no text was produced.
let text = if !transcript.text.trim().is_empty() {
transcript.text.trim().to_string()
} else if transcript.token_ids.is_empty() {
// No tokens at all: nothing was said (a VAD false-trigger that
// slipped past the gate). Skip the turn instead of forwarding the
// "<transcript 0 tokens>" placeholder to the LLM.
debug!("no tokens decoded, skipping turn");
return;
} else {
match &self.tokenizer {
Some(tok) => tok.decode(&transcript.token_ids),
None => format!("<transcript {} tokens>", transcript.token_ids.len()),
}
};
if text.is_empty() {
if text.trim().is_empty() {
debug!("empty transcript after decode, skipping turn");
return;
}
@@ -592,9 +598,10 @@ impl TurnManager {
// Tool-calling loop. Each round streams one reply; tool call blocks are
// stripped from the spoken/displayed text (the model emits them as
// text — see `crate::tools`), the calls run over HTTP, and the results
// are handed back for the next round. Bounded by
// `LLM_TOOL_MAX_ITERATIONS`. With no tools configured the loop runs
// exactly once.
// are handed back for the next round. A round that streams back empty
// is re-asked without streaming, where the server returns the call it
// dropped from the SSE path. Bounded by `LLM_TOOL_MAX_ITERATIONS`. With
// no tools configured the loop runs exactly once.
let registry = llm.tools().cloned();
let max_rounds = self.config.llm_tool_max_iterations.max(1);
let mut spoken = String::new();
@@ -678,7 +685,36 @@ impl TurnManager {
break;
}
let calls = stripper.take_calls();
let mut calls = stripper.take_calls();
// Streaming rescue. Some OpenAI-compatible servers swallow the
// model's native tool call on the SSE path — vLLM with a
// `--tool-call-parser` that does not match the model's format ends
// the stream with no content at all — while the same request
// *without* streaming returns the call intact. When a round
// produced nothing and tools are configured, re-ask once,
// non-streaming, and take the call from there.
if calls.is_empty()
&& registry.is_some()
&& !cancel.is_cancelled()
&& spoken.trim().is_empty()
&& round + 1 < max_rounds
{
match llm.chat_reply(&messages, &cancel).await {
Ok(reply) => {
calls = reply.tool_calls;
if calls.is_empty() && !reply.text.trim().is_empty() {
// No call after all: the blocking reply holds the
// actual answer — speak it rather than go silent.
self.emit(json!({ "type": "assistant_delta", "text": &reply.text }));
spoken.push_str(&reply.text);
let _ = sentence_tx.send(reply.text.trim().to_string());
}
}
Err(e) => error!(error = %e, "tool-call fallback request failed"),
}
}
if calls.is_empty() {
break;
}
@@ -730,11 +766,21 @@ impl TurnManager {
}
/// Whether the agent is currently speaking (TTS in flight or queued audio).
/// Used by the VAD-driven barge-in to avoid flushing an idle mixer.
/// Used by the backchannel gate to avoid cuing on top of the agent's own
/// TTS — see [`Self::is_audible`] for the barge-in gate.
pub fn is_speaking(&self) -> bool {
self.mixer.is_active()
}
/// Whether the agent has *audible* output (reply playing out, or a
/// latency-mask cue queued). Unlike [`Self::is_speaking`] this is `false`
/// while the reply is still being synthesised or sits in the pre-buffer.
/// Used by the VAD-driven barge-in: a transient must not cancel a reply
/// before the user has heard any of it.
pub fn is_audible(&self) -> bool {
self.mixer.is_audible()
}
/// Update the voice-cloning reference at runtime. Pass `None` for
/// `audio_path` to clear the reference.
pub fn set_reference(&self, audio_path: Option<String>, text: Option<String>) {