valenti/vocero-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate ApiPilot tools and add light-theme voice UI

Browse Source
This commit is contained in:
valenti
2026-10-05 08:25:29 -04:00
parent 413ba08c21
commit 4d7c7a23df
9 changed files with 6930 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
speech_to_speech/LLM/lm_output_processor.py
+90 -6
View File
@@ -4,16 +4,80 @@ LLM Output Processor
Intercepts LLM output to:
1. Extract tool calls and send them via text_output_queue
2. Forward clean text to TTS pipeline
3. 2026-09-25: SANITIZE APIPILOT internal markers before clients / TTS (anti-leak tecnico)
"""
from __future__ import annotations
import logging
import re
from collections.abc import Iterator
from queue import Queue
from speech_to_speech.baseHandler import BaseHandler
from speech_to_speech.pipeline.events import AssistantTextEvent, ResponseFailedEvent, TokenUsageEvent
# 2026-09-25 SANITIZE APIPILOT CALL/RESULT markers in LLM output BEFORE:
# (a) AssistantTextEvent viene mandato ai client WS (e loggato come <<< TEXT:/ASSISTANT:)
# (b) TTSInput viene mandato a Qwen3-TTS (che lo pronuncerebbe ad alta voce).
# Formati osservati reali oggi (upstream Vocero start_pipeline_rag):
# 1) <<<APIPILOT_CALL:NAME=tool,ARGUMENTS=json>>> standard 3x
# 2) <APIPILOT_CALL:NAME=get_user_briefing,ARGUMENTS={"format":"voice"}
# singolo '<' '>' + args inventati da AI (format=voice)
# 3) <<<APIPILOT_CALL:...}> \n>> spezzato multi-riga
# 4) >>>APIPILOT_CALL:...<<< invertito (errore sintassi AI)
# 5) <<<APIPILOT_RESULT:PASS2>>> + blocco result.
_APIPILOT_CALL_NORM = re.compile(
r"[<]{1,3}\s*APIPILOT_CALL:NAME=([A-Za-z0-9_.-]+),ARGUMENTS=(.*?)[>]{1,3}",
re.IGNORECASE | re.DOTALL,
)
_APIPILOT_CALL_INV = re.compile(
r"[>]{1,3}\s*APIPILOT_CALL:NAME=([A-Za-z0-9_.-]+),ARGUMENTS=(.*?)[<]{1,3}",
re.IGNORECASE | re.DOTALL,
)
_APIPILOT_RESULT = re.compile(
r"[<]{1,3}\s*APIPILOT_RESULT(?::PASS2)?[^<>\n]{0,4000}[>]{1,3}",
re.IGNORECASE | re.DOTALL,
)
_APIPILOT_SPANISH_PLACEHOLDER = ("Voy a consultar tus datos, un momento por favor.")
def _sanitize_apipilot_before_client_and_tts(text: str) -> tuple[str, bool, str]:
if not isinstance(text, str) or not text.strip():
return text, False, ""
found_call = False
found_call_tool = ""
def _repl_call(m: re.Match) -> str:
nonlocal found_call, found_call_tool
found_call = True
try:
nm = m.group(1)
if isinstance(nm, str) and nm.strip():
found_call_tool = nm.strip()
except Exception:
pass
return f" {_APIPILOT_SPANISH_PLACEHOLDER} "
new_text = _APIPILOT_CALL_NORM.sub(_repl_call, text)
if not found_call:
new_text = _APIPILOT_CALL_INV.sub(_repl_call, new_text)
found_result = False
if _APIPILOT_RESULT.search(new_text):
new_text = _APIPILOT_RESULT.sub(" ", new_text)
found_result = True
# safety catchall frammenti marker senza wrapper (se marker troncato mid-stream)
low = new_text.lower()
if "apipilot" in low and ("call" in low or "result" in low or "name=" in low or "arguments=" in low):
new_text = re.sub(r"(?i)\bAPIPILOT(?:_CALL|_RESULT)?\b", " ", new_text)
new_text = re.sub(r"(?i)\b(?:NAME|ARGUMENTS|STATUS|PASS2)\s*[:=]", " ", new_text)
found_call = True
if found_call or found_result:
new_text = re.sub(r"\s{2,}", " ", new_text).strip()
tag = ""
if found_call:
tag = f"CALL({found_call_tool or '?'})"
if found_result:
tag = (tag + "+" if tag else "") + "RESULT"
return new_text, (found_call or found_result), tag
from speech_to_speech.pipeline.handler_types import LLMOut, TTSIn
from speech_to_speech.pipeline.messages import EndOfResponse, LLMResponseChunk, TokenUsage, TTSInput
from speech_to_speech.pipeline.queue_types import TextEventItem
@@ -120,24 +184,44 @@ class LMOutputProcessor(BaseHandler[LLMOut, TTSIn]):
logger.debug(f"LM processor: text='{lm_output.text}', tools={lm_output.tools}")
# 2026-09-25 SANITIZE SINGOLO PUNTO CENTRALE: PRIMA di AssistantTextEvent (client/log)
# E PRIMA di TTSInput (Qwen3-TTS che pronuncerebbe marker APIPILOT CALL).
clean_text = lm_output.text
any_sanitized = False
sanitize_tag = ""
if clean_text and clean_text.strip():
clean_text, any_sanitized, sanitize_tag = _sanitize_apipilot_before_client_and_tts(clean_text)
if any_sanitized:
logger.warning(
"[LM-OUTPUT-APIPILOT-SANITIZE] %s — tool=%s rawLen=%d cleanLen=%d (rimosso / sostituito con placeholder spagnolo PRIMA di client + TTS).",
sanitize_tag or "-",
(sanitize_tag if sanitize_tag else "marker"),
len(lm_output.text or ""),
len(clean_text or ""),
)
# evita che lo stesso chunk (marker solo) arrivi a TTS come "" → Hello.
# Se dopo sanitize testo vuoto MA avevamo trovato marker → placeholder.
if not clean_text.strip():
clean_text = _APIPILOT_SPANISH_PLACEHOLDER
if self.text_output_queue is not None:
event = AssistantTextEvent(
text=lm_output.text,
text=clean_text,
turn_id=lm_output.turn_id,
turn_revision=lm_output.turn_revision,
cancel_generation=lm_output.cancel_generation,
)
if lm_output.tools:
event.tools = lm_output.tools
logger.info(f"Sending to clients: text='{lm_output.text}', tools={[t.name for t in lm_output.tools]}")
logger.info(f"Sending to clients: text='{clean_text}', tools={[t.name for t in lm_output.tools]}")
else:
logger.debug(f"Sending to clients: text='{lm_output.text}' (no tools)")
logger.debug(f"Sending to clients: text='{clean_text}' (no tools)")
self.text_output_queue.put(event)
if lm_output.text and response_wants_audio(lm_output.response):
logger.debug(f"Forwarding to TTS: '{lm_output.text}'")
if clean_text and response_wants_audio(lm_output.response):
logger.debug(f"Forwarding to TTS: '{clean_text}' (original marker was: {sanitize_tag if any_sanitized else 'none'})")
yield TTSInput(
text=lm_output.text,
text=clean_text,
language_code=lm_output.language_code,
runtime_config=lm_output.runtime_config,
response=lm_output.response,
speech_to_speech/TTS/qwen3_tts_handler.py
+86 -1
View File
@@ -12,6 +12,78 @@ import math
import re
import tempfile
import unicodedata
# 2026-09-25 SANITIZE APIPILOT MARKERS PRIMA DI TTS (anti-leak pronunciato da Qwen3).
# Lato server S2S Python abbiamo gia' gate + sanitize ma come DOPPIA RETE
# di sicurezza sanitizziamo DIRETTAMENTE il testo che entra in Qwen3-TTS.
# Formati osservati reali oggi:
# (a) <<<APIPILOT_CALL:NAME=tool,ARGUMENTS=json>>> standard 3 parentesi
# (b) <APIPILOT_CALL:NAME=tool,ARGUMENTS={"format":"voice"}
# singolo < > (caso user oggi)
# (c) <<<APIPILOT_CALL:...}> \n>> spezzato piu' righe
# (d) >>>APIPILOT_CALL:...<<< invertito
# Regola: CALL -> placeholder spagnolo (frase naturale utente), RESULT -> strip.
_APIPILOT_CALL_NORMAL_RE = re.compile(
r"[<]{1,3}\s*APIPILOT_CALL:NAME=([A-Za-z0-9_.-]+),ARGUMENTS=(.*?)[>]{1,3}",
re.IGNORECASE | re.DOTALL,
)
_APIPILOT_CALL_INVERTED_RE = re.compile(
r"[>]{1,3}\s*APIPILOT_CALL:NAME=([A-Za-z0-9_.-]+),ARGUMENTS=(.*?)[<]{1,3}",
re.IGNORECASE | re.DOTALL,
)
_APIPILOT_RESULT_RE = re.compile(
r"[<]{1,3}\s*APIPILOT_RESULT(?::PASS2)?[^<>\n]{0,2000}[>]{1,3}",
re.IGNORECASE | re.DOTALL,
)
_APIPILOT_SPANISH_PLACEHOLDER = "Voy a consultar tus datos, un momento por favor."
def _tts_sanitize_apipilot_markers(text: str) -> tuple[str, bool, str]:
"""Rimuove marker APIPILOT dal testo TTS.
Returns (sanitized_text, found_any, log_tag).
"""
if not isinstance(text, str) or not text.strip():
return text, False, "empty"
found_call = False
found_result = False
tool_name = ""
# 1) marker CALL standard + invertito -> placeholder spagnolo
def _repl_call(m: re.Match) -> str:
nonlocal found_call, tool_name
found_call = True
try:
nm = m.group(1)
if isinstance(nm, str) and nm.strip():
tool_name = nm.strip()
except Exception:
pass
return f" {_APIPILOT_SPANISH_PLACEHOLDER} "
text2 = _APIPILOT_CALL_NORMAL_RE.sub(_repl_call, text)
if not found_call:
text2 = _APIPILOT_CALL_INVERTED_RE.sub(_repl_call, text2)
# 2) marker RESULT -> strip silenzioso (non sono pronunciabili)
if _APIPILOT_RESULT_RE.search(text2):
found_result = True
text2 = _APIPILOT_RESULT_RE.sub(" ", text2)
# 3) safety: FRAMMENTI marker singoli che non hanno wrapper completo
low = text2.lower()
if "apipilot" in low and ("call" in low or "result" in low):
import re as _re_sf
text2 = _re_sf.sub(r"(?i)\bAPIPILOT(?:_CALL|_RESULT)?\b", " ", text2)
text2 = _re_sf.sub(r"(?i)\b(?:NAME|ARGUMENTS|STATUS|PASS2)\s*[:=]", " ", text2)
found_call = True
if found_call or found_result:
text2 = re.sub(r"\s{2,}", " ", text2).strip()
tag = "-"
if found_call and found_result:
tag = f"CALL({tool_name or '?'})+RESULT"
elif found_call:
tag = f"CALL({tool_name or '?'})"
elif found_result:
tag = "RESULT"
return text2, (found_call or found_result), tag
from collections.abc import Callable
from pathlib import Path
from sys import platform
@@ -709,7 +781,20 @@ class Qwen3TTSHandler(BaseHandler[TTSIn, TTSOut]):
coalesced_text, _language_code, _saw_end_of_response = self._coalesce_pending_tts_input(tts_input)
text = coalesced_text or "Hello."
raw_text = coalesced_text or "Hello."
# 2026-09-25 SANITIZE APIPILOT PRIMA DI TTS (doppia rete sicurezza):
# Anche se lato server S2s Python abbiamo gia' gate audio drop + sanitize,
# Qwen3-TTS puo' ricevere marker per race / nuovi formati. Quindi QUI
# eliminiamo / sostituiamo con placeholder PRIMA di chiamare generate().
text, found_marker, sanitize_tag = _tts_sanitize_apipilot_markers(raw_text)
if found_marker:
logger.warning(
"[TTS-APIPILOT-SANITIZE] marker=%s rawLen=%d sanitizedLen=%d → sostituito con placeholder spagnolo / strip.",
sanitize_tag, len(raw_text or ""), len(text or ""),
)
if not text.strip():
# Safety se il marker era l'unica cosa: ripetiamo placeholder invece di Hello.
text = _APIPILOT_SPANISH_PLACEHOLDER
model_type = self._model_type()
self._apply_session_voice_override(model_type, runtime_config, response)