3393 lines
127 KiB
JavaScript
3393 lines
127 KiB
JavaScript
// @ts-check
|
|
/**
|
|
* Minimal voice conversation app, talking to a Hugging Face speech-to-speech
|
|
* backend over **WebSocket** (drop-in alternative to the WebRTC variant).
|
|
*
|
|
* Click the orb -> we ask for the mic, POST a session on the LB, open a
|
|
* WebSocket on the routed compute endpoint, push session.update + mic
|
|
* audio, play back the TTS audio. The orb visually reflects the live
|
|
* state (idle, connecting, listening, user-speaking, processing,
|
|
* ai-speaking).
|
|
*
|
|
* The only meaningful difference vs. the WebRTC main.js is that the
|
|
* client owns its own AudioContext (no `attachOutputTrack`), so we hand
|
|
* it the MediaStream directly.
|
|
*
|
|
* @typedef {"idle" | "connecting" | "queued" | "your-turn" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState
|
|
*/
|
|
|
|
import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js?v=20260826001";
|
|
import { $, escHtml, truncateError, DEBUG } from "./ui/dom.js?v=20260826001";
|
|
import { ChatView } from "./ui/chat.js?v=20260826001";
|
|
import { Account } from "./ui/account.js?v=20260826001";
|
|
|
|
const DEFAULT_VOICE = "Aiden";
|
|
const DEFAULT_INSTRUCTIONS =
|
|
"You are a friendly voice assistant. " +
|
|
"Keep replies short, warm, and spoken. Avoid long monologues.";
|
|
|
|
// Appended to the user's instructions whenever at least one tool is enabled.
|
|
// Stops the model from announcing capabilities ("Yes, I can search") and then
|
|
// idling for the next turn — it should act immediately in the same response.
|
|
const TOOL_USE_HINT =
|
|
" When the user's request calls for one of your tools, do not describe your " +
|
|
"capabilities or say you can do it and wait for another turn. Instead, say " +
|
|
'a brief acknowledgement like "Let me search for that..." and call the tool ' +
|
|
"right away in the same response.";
|
|
|
|
const MCP_SEARCH_HINT =
|
|
" When the user asks for information, review the full MCP tool catalog, choose " +
|
|
"the single best tool for the request, and call it immediately when it can solve " +
|
|
"the task. If required parameters are missing, ask a short follow-up question. " +
|
|
"If no MCP tool fits, answer normally or use the other available tools.";
|
|
const OPENAI_RENDER_MARKER = "[VOCERO_RENDER_REPLY]";
|
|
const OPENAI_RENDER_HINT =
|
|
` Do not answer ordinary user speech directly. Stay silent unless the latest user message starts with ` +
|
|
`${OPENAI_RENDER_MARKER}. When it does, speak only the content after the marker naturally, without adding, ` +
|
|
`removing, or changing facts. Do not call tools while rendering that message.`;
|
|
const VOCERO_USER_CONTEXT_HINT =
|
|
" Caller profile resolved from the page phoneNumber: ";
|
|
/** @type {[RegExp, string][]} */
|
|
const MCP_SEARCH_TRANSLATION_PHRASES = [
|
|
[/informaciones del usuario/g, "user information"],
|
|
[/informacion del usuario/g, "user information"],
|
|
[/informazioni utente/g, "user information"],
|
|
[/informazione utente/g, "user information"],
|
|
[/report del dia/g, "day report"],
|
|
[/reporte del dia/g, "day report"],
|
|
[/report di oggi/g, "today report"],
|
|
[/reporte de hoy/g, "today report"],
|
|
[/report giornaliero/g, "daily report"],
|
|
[/reporte diario/g, "daily report"],
|
|
[/reporte de ayer/g, "yesterday report"],
|
|
[/reporte del día anterior/g, "previous day report"],
|
|
[/report di ieri/g, "yesterday report"],
|
|
[/report del giorno prima/g, "previous day report"],
|
|
[/último reporte/g, "latest report"],
|
|
[/ultimo reporte/g, "latest report"],
|
|
[/ultimo report/g, "latest report"],
|
|
[/report più recente/g, "most recent report"],
|
|
[/reporte más reciente/g, "most recent report"],
|
|
[/reporte mas reciente/g, "most recent report"],
|
|
[/más reciente/g, "most recent"],
|
|
[/mas reciente/g, "most recent"],
|
|
[/più recente/g, "most recent"],
|
|
[/listado de reportes/g, "list reports report list"],
|
|
[/lista di report/g, "list reports report list"],
|
|
[/reportes recientes/g, "recent reports list"],
|
|
[/report recenti/g, "recent reports list"],
|
|
[/mi numero es/g, "my number is"],
|
|
[/il mio numero e/g, "my number is"],
|
|
[/numero di telefono/g, "phone number"],
|
|
[/numero telefonico/g, "phone number"],
|
|
[/numero de telefono/g, "phone number"],
|
|
[/numero de whatsapp/g, "whatsapp number"],
|
|
[/informacion api/g, "api information"],
|
|
[/informazioni api/g, "api information"],
|
|
];
|
|
/** @type {Record<string, string>} */
|
|
const MCP_SEARCH_TRANSLATION_TOKENS = {
|
|
ayuda: "help",
|
|
api: "api",
|
|
ayer: "yesterday",
|
|
ieri: "yesterday",
|
|
anterior: "previous",
|
|
precedente: "previous",
|
|
dami: "give",
|
|
dame: "give",
|
|
dammi: "give",
|
|
buscar: "search",
|
|
busca: "search",
|
|
busqueda: "search",
|
|
call: "call",
|
|
catalogo: "catalogue",
|
|
catalogue: "catalogue",
|
|
cerca: "search",
|
|
chiama: "call",
|
|
cliente: "customer",
|
|
clientes: "customers",
|
|
customer: "customer",
|
|
detalle: "detail",
|
|
detalles: "details",
|
|
dettagli: "details",
|
|
dia: "day",
|
|
diario: "daily",
|
|
dio: "day",
|
|
documento: "document",
|
|
documentos: "documents",
|
|
elenco: "list",
|
|
email: "email",
|
|
estado: "status",
|
|
estadoo: "status",
|
|
historial: "history",
|
|
hoy: "today",
|
|
informacion: "information",
|
|
informazioni: "information",
|
|
informe: "report",
|
|
informes: "reports",
|
|
lista: "list",
|
|
listado: "list",
|
|
liste: "lists",
|
|
mostrar: "show",
|
|
mostra: "show",
|
|
numero: "number",
|
|
numeros: "numbers",
|
|
obtener: "get",
|
|
ottieni: "get",
|
|
perfil: "profile",
|
|
profilo: "profile",
|
|
reciente: "recent",
|
|
recientes: "recent",
|
|
recente: "recent",
|
|
recenti: "recent",
|
|
reporte: "report",
|
|
reportes: "reports",
|
|
reporto: "report",
|
|
stato: "status",
|
|
telefono: "phone",
|
|
telefonoo: "phone",
|
|
usuario: "user",
|
|
utenti: "users",
|
|
utente: "user",
|
|
ultimo: "latest",
|
|
ultimi: "latest",
|
|
último: "latest",
|
|
últimos: "latest",
|
|
whatsapp: "whatsapp",
|
|
};
|
|
|
|
const STORAGE_KEYS = {
|
|
directUrl: "s2s.ws.directUrl",
|
|
voice: "s2s.ws.voice",
|
|
instructions: "s2s.ws.instructions",
|
|
tools: "s2s.ws.tools",
|
|
searchKey: "s2s.ws.searchKey",
|
|
noiseGate: "s2s.ws.noiseGate",
|
|
};
|
|
|
|
// ── Wake Word ("hola vocero") ─────────────────────────────────────────────
|
|
// Local browser Web Speech API (SpeechRecognition) with a tight JSGF grammar
|
|
// restricted to the trigger phrase, continuous/interim listening while the app
|
|
// is idle/error, and auto-restart on end so it keeps arming across Chrome's
|
|
// ~60s silence timeouts. On unsupported browsers (Firefox, Safari desktop <16)
|
|
// this module is a silent no-op and the caption stays actionable via tap only.
|
|
const WAKE_WORD_LANG = "es-ES";
|
|
const WAKE_WORD_PHRASES = [
|
|
"hola vocero",
|
|
"ola vocero",
|
|
"hola vocero",
|
|
"hola vocerò",
|
|
"ola vocerò",
|
|
"olá vocero",
|
|
"oye vocero",
|
|
"hola vocerro",
|
|
"hola vo zero",
|
|
"ola vo zero",
|
|
];
|
|
const WAKE_WORD_JSGF_GRAMMAR =
|
|
"#JSGF V1.0; grammar wake_word; public <wake> = hola vocero | ola vocero | hola vocerò | ola vocerò | olá vocero | oye vocero | hola vocerro ;";
|
|
const WAKE_WORD_RESTART_DELAY_MS = 250;
|
|
const WAKE_WORD_MAX_INTERIM_LEN = 64;
|
|
|
|
/** @typedef {any & { continuous: boolean, interimResults: boolean, lang: string, grammars: any, maxAlternatives: number, start: () => void, stop: () => void, abort: () => void, onresult: any, onerror: any, onend: any, onstart: any, onaudioend: any }} WakeRec */
|
|
/** @type {WakeRec | null} */
|
|
let wakeWordEngine = null;
|
|
let wakeWordArmed = false;
|
|
let wakeWordRestartTimer = 0;
|
|
let pendingWakeWordConfirmationText = "";
|
|
|
|
/** @param {string} s @returns {string} */
|
|
function _normalizeWakeWord(s) {
|
|
return (s || "")
|
|
.toLowerCase()
|
|
.normalize("NFD")
|
|
.replace(/[\u0300-\u036f]/g, "")
|
|
.replace(/[^a-z0-9\s]/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
/** @param {string} transcript */
|
|
function _matchesWakeWord(transcript) {
|
|
const hay = _normalizeWakeWord(transcript);
|
|
if (!hay) return false;
|
|
// Strict: contains a whole-phrase substring match of any variant.
|
|
for (const raw of WAKE_WORD_PHRASES) {
|
|
const needle = _normalizeWakeWord(raw);
|
|
if (needle && hay.includes(needle)) return true;
|
|
}
|
|
// Loose fallback for off-by-one phonemes: allow "vocero" preceded by any
|
|
// 3-5 char greeting-like word (e.g. "ola", "hola", "oye", "uy") to catch
|
|
// accent variations the grammar misses.
|
|
if (/\b(vocero|vocerro|vocerò|vo.?zero)\b/.test(hay) &&
|
|
/\b(hola|ola|oye|uy|holi|oli)\b/.test(hay)) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/** @param {WakeRec} engine */
|
|
function _installWakeWordHandlers(engine) {
|
|
engine.onresult = (/** @type {any} */ event) => {
|
|
if (!wakeWordArmed) return;
|
|
let aggregate = "";
|
|
for (let i = event.resultIndex; i < event.results.length; i += 1) {
|
|
const res = event.results[i];
|
|
const alt = res[0]?.transcript || "";
|
|
aggregate += ` ${alt}`;
|
|
if (aggregate.length > WAKE_WORD_MAX_INTERIM_LEN) break;
|
|
if (_matchesWakeWord(alt) || _matchesWakeWord(aggregate)) {
|
|
if (DEBUG) console.debug(`[WAKE WORD] MATCH transcript=${JSON.stringify(alt + (aggregate !== ` ${alt}` ? ` aggregate=${JSON.stringify(aggregate)}` : ""))}`);
|
|
console.info(`[WAKE WORD] ✓ Trigger fired — starting conversation`);
|
|
// Disarm *before* click: stop sharing the mic with SpeechRecognition
|
|
// while we're about to open getUserMedia for the realtime session.
|
|
wakeWordArmed = false;
|
|
try { engine.stop(); } catch { /* ignore */ }
|
|
if (wakeWordRestartTimer) { clearTimeout(wakeWordRestartTimer); wakeWordRestartTimer = 0; }
|
|
// Remember the exact phrase so onClientStatus(connected) can inject it
|
|
// into the live conversation and ask the model to reply to it (instead
|
|
// of silently waiting for the next user utterance).
|
|
const spoken = (alt || "").trim() || (aggregate || "").trim() || "hola vocero";
|
|
pendingWakeWordConfirmationText = spoken;
|
|
console.info(`[WAKE WORD] 🔁 Pending confirmation text: ${JSON.stringify(pendingWakeWordConfirmationText)}`);
|
|
// Pretend the user tapped the orb. If the AudioContext fails to
|
|
// resume (iOS Safari needs a real tap), doStart() will surface the
|
|
// error with the normal "Tap to retry" affordance.
|
|
circleBtn.click();
|
|
return;
|
|
}
|
|
}
|
|
if (DEBUG) {
|
|
const cleaned = aggregate.trim();
|
|
if (cleaned) console.debug(`[WAKE WORD] interim transcript=${JSON.stringify(cleaned.slice(0, WAKE_WORD_MAX_INTERIM_LEN))}`);
|
|
}
|
|
};
|
|
|
|
engine.onerror = (/** @type {any} */ event) => {
|
|
const err = event?.error || String(event || "unknown");
|
|
// "no-speech" / "audio-capture" / "aborted" are benign.
|
|
if (["no-speech", "aborted"].includes(err)) return;
|
|
console.warn(`[WAKE WORD] engine error: ${err}${event?.message ? ` (${event.message})` : ""}`);
|
|
// "not-allowed" or "service-not-allowed": the user/OS blocked the mic for
|
|
// the recognizer — permanently disarm so we don't thrash the permission.
|
|
if (err === "not-allowed" || err === "service-not-allowed") {
|
|
console.warn("[WAKE WORD] ✋ Permission denied — permanently disarming wake word (tap the orb to start)");
|
|
wakeWordArmed = false;
|
|
try { engine.stop(); } catch { /* ignore */ }
|
|
if (wakeWordRestartTimer) { clearTimeout(wakeWordRestartTimer); wakeWordRestartTimer = 0; }
|
|
}
|
|
};
|
|
|
|
engine.onend = () => {
|
|
if (!wakeWordArmed) return;
|
|
// Chrome terminates continuous recognition after ~60s of silence.
|
|
// Re-arm on the next tick so a single failed start doesn't loop.
|
|
if (wakeWordRestartTimer) clearTimeout(wakeWordRestartTimer);
|
|
wakeWordRestartTimer = window.setTimeout(() => {
|
|
if (!wakeWordArmed) return;
|
|
try {
|
|
engine.start();
|
|
if (DEBUG) console.debug("[WAKE WORD] 🔁 restarted continuous listener");
|
|
} catch (err) {
|
|
// Already-started is harmless; anything else we swallow briefly.
|
|
if (DEBUG) console.debug("[WAKE WORD] restart skipped:", err && err instanceof Error ? err.message : err);
|
|
}
|
|
}, WAKE_WORD_RESTART_DELAY_MS);
|
|
};
|
|
}
|
|
|
|
function startWakeWordListener() {
|
|
if (wakeWordArmed) return;
|
|
// Only in Chromium-derived and recent Safari WebViews with SpeechRecognition.
|
|
// @ts-ignore - vendor prefixes are intentional.
|
|
const Ctor = window.SpeechRecognition || window.webkitSpeechRecognition || window.mozSpeechRecognition;
|
|
if (!Ctor) {
|
|
if (DEBUG) console.debug("[WAKE WORD] ⚠️ Web Speech Recognition unavailable on this browser — tap the orb to start");
|
|
return;
|
|
}
|
|
try {
|
|
/** @type {WakeRec} */
|
|
// @ts-ignore
|
|
const engine = new Ctor();
|
|
engine.continuous = true;
|
|
engine.interimResults = true;
|
|
engine.lang = WAKE_WORD_LANG;
|
|
engine.maxAlternatives = 1;
|
|
// Narrow the grammar so interim results are biased toward the phrase.
|
|
try {
|
|
// @ts-ignore
|
|
const list = window.SpeechGrammarList || window.webkitSpeechGrammarList;
|
|
if (list) {
|
|
// @ts-ignore
|
|
const gl = new list();
|
|
gl.addFromString(WAKE_WORD_JSGF_GRAMMAR, 1.0);
|
|
engine.grammars = gl;
|
|
}
|
|
} catch {
|
|
// Grammar support is optional; fallback to pure transcript matching.
|
|
}
|
|
_installWakeWordHandlers(engine);
|
|
engine.start();
|
|
wakeWordEngine = engine;
|
|
wakeWordArmed = true;
|
|
console.info("[WAKE WORD] 🎯 Armed — listening for \"Hola Vocero\"");
|
|
} catch (err) {
|
|
console.warn("[WAKE WORD] failed to start arming:", err && err instanceof Error ? err.message : err);
|
|
wakeWordArmed = false;
|
|
}
|
|
}
|
|
|
|
function stopWakeWordListener() {
|
|
if (!wakeWordArmed && !wakeWordEngine && !pendingWakeWordConfirmationText) return;
|
|
wakeWordArmed = false;
|
|
if (wakeWordRestartTimer) { clearTimeout(wakeWordRestartTimer); wakeWordRestartTimer = 0; }
|
|
if (wakeWordEngine) {
|
|
try { wakeWordEngine.abort(); } catch { /* ignore */ }
|
|
try { wakeWordEngine.stop(); } catch { /* ignore */ }
|
|
wakeWordEngine = null;
|
|
}
|
|
if (DEBUG) console.debug("[WAKE WORD] ⏸️ Disarmed for live session");
|
|
}
|
|
|
|
// ── Noise gate ──────────────────────────────────────────────────────────────
|
|
// The Settings cursor sets the gate's open threshold in dBFS. Its leftmost
|
|
// position is an OFF detent (gate disabled, pure passthrough); the rest of the
|
|
// travel is the active threshold. The cursor shares the meter's dB axis, so the
|
|
// handle sits on the level bar — raise it until room noise stops lighting it up.
|
|
// The slider range IS the shared axis: the live meter fill and the threshold
|
|
// thumb both map across [GATE_OFF_DB, GATE_MAX_DB], so the thumb sits exactly
|
|
// where the gate cuts on the same scale as the level bar.
|
|
const GATE_OFF_DB = -66; // slider minimum = off / bottom of the meter axis
|
|
const GATE_MAX_DB = -3; // slider maximum = most aggressive / top of the meter axis
|
|
const GATE_DEFAULT_DB = -50; // first-run default: a gentle gate, enabled
|
|
|
|
/** @param {number} thresholdDb @returns {import("./ws/s2s-ws-client.js").NoiseGate} */
|
|
function gateParams(thresholdDb) {
|
|
return { enabled: thresholdDb > GATE_OFF_DB, thresholdDb };
|
|
}
|
|
|
|
// ── Tools ─────────────────────────────────────────────────────────────────
|
|
// Function tools we declare to the backend. The model decides when to call
|
|
// one; the executor below runs it and returns the result (see runTool).
|
|
/** @type {Record<string, import("./ws/s2s-ws-client.js").ToolDef>} */
|
|
const TOOL_DEFS = {
|
|
web_search: {
|
|
type: "function",
|
|
name: "web_search",
|
|
description:
|
|
"Search the web for current or factual information you don't already know " +
|
|
"(news, prices, facts, documentation). Returns the top results with titles, " +
|
|
"snippets and URLs.",
|
|
parameters: {
|
|
type: "object",
|
|
properties: { query: { type: "string", description: "The search query." } },
|
|
required: ["query"],
|
|
},
|
|
},
|
|
};
|
|
|
|
/** @type {import("./ws/s2s-ws-client.js").ToolDef[]} */
|
|
let agentToolDefs = [];
|
|
|
|
function loadSettings() {
|
|
return {
|
|
directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "",
|
|
voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE,
|
|
instructions: localStorage.getItem(STORAGE_KEYS.instructions) || DEFAULT_INSTRUCTIONS,
|
|
noiseGate: loadGateThreshold(),
|
|
};
|
|
}
|
|
|
|
/** Stored gate threshold (dBFS), clamped to the slider range. Defaults to a
|
|
* gentle enabled gate (GATE_DEFAULT_DB) when the user hasn't set one yet. */
|
|
function loadGateThreshold() {
|
|
const stored = localStorage.getItem(STORAGE_KEYS.noiseGate);
|
|
// getItem returns null when unset, and Number(null) === 0 (finite!), so guard
|
|
// the missing/empty case explicitly before coercing — otherwise the default
|
|
// never fires and 0 clamps to the slider max.
|
|
if (stored === null || stored === "") return GATE_DEFAULT_DB;
|
|
const raw = Number(stored);
|
|
if (!Number.isFinite(raw)) return GATE_DEFAULT_DB;
|
|
return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(raw)));
|
|
}
|
|
|
|
/** @param {ReturnType<typeof loadSettings>} s */
|
|
function saveSettings(s) {
|
|
localStorage.setItem(STORAGE_KEYS.directUrl, s.directUrl);
|
|
localStorage.setItem(STORAGE_KEYS.voice, s.voice);
|
|
localStorage.setItem(STORAGE_KEYS.instructions, s.instructions);
|
|
localStorage.setItem(STORAGE_KEYS.noiseGate, String(s.noiseGate));
|
|
}
|
|
|
|
/** @returns {{ web_search: boolean }} */
|
|
function loadTools() {
|
|
try {
|
|
const raw = JSON.parse(localStorage.getItem(STORAGE_KEYS.tools) || "{}");
|
|
return {
|
|
web_search: raw.web_search ?? true,
|
|
};
|
|
} catch {
|
|
return { web_search: true };
|
|
}
|
|
}
|
|
|
|
function saveTools() {
|
|
localStorage.setItem(STORAGE_KEYS.tools, JSON.stringify(toolsEnabled));
|
|
}
|
|
|
|
/** @type {Record<AppState, { caption: string; disabled: boolean }>} */
|
|
const STATE_VIEWS = {
|
|
idle: { caption: "Tap or say \"Hola Vocero\"", disabled: false },
|
|
connecting: { caption: "Connecting", disabled: true },
|
|
queued: { caption: "Finding you a spot…", disabled: true },
|
|
"your-turn": { caption: "You're up! 🎉", disabled: true },
|
|
listening: { caption: "", disabled: false },
|
|
"user-speaking": { caption: "", disabled: false },
|
|
processing: { caption: "", disabled: false },
|
|
"ai-speaking": { caption: "", disabled: false },
|
|
error: { caption: "Tap to retry", disabled: false },
|
|
};
|
|
|
|
/** @type {Record<AppState, string>} */
|
|
const STATE_CLASS = {
|
|
idle: "state-idle",
|
|
connecting: "state-connecting",
|
|
queued: "state-queued",
|
|
"your-turn": "state-your-turn",
|
|
listening: "state-listening",
|
|
"user-speaking": "state-user-speaking",
|
|
processing: "state-processing",
|
|
"ai-speaking": "state-ai-speaking",
|
|
error: "state-error",
|
|
};
|
|
|
|
/** @type {ReadonlySet<AppState>} */
|
|
const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]);
|
|
|
|
/** @type {HTMLButtonElement} */
|
|
const circleBtn = $("#main-circle");
|
|
/** @type {HTMLParagraphElement} */
|
|
const circleCaption = $("#circle-caption");
|
|
/** @type {HTMLParagraphElement} */
|
|
const circleSubcaption = $("#circle-subcaption");
|
|
/** @type {HTMLElement} */
|
|
const orbWrap = $(".orb-wrap");
|
|
/** @type {HTMLButtonElement} */
|
|
const micBtn = $("#mic-btn");
|
|
/** @type {HTMLButtonElement} */
|
|
const stopBtn = $("#stop-btn");
|
|
/** @type {HTMLElement} */
|
|
const queueActions = $("#queue-actions");
|
|
/** @type {HTMLButtonElement} */
|
|
const joinQueueBtn = $("#join-queue-btn");
|
|
/** @type {HTMLButtonElement} */
|
|
const leaveQueueBtn = $("#leave-queue-btn");
|
|
|
|
/** @type {HTMLButtonElement} */
|
|
const settingsBtn = $("#settings-btn");
|
|
/** @type {HTMLDialogElement} */
|
|
const settingsModal = $("#settings-modal");
|
|
|
|
/** @type {HTMLButtonElement | null} */
|
|
const aboutBtn = document.querySelector("#about-btn");
|
|
/** @type {HTMLDialogElement} */
|
|
const aboutModal = $("#about-modal");
|
|
/** @type {HTMLButtonElement} */
|
|
const aboutClose = $("#about-close");
|
|
|
|
/** @type {HTMLButtonElement} */
|
|
const toolsBtn = $("#tools-btn");
|
|
/** @type {HTMLDialogElement} */
|
|
const toolsModal = $("#tools-modal");
|
|
/** @type {HTMLButtonElement} */
|
|
const toolsClose = $("#tools-close");
|
|
/** @type {HTMLInputElement} */
|
|
const toolWebSwitch = $("#tool-web");
|
|
/** @type {HTMLElement} */
|
|
const toolWebRow = $("#tool-web-row");
|
|
/** @type {HTMLElement} */
|
|
const toolWebHint = $("#tool-web-hint");
|
|
/** @type {HTMLInputElement} */
|
|
const toolCamSwitch = $("#tool-cam");
|
|
/** @type {HTMLElement} */
|
|
const toolCamHint = $("#tool-cam-hint");
|
|
/** @type {HTMLElement} */
|
|
const camPip = $("#cam-pip");
|
|
/** @type {HTMLVideoElement} */
|
|
const camVideo = $("#cam-video");
|
|
/** @type {HTMLElement} */
|
|
const agentToolsHint = $("#agent-tools-hint");
|
|
/** @type {HTMLElement} */
|
|
const agentToolsList = $("#agent-tools-list");
|
|
/** @type {HTMLInputElement} */
|
|
const searchKeyInput = $("#search-key");
|
|
|
|
/** @type {HTMLInputElement} */
|
|
const inputLbUrl = $("#lb-url");
|
|
/** @type {HTMLElement} */
|
|
const connField = $("#conn-field");
|
|
/** @type {HTMLElement} */
|
|
const connHint = $("#conn-hint");
|
|
/** @type {HTMLSelectElement} */
|
|
const inputVoice = $("#voice");
|
|
/** @type {HTMLTextAreaElement} */
|
|
const inputInstructions = $("#instructions");
|
|
/** @type {HTMLElement} */
|
|
const primaryPromptMeta = $("#primary-prompt-meta");
|
|
/** @type {HTMLElement} */
|
|
const identUserName = $("#ident-user-name");
|
|
/** @type {HTMLInputElement} */
|
|
const inputNoiseGate = $("#noise-gate");
|
|
/** @type {HTMLElement} */
|
|
const gateValue = $("#gate-value");
|
|
/** @type {HTMLElement} */
|
|
const gateMeterFill = $("#gate-meter-fill");
|
|
/** @type {HTMLElement} */
|
|
const micGate = $("#mic-gate");
|
|
const mgaArc = /** @type {SVGSVGElement} */ (document.querySelector("#mic-gate-arc"));
|
|
const mgaTrack = /** @type {SVGPathElement} */ (document.querySelector("#mga-track"));
|
|
const mgaFill = /** @type {SVGPathElement} */ (document.querySelector("#mga-fill"));
|
|
const mgaHit = /** @type {SVGPathElement} */ (document.querySelector("#mga-hit"));
|
|
const mgaHandle = /** @type {SVGCircleElement} */ (document.querySelector("#mga-handle"));
|
|
/** @type {HTMLButtonElement} */
|
|
const restartBtn = $("#restart-conversation");
|
|
/** @type {HTMLElement} */
|
|
const restartHint = $("#restart-hint");
|
|
const settingsForm = /** @type {HTMLFormElement} */ (settingsModal.querySelector("form"));
|
|
|
|
/** @type {AppState} */
|
|
let currentState = "idle";
|
|
let settings = loadSettings();
|
|
|
|
// ── Connection target ────────────────────────────────────────────────────────
|
|
// Three modes, decided by the deploy via /api/config:
|
|
// • SPEECH_TO_SPEECH_URL set -> direct mode pinned by the deploy: the browser
|
|
// connects straight to that URL, shown read-only in Settings. Overrides the
|
|
// load balancer entirely.
|
|
// • LOAD_BALANCER_URL set -> original flow: POST the same-origin /api/session
|
|
// proxy (the server forwards to the LB; the LB address is never sent here).
|
|
// • neither (allowDirect) -> the user sets a speech-to-speech server URL and
|
|
// the browser connects to it directly (no load balancer, no /session).
|
|
let lbMode = false;
|
|
let reasoningType = "PROXY";
|
|
// Fail open: direct entry is allowed unless /api/config reports an LB URL. This
|
|
// way a missing/unreachable config (e.g. static hosting) leaves the field
|
|
// usable rather than locked.
|
|
let allowDirect = true;
|
|
// Deploy-pinned s2s URL (SPEECH_TO_SPEECH_URL). Non-empty -> locked direct
|
|
// mode: the field displays it read-only and the saved user URL is untouched.
|
|
let pinnedUrl = "";
|
|
// Deploy-pinned primary prompt (S2S_PRIMARY_PROMPT). Non-empty -> locked prompt.
|
|
let pinnedPrompt = "";
|
|
let pinnedPromptSource = "";
|
|
|
|
// ── Tool state ──────────────────────────────────────────────────────────────
|
|
let toolsEnabled = loadTools();
|
|
// Whether the server holds a Serper key (learned from /api/config on load).
|
|
let serverSearchKey = false;
|
|
// A user-supplied key (fallback when the deploy has none). localStorage only.
|
|
let userSearchKey = localStorage.getItem(STORAGE_KEYS.searchKey) || "";
|
|
let searchMcpEnabled = false;
|
|
let lastUserQuery = "";
|
|
/** @type {{ tool: import("./ws/s2s-ws-client.js").ToolDef, score: number }[]} */
|
|
let rankedAgentTools = [];
|
|
let liveInstructionsSnapshot = "";
|
|
let liveToolPolicySnapshot = "";
|
|
/** @type {Record<string, { argsText: string, status: "idle" | "running" | "success" | "error", resultText: string, lastAction: "call" | "send" }>} */
|
|
let mcpToolCallState = Object.create(null);
|
|
const pageParams = new URLSearchParams(window.location.search);
|
|
const voceroUserPhoneNumber = (pageParams.get("phoneNumber") || "").trim();
|
|
/** @type {Record<string, unknown> | null} */
|
|
let voceroUserProfile = null;
|
|
let voceroUserProfileLoaded = false;
|
|
const AUTO_MCP_MIN_SCORE = 20;
|
|
const AUTO_MCP_MIN_SCORE_WITH_LEAD = 12;
|
|
const AUTO_MCP_MIN_LEAD = 4;
|
|
const AUTO_MCP_RELAXED_REPORT_SCORE = 15;
|
|
const AUTO_MCP_FAST_DELAY_MS = 150;
|
|
const AUTO_MCP_FALLBACK_DELAY_MS = 1200;
|
|
let autoMcpInFlight = false;
|
|
let lastAutoMcpTurnKey = "";
|
|
let pendingAutoMcpTimer = 0;
|
|
let lastDeclaredAssistantToolKey = "";
|
|
const processedReasoningTurns = new Set();
|
|
|
|
/**
|
|
* @param {string} hypothesisId
|
|
* @param {string} location
|
|
* @param {string} msg
|
|
* @param {Record<string, unknown>} [data]
|
|
*/
|
|
function reportDebug(hypothesisId, location, msg, data = {}) {
|
|
void hypothesisId;
|
|
void location;
|
|
void msg;
|
|
void data;
|
|
}
|
|
|
|
/** Search is usable if the server has a key or the user supplied one. */
|
|
function searchAvailable() {
|
|
return serverSearchKey || !!userSearchKey;
|
|
}
|
|
|
|
/** Tool definitions for the currently-enabled (and usable) tools. */
|
|
function activeToolDefs() {
|
|
const defs = [];
|
|
if (toolsEnabled.web_search && searchAvailable()) defs.push(TOOL_DEFS.web_search);
|
|
return defs.concat(agentToolDefs);
|
|
}
|
|
|
|
/** @returns {string} */
|
|
function voceroUserContextText() {
|
|
if (!voceroUserProfileLoaded || !voceroUserProfile || !voceroUserPhoneNumber) return "";
|
|
let serialized = JSON.stringify(voceroUserProfile);
|
|
if (serialized.length > 1600) serialized = `${serialized.slice(0, 1600)}...`;
|
|
return `${VOCERO_USER_CONTEXT_HINT}${serialized}. Use this as trusted caller context when relevant.`;
|
|
}
|
|
|
|
/**
|
|
* @param {Record<string, unknown> | null} profile
|
|
* @returns {string}
|
|
*/
|
|
function extractVoceroUserName(profile) {
|
|
if (!profile) return "";
|
|
const directCandidates = [
|
|
profile.name,
|
|
profile.fullName,
|
|
profile.full_name,
|
|
profile.displayName,
|
|
profile.display_name,
|
|
profile.username,
|
|
];
|
|
for (const value of directCandidates) {
|
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
}
|
|
const firstName = typeof profile.firstName === "string" ? profile.firstName.trim() : "";
|
|
const lastName = typeof profile.lastName === "string" ? profile.lastName.trim() : "";
|
|
if (firstName || lastName) return `${firstName} ${lastName}`.trim();
|
|
const nestedKeys = ["user", "customer", "profile", "contact"];
|
|
for (const key of nestedKeys) {
|
|
const nested = profile[key];
|
|
if (nested && typeof nested === "object" && !Array.isArray(nested)) {
|
|
const nestedName = extractVoceroUserName(/** @type {Record<string, unknown>} */ (nested));
|
|
if (nestedName) return nestedName;
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function syncVoceroHeaderUser() {
|
|
const name = extractVoceroUserName(voceroUserProfile);
|
|
identUserName.hidden = !name;
|
|
identUserName.textContent = name ? `Utente: ${name}` : "";
|
|
}
|
|
|
|
/**
|
|
* @param {import("./ws/s2s-ws-client.js").ToolDef[]} tools
|
|
* @returns {string}
|
|
*/
|
|
function buildMcpToolCatalogSummary(tools) {
|
|
return tools
|
|
.map((tool) => {
|
|
const required = requiredToolParams(tool.parameters);
|
|
const requiredText = required.length ? ` required: ${required.join(", ")}` : " required: none";
|
|
const desc = (tool.description || "No description provided.")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
return `${tool.name} - ${desc}.${requiredText}.`;
|
|
})
|
|
.join(" ");
|
|
}
|
|
|
|
/** Instructions used by the reasoning engine. */
|
|
function effectiveReasoningInstructions() {
|
|
const base = pinnedPrompt || settings.instructions;
|
|
let instructions = base;
|
|
if (activeToolDefs().length) instructions += TOOL_USE_HINT;
|
|
if (searchMcpEnabled && agentToolDefs.length) {
|
|
instructions += MCP_SEARCH_HINT;
|
|
instructions += ` Available MCP tools: ${buildMcpToolCatalogSummary(agentToolDefs)}`;
|
|
}
|
|
const suggested = topSuggestedAgentTools(3);
|
|
if (searchMcpEnabled && suggested.length) {
|
|
const lines = suggested
|
|
.map((entry, index) => `${index + 1}. ${entry.tool.name}: ${entry.tool.description || "No description provided."}`)
|
|
.join(" ");
|
|
instructions += ` Current user request: "${lastUserQuery}". Client-side ranking hint only: ${lines}. Use this hint only as a secondary signal after reviewing the full MCP catalog.`;
|
|
}
|
|
const userContext = voceroUserContextText();
|
|
if (userContext) instructions += userContext;
|
|
return instructions;
|
|
}
|
|
|
|
/** Instructions pushed to the realtime ASR/TTS backend. */
|
|
function effectiveInstructions() {
|
|
const instructions = effectiveReasoningInstructions();
|
|
return reasoningType === "OPENAI" ? `${instructions}${OPENAI_RENDER_HINT}` : instructions;
|
|
}
|
|
|
|
/**
|
|
* @param {string} answer
|
|
* @returns {string}
|
|
*/
|
|
function buildRenderOnlyPrompt(answer) {
|
|
const clean = String(answer || "").trim();
|
|
return clean ? `${OPENAI_RENDER_MARKER}\n${clean}` : "";
|
|
}
|
|
|
|
/**
|
|
* @returns {string}
|
|
*/
|
|
function preferredResponseLanguage() {
|
|
const lang = (navigator.language || document.documentElement.lang || "").trim();
|
|
return lang || "";
|
|
}
|
|
|
|
/**
|
|
* @param {string} transcript
|
|
* @returns {Promise<{ answer: string, log_only: boolean, raw: any }>}
|
|
*/
|
|
async function fetchOpenAiReasoningAnswer(transcript) {
|
|
const res = await fetch("api/reasoning", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
transcript,
|
|
instructions: effectiveReasoningInstructions(),
|
|
language: preferredResponseLanguage(),
|
|
user_context: voceroUserContextText(),
|
|
phoneNumber: voceroUserPhoneNumber || null,
|
|
}),
|
|
});
|
|
if (!res.ok) {
|
|
let detail = String(res.status);
|
|
try {
|
|
const json = await res.json();
|
|
if (json?.detail) detail = String(json.detail);
|
|
} catch {}
|
|
throw new Error(`reasoning error (${detail})`);
|
|
}
|
|
const json = await res.json();
|
|
const answer = typeof json?.answer === "string" ? json.answer.trim() : "";
|
|
const log_only = Boolean(json && json.log_only === true);
|
|
// Log-only mode: può tornare answer="" — non è un errore.
|
|
if (!answer && !log_only) throw new Error("reasoning returned no answer");
|
|
return { answer, log_only, raw: json };
|
|
}
|
|
|
|
/**
|
|
* @param {string} transcript
|
|
* @param {string} itemId
|
|
*/
|
|
async function maybeRunOpenAiReasoning(transcript, itemId) {
|
|
if (!["OPENAI", "NODE_RED"].includes(reasoningType)) return;
|
|
const text = transcript.trim();
|
|
const turnKey = itemId || text;
|
|
if (!text || !turnKey || processedReasoningTurns.has(turnKey)) return;
|
|
processedReasoningTurns.add(turnKey);
|
|
if (!client || !LIVE_STATES.has(currentState)) return;
|
|
clearPendingAutoMcpFallback();
|
|
reportDebug("R", "main.js:maybeRunOpenAiReasoning", "starting external reasoning turn", {
|
|
itemId,
|
|
transcriptLength: text.length,
|
|
currentState,
|
|
});
|
|
let result;
|
|
try {
|
|
result = await fetchOpenAiReasoningAnswer(text);
|
|
} catch (err) {
|
|
const msg = err instanceof Error ? err.message : String(err);
|
|
console.warn(`[Vocero REASONING] error itemId=${JSON.stringify(itemId)} error=${msg}`);
|
|
reportDebug("R", "main.js:maybeRunOpenAiReasoning", "external reasoning failed", {
|
|
itemId,
|
|
error: msg,
|
|
});
|
|
return;
|
|
}
|
|
// ── LOG-ONLY MODE (FASE 1) ─────────────────────────────────────────────
|
|
// Node-RED si limita a loggare e torna log_only=true.
|
|
// NON cancelliamo la risposta backend, NON iniettiamo testo, NON parliamo.
|
|
// Il flusso resta COMPLETAMENTE a carico del backend S2S nativo.
|
|
if (result && result.log_only) {
|
|
reportDebug("R", "main.js:maybeRunOpenAiReasoning", "external reasoning in LOG_ONLY mode — pass through to native S2S backend", {
|
|
itemId,
|
|
reasoningType,
|
|
intent: (result.raw && result.raw.intent) || "log_only",
|
|
});
|
|
console.debug(
|
|
`[Vocero REASONING LOG_ONLY] mode=log_only engine=${reasoningType} ` +
|
|
`transcript=${JSON.stringify(text.slice(0, 160))} ` +
|
|
`intent=${JSON.stringify((result.raw && result.raw.intent) || "log_only")}`
|
|
);
|
|
return;
|
|
}
|
|
// ── MODE STANDARD (fasi successive) ────────────────────────────────────
|
|
// Il reasoning esterno vince: annulla la risposta nativa, inserisci il
|
|
// testo dell'assistente e parla via SpeechSynthesis.
|
|
const answer = (result && result.answer) || "";
|
|
if (!answer) {
|
|
console.warn(`[Vocero REASONING] empty non-log-only answer itemId=${JSON.stringify(itemId)}`);
|
|
return;
|
|
}
|
|
if (typeof client.cancelActiveResponse === "function") {
|
|
client.cancelActiveResponse();
|
|
} else if (typeof client.suppressAssistantAudio === "function") {
|
|
client.suppressAssistantAudio();
|
|
}
|
|
reportDebug("R", "main.js:maybeRunOpenAiReasoning", "received external reasoning answer", {
|
|
itemId,
|
|
answerLength: answer.length,
|
|
});
|
|
if (typeof client.appendAssistantText === "function") {
|
|
client.appendAssistantText(answer);
|
|
} else {
|
|
const renderPrompt = buildRenderOnlyPrompt(answer);
|
|
if (renderPrompt) {
|
|
client.sendUserText(renderPrompt);
|
|
client.requestResponse();
|
|
}
|
|
}
|
|
try {
|
|
if ("speechSynthesis" in window) {
|
|
const utter = new SpeechSynthesisUtterance(answer);
|
|
const want = (preferredResponseLanguage() || "en").slice(0, 2).toLowerCase();
|
|
const voices = window.speechSynthesis.getVoices
|
|
? window.speechSynthesis.getVoices()
|
|
: [];
|
|
const match = Array.isArray(voices) && voices.length > 0
|
|
? (voices.find(v => (v.lang || "").slice(0, 2).toLowerCase() === want) || voices[0])
|
|
: null;
|
|
if (match) utter.voice = match;
|
|
if (want) utter.lang = want;
|
|
utter.rate = 1.0;
|
|
utter.pitch = 1.0;
|
|
window.speechSynthesis.cancel();
|
|
window.speechSynthesis.speak(utter);
|
|
}
|
|
} catch (ttsErr) {
|
|
console.warn("[Vocero REASONING] speechSynthesis failed:", ttsErr);
|
|
}
|
|
}
|
|
|
|
/** @param {string} instructions */
|
|
function pushLiveInstructions(instructions) {
|
|
if (!client || !LIVE_STATES.has(currentState)) return;
|
|
if (instructions === liveInstructionsSnapshot) return;
|
|
liveInstructionsSnapshot = instructions;
|
|
// #region debug-point C:push-live-instructions
|
|
reportDebug("C", "main.js:pushLiveInstructions", "pushing computed instructions to live session", {
|
|
instructionLength: instructions.length,
|
|
lastUserQuery,
|
|
suggestedTools: topSuggestedAgentTools(3).map((entry) => entry.tool.name),
|
|
});
|
|
// #endregion
|
|
client.updateSession({ instructions });
|
|
}
|
|
|
|
function pushLiveToolPolicy() {
|
|
if (!client || !LIVE_STATES.has(currentState)) return;
|
|
const tools = activeToolDefs();
|
|
const toolChoice = tools.length ? "auto" : "none";
|
|
const snapshot = JSON.stringify({ toolChoice, toolNames: tools.map((tool) => tool.name) });
|
|
if (snapshot === liveToolPolicySnapshot) return;
|
|
liveToolPolicySnapshot = snapshot;
|
|
// #region debug-point B:push-live-tool-policy
|
|
reportDebug("B", "main.js:pushLiveToolPolicy", "pushing live tool policy", {
|
|
toolChoice,
|
|
toolNames: tools.map((tool) => tool.name),
|
|
lastUserQuery,
|
|
searchMcpEnabled,
|
|
});
|
|
// #endregion
|
|
client.setTools(tools, toolChoice);
|
|
}
|
|
|
|
function clearPendingAutoMcpFallback() {
|
|
if (!pendingAutoMcpTimer) return;
|
|
window.clearTimeout(pendingAutoMcpTimer);
|
|
pendingAutoMcpTimer = 0;
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @returns {import("./ws/s2s-ws-client.js").ToolDef | undefined}
|
|
*/
|
|
function extractDeclaredAssistantTool(text) {
|
|
const match = text.match(/<\s*calls?\s+tool\s*:\s*([a-z0-9_.-]+)\s*>/i);
|
|
if (!match?.[1]) return undefined;
|
|
const toolName = match[1].trim();
|
|
return findAgentToolDef(toolName);
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @param {string} responseId
|
|
*/
|
|
function maybeRunDeclaredAssistantTool(text, responseId = "") {
|
|
if (!client || !LIVE_STATES.has(currentState) || autoMcpInFlight) return;
|
|
const tool = extractDeclaredAssistantTool(text);
|
|
if (!tool) return;
|
|
const dedupeBase = responseId || normalizeMcpSearchText(text);
|
|
const dedupeKey = `${dedupeBase}::${tool.name}`;
|
|
if (dedupeKey === lastDeclaredAssistantToolKey) return;
|
|
lastDeclaredAssistantToolKey = dedupeKey;
|
|
clearPendingAutoMcpFallback();
|
|
autoMcpInFlight = true;
|
|
console.debug(
|
|
`[Vocero MCP DECLARED] trigger responseId=${JSON.stringify(responseId)} tool=${tool.name} ` +
|
|
`query=${JSON.stringify(lastUserQuery)}`
|
|
);
|
|
client.suppressAssistantAudio?.();
|
|
chat.onToolCall(tool.name);
|
|
void callAgentToolFromPanel(tool.name, true, lastUserQuery).finally(() => {
|
|
autoMcpInFlight = false;
|
|
});
|
|
}
|
|
|
|
/** Push the active tool set to a live session so toggles apply mid-call. */
|
|
function pushToolsToSession() {
|
|
if (!client || !LIVE_STATES.has(currentState)) return;
|
|
const tools = activeToolDefs();
|
|
// #region debug-point B:push-tools-session
|
|
reportDebug("B", "main.js:pushToolsToSession", "pushing tools to live session", {
|
|
state: currentState,
|
|
toolNames: tools.map((tool) => tool.name),
|
|
toolCount: tools.length,
|
|
searchMcpEnabled,
|
|
});
|
|
// #endregion
|
|
liveToolPolicySnapshot = "";
|
|
client.setTools(tools, tools.length ? "auto" : "none");
|
|
// The hidden tool-use hint depends on whether any tool is active, so refresh
|
|
// instructions alongside the tool set.
|
|
const instructions = effectiveInstructions();
|
|
// #region debug-point C:push-instructions-session
|
|
reportDebug("C", "main.js:pushToolsToSession", "updating live session instructions", {
|
|
instructionLength: instructions.length,
|
|
hasSuggestedTools: topSuggestedAgentTools(3).length > 0,
|
|
lastUserQuery,
|
|
});
|
|
// #endregion
|
|
pushLiveInstructions(instructions);
|
|
}
|
|
|
|
// ── Chat view ───────────────────────────────────────────────────────────────
|
|
// Owns the history panel, the ephemeral bubbles, and all transcript/tool
|
|
// streaming state. The client's events are forwarded to its on* methods.
|
|
const chat = new ChatView();
|
|
|
|
// ── Account / limiter ─────────────────────────────────────────────────────
|
|
// Login chip + daily-limit modal (inert unless the deploy is in LB mode). The
|
|
// server meters conversation time; the client just heartbeats a live session
|
|
// and tears down when the server reports the budget is spent.
|
|
const account = new Account();
|
|
let limiterOn = false;
|
|
let heartbeatTimer = 0;
|
|
let trackedSessionId = "";
|
|
let trackedTier = "";
|
|
// The waiting-queue ticket id while we're in line (else ""). Used to leave the
|
|
// queue on teardown / tab-close so we don't hold a phantom place.
|
|
let queuedTicketId = "";
|
|
|
|
// ── Idle auto-close (30s of silence, never during AI speech) ─────────────
|
|
// The user asked the voice assistant to go to sleep silently after 30s of no
|
|
// activity. "Activity" is any meaningful event on either side: the user
|
|
// started speaking (VAD), the assistant started speaking (TTS), a partial or
|
|
// final transcript arrived, a tool call was requested or returned, or the
|
|
// client moved out of a transient pre-live state (connecting / queue /
|
|
// your-turn). Once the session is live, every tick of activity resets the
|
|
// countdown back to 30s; if 30s expire without a single event we close the
|
|
// socket, refund any reserved budget, and drop back to the idle orb so the
|
|
// microphone stops listening and the daily budget isn't eaten by a forgotten
|
|
// tab. The countdown is intentionally disabled in the pre-live states (we
|
|
// don't want to close while the user is still waiting in queue) and during
|
|
// `ai-speaking` — if the *system* is still talking we NEVER close the
|
|
// session mid-reply, even on very long TTS turns (the TTS engine itself will
|
|
// fire transcript/response-finished events that re-arm the timer afterward).
|
|
const IDLE_TIMEOUT_MS = 30 * 1000;
|
|
let idleCloseTimer = 0;
|
|
let idleCloseLastActivityAtMs = 0;
|
|
/** @type {Set<string>} - States where a live session is open and the user could be silent. `ai-speaking` is intentionally excluded so we never kill a reply mid-TTS. */
|
|
const IDLE_COUNT_STATES = new Set(["listening", "user-speaking", "processing"]);
|
|
|
|
function _kickIdleTimer(reason = "activity") {
|
|
idleCloseLastActivityAtMs = Date.now();
|
|
if (!IDLE_COUNT_STATES.has(currentState)) {
|
|
if (idleCloseTimer) { clearTimeout(idleCloseTimer); idleCloseTimer = 0; }
|
|
return;
|
|
}
|
|
if (idleCloseTimer) clearTimeout(idleCloseTimer);
|
|
idleCloseTimer = window.setTimeout(() => {
|
|
const staleMs = Date.now() - idleCloseLastActivityAtMs;
|
|
if (staleMs + 500 < IDLE_TIMEOUT_MS) {
|
|
// Race with a very recent kick: re-arm rather than close.
|
|
if (DEBUG) console.debug(`[IDLE] fire skipped (staleMs=${staleMs} < ${IDLE_TIMEOUT_MS}), re-arming`);
|
|
_kickIdleTimer("re-arm");
|
|
return;
|
|
}
|
|
console.info(`[IDLE] ⏰ Closing session after ${(staleMs / 1000).toFixed(1)}s of inactivity (timeout=${IDLE_TIMEOUT_MS / 1000}s) — reason=${reason}`);
|
|
idleCloseTimer = 0;
|
|
// Friendly caption flash right before teardown moves us back to the idle
|
|
// copy (so the user understands *why* the call ended).
|
|
setCaption("Chiuso per inattività", "muted");
|
|
void (async () => {
|
|
try {
|
|
await teardown();
|
|
} catch (err) {
|
|
console.warn("[IDLE] teardown failed during auto-close:", err && err instanceof Error ? err.message : err);
|
|
}
|
|
// teardown() calls setState("idle") which restores the normal idle
|
|
// caption. Leave a visible breadcrumb on the subcaption briefly so the
|
|
// user can tell what happened instead of wondering why the orb reset.
|
|
circleSubcaption.textContent = "30 sec sil.";
|
|
circleSubcaption.hidden = false;
|
|
window.setTimeout(() => { circleSubcaption.hidden = true; }, 8000);
|
|
})();
|
|
}, IDLE_TIMEOUT_MS);
|
|
if (DEBUG) {
|
|
console.debug(`[IDLE] 🔄 reset timer (${IDLE_TIMEOUT_MS / 1000}s) reason=${reason} state=${currentState}`);
|
|
}
|
|
}
|
|
|
|
function _cancelIdleTimer() {
|
|
if (idleCloseTimer) { clearTimeout(idleCloseTimer); idleCloseTimer = 0; }
|
|
idleCloseLastActivityAtMs = 0;
|
|
if (DEBUG) console.debug(`[IDLE] ⛔ cancelled timer state=${currentState}`);
|
|
}
|
|
|
|
/** @type {S2sWsRealtimeClient | null} */
|
|
let client = null;
|
|
/** @type {MediaStream | null} */
|
|
let micStream = null;
|
|
let micMuted = false;
|
|
|
|
/** @param {AppState} next */
|
|
function setState(next) {
|
|
currentState = next;
|
|
const view = STATE_VIEWS[next];
|
|
circleBtn.disabled = view.disabled;
|
|
circleBtn.className = `circle ${STATE_CLASS[next]}`;
|
|
if (next !== "error") setCaption(view.caption);
|
|
|
|
const live = LIVE_STATES.has(next);
|
|
orbWrap.classList.toggle("live", live);
|
|
micBtn.setAttribute("aria-hidden", live ? "false" : "true");
|
|
stopBtn.setAttribute("aria-hidden", live ? "false" : "true");
|
|
micBtn.tabIndex = live ? 0 : -1;
|
|
stopBtn.tabIndex = live ? 0 : -1;
|
|
|
|
// Queue affordances: "Leave queue" whenever we're in line; "Join now" only once
|
|
// it's our turn (a slot is held for us). Both live under #queue-actions.
|
|
const yourTurn = next === "your-turn";
|
|
const inLine = next === "queued" || yourTurn;
|
|
queueActions.hidden = !inLine;
|
|
joinQueueBtn.hidden = !yourTurn;
|
|
joinQueueBtn.tabIndex = yourTurn ? 0 : -1;
|
|
leaveQueueBtn.hidden = !inLine;
|
|
leaveQueueBtn.tabIndex = inLine ? 0 : -1;
|
|
if (!yourTurn) stopJoinCountdown();
|
|
|
|
// Warm reassurance under the terse position, only while waiting in line.
|
|
if (next === "queued") {
|
|
circleSubcaption.textContent =
|
|
"Sorry, we overhugged! 🤗 Every slot is busy, so we saved you a spot. Hang tight, you're moving up.";
|
|
circleSubcaption.hidden = false;
|
|
} else {
|
|
circleSubcaption.hidden = true;
|
|
}
|
|
|
|
// Wake word: armed only when idle/error (no live mic contention with the
|
|
// realtime session), and disarmed the moment we move into any other state.
|
|
if (next === "idle" || next === "error") {
|
|
startWakeWordListener();
|
|
} else {
|
|
stopWakeWordListener();
|
|
}
|
|
|
|
// Idle auto-close: arm the countdown whenever we land in a live conversing
|
|
// state (user/assistant turn, processing), and cancel it whenever we leave
|
|
// those states (idle, queue, error, teardown). Moving *to* a live state is
|
|
// itself counted as activity so the 10s window starts fresh.
|
|
if (IDLE_COUNT_STATES.has(next)) {
|
|
_kickIdleTimer(`state→${next}`);
|
|
} else {
|
|
_cancelIdleTimer();
|
|
}
|
|
|
|
updateRestartAvailability();
|
|
}
|
|
|
|
function updateRestartAvailability() {
|
|
// Restart works from any settled state — it tears down a live call (if any)
|
|
// and reconnects with the current settings. Only block while mid-connect or
|
|
// while waiting in the queue (restarting from there would just re-queue).
|
|
restartBtn.disabled =
|
|
currentState === "connecting" || currentState === "queued" || currentState === "your-turn";
|
|
restartHint.hidden = false;
|
|
restartHint.textContent = LIVE_STATES.has(currentState)
|
|
? "Reconnects now with the settings above."
|
|
: "Starts a conversation with the settings above.";
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @param {"" | "error" | "muted"} [kind]
|
|
*/
|
|
function setCaption(text, kind = "") {
|
|
const trimmed = text.trim();
|
|
circleCaption.textContent = trimmed;
|
|
circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`;
|
|
}
|
|
|
|
function openSettings() {
|
|
syncConnectionUi();
|
|
inputVoice.value = settings.voice;
|
|
inputInstructions.value = pinnedPrompt || settings.instructions;
|
|
inputInstructions.readOnly = !!pinnedPrompt;
|
|
primaryPromptMeta.hidden = !pinnedPrompt;
|
|
primaryPromptMeta.textContent = pinnedPrompt
|
|
? pinnedPromptSource
|
|
? `Primary prompt from ${pinnedPromptSource}`
|
|
: "Primary prompt pinned by this deployment."
|
|
: "";
|
|
syncGateUi();
|
|
updateRestartAvailability();
|
|
settingsModal.showModal();
|
|
}
|
|
|
|
/** dB position (clamped to the slider axis) as a 0..1 fraction of the track.
|
|
* @param {number} db */
|
|
function dbToFraction(db) {
|
|
const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db));
|
|
return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB);
|
|
}
|
|
|
|
/** @param {number} f @returns {number} dB at a 0..1 position on the gate axis. */
|
|
function fractionToDb(f) {
|
|
const clamped = Math.min(1, Math.max(0, f));
|
|
return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB));
|
|
}
|
|
|
|
// ── Radial gate arc (around the mic button, live during a call) ─────────────
|
|
// A 270° arc with the gap facing the orb (right). Fraction 0 (=Off) sits at the
|
|
// bottom-ish start; 1 (=max) at the top-ish end. The level fill and the
|
|
// threshold handle ride this same axis, mirroring the Settings widget.
|
|
const ARC_R = 40;
|
|
// A ~200° arc centred on the left (180°) so the wide gap faces the orb (right).
|
|
const ARC_SPAN_DEG = 200;
|
|
const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // lower-left start; Off end
|
|
|
|
/** Point at fraction f (0..1) and radius r, in the 0..100 viewBox.
|
|
* @param {number} f @param {number} [r] */
|
|
function arcPoint(f, r = ARC_R) {
|
|
const deg = ARC_START_DEG + f * ARC_SPAN_DEG;
|
|
const rad = (deg * Math.PI) / 180;
|
|
return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) };
|
|
}
|
|
|
|
/** SVG path `d` for the full 0..1 arc (clockwise). */
|
|
function fullArcD() {
|
|
const a = arcPoint(0);
|
|
const b = arcPoint(1);
|
|
const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0;
|
|
return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`;
|
|
}
|
|
|
|
/** One-time geometry: track, fill (dash-revealed) and the transparent hit band. */
|
|
function initGateArc() {
|
|
const d = fullArcD();
|
|
mgaTrack.setAttribute("d", d);
|
|
mgaFill.setAttribute("d", d);
|
|
mgaHit.setAttribute("d", d);
|
|
// pathLength 100 lets us reveal the fill by fraction via dashoffset.
|
|
mgaFill.setAttribute("pathLength", "100");
|
|
mgaFill.style.strokeDasharray = "100 100";
|
|
mgaFill.style.strokeDashoffset = "100"; // empty until levels arrive
|
|
renderGateHandle();
|
|
}
|
|
|
|
/** Place the threshold bead on the arc at the stored threshold; flag off state. */
|
|
function renderGateHandle() {
|
|
const off = settings.noiseGate <= GATE_OFF_DB;
|
|
const p = arcPoint(dbToFraction(settings.noiseGate));
|
|
mgaHandle.setAttribute("cx", String(p.x));
|
|
mgaHandle.setAttribute("cy", String(p.y));
|
|
micGate.classList.toggle("gate-off", off);
|
|
}
|
|
|
|
/** Paint a 0..1 live level onto the arc fill (and the Settings meter if open).
|
|
* Brightens the tick when the level crosses the threshold — i.e. the gate is
|
|
* actually open — but only when gating is enabled.
|
|
* @param {number} rms */
|
|
function paintInputLevel(rms) {
|
|
const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB;
|
|
const f = dbToFraction(db);
|
|
mgaFill.style.strokeDashoffset = String(100 * (1 - f));
|
|
if (settingsModal.open) gateMeterFill.style.width = `${f * 100}%`;
|
|
const enabled = settings.noiseGate > GATE_OFF_DB;
|
|
micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(settings.noiseGate));
|
|
}
|
|
|
|
/** The single place that commits a new gate threshold: updates both controls,
|
|
* persists, and applies live to the running session.
|
|
* @param {number} db */
|
|
function setGateThreshold(db) {
|
|
settings.noiseGate = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db)));
|
|
const off = settings.noiseGate <= GATE_OFF_DB;
|
|
inputNoiseGate.value = String(settings.noiseGate);
|
|
gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`;
|
|
renderGateHandle();
|
|
localStorage.setItem(STORAGE_KEYS.noiseGate, String(settings.noiseGate));
|
|
if (client && LIVE_STATES.has(currentState)) {
|
|
client.setNoiseGate(gateParams(settings.noiseGate));
|
|
}
|
|
}
|
|
|
|
/** Reflect the stored gate threshold into the slider, label and arc handle. */
|
|
function syncGateUi() {
|
|
inputNoiseGate.value = String(settings.noiseGate);
|
|
const off = settings.noiseGate <= GATE_OFF_DB;
|
|
gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`;
|
|
renderGateHandle();
|
|
}
|
|
|
|
// Drag along the arc band to set the threshold (a tap on the glyph still mutes).
|
|
let gateDragging = false;
|
|
/** @param {PointerEvent} e */
|
|
function gatePointerToDb(e) {
|
|
const rect = mgaArc.getBoundingClientRect();
|
|
const cx = rect.left + rect.width / 2;
|
|
const cy = rect.top + rect.height / 2;
|
|
let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI;
|
|
if (deg < 0) deg += 360;
|
|
// Map the on-arc angle to a fraction; angles in the right-side gap fall
|
|
// outside [0,1] and fractionToDb clamps them to the nearest end (just-below
|
|
// start -> Off, just-past end -> max).
|
|
const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG;
|
|
return fractionToDb(f);
|
|
}
|
|
mgaHit.addEventListener("pointerdown", (e) => {
|
|
gateDragging = true;
|
|
mgaHit.setPointerCapture(e.pointerId);
|
|
setGateThreshold(gatePointerToDb(e));
|
|
});
|
|
mgaHit.addEventListener("pointermove", (e) => {
|
|
if (gateDragging) setGateThreshold(gatePointerToDb(e));
|
|
});
|
|
const endGateDrag = (/** @type {PointerEvent} */ e) => {
|
|
if (!gateDragging) return;
|
|
gateDragging = false;
|
|
try { mgaHit.releasePointerCapture(e.pointerId); } catch {}
|
|
};
|
|
mgaHit.addEventListener("pointerup", endGateDrag);
|
|
mgaHit.addEventListener("pointercancel", endGateDrag);
|
|
|
|
settingsBtn.addEventListener("click", openSettings);
|
|
|
|
// About panel: native <dialog>, Esc closes for free; also close on the X and
|
|
// on a click in the backdrop (a click whose target is the dialog itself).
|
|
aboutBtn?.addEventListener("click", () => aboutModal.showModal());
|
|
// Mobile twin of the (i), living in the right-hand control cluster.
|
|
$("#about-btn-m").addEventListener("click", () => aboutModal.showModal());
|
|
aboutClose.addEventListener("click", () => aboutModal.close());
|
|
aboutModal.addEventListener("click", (e) => {
|
|
if (e.target === aboutModal) aboutModal.close();
|
|
});
|
|
|
|
// ── Tools panel ───────────────────────────────────────────────────────────
|
|
|
|
/** Reflect the current tool state into the panel controls. */
|
|
function syncToolsUi() {
|
|
const openToolNames = new Set(
|
|
Array.from(agentToolsList.querySelectorAll("details[open][data-mcp-tool-item]"))
|
|
.map((item) => item.getAttribute("data-mcp-tool-item") || "")
|
|
.filter(Boolean),
|
|
);
|
|
const avail = searchAvailable();
|
|
toolWebSwitch.checked = toolsEnabled.web_search && avail;
|
|
toolWebSwitch.disabled = !avail;
|
|
toolWebRow.classList.toggle("disabled", !avail);
|
|
toolCamSwitch.closest("div")?.setAttribute("hidden", "");
|
|
toolCamHint.setAttribute("hidden", "");
|
|
camPip.setAttribute("hidden", "");
|
|
|
|
if (serverSearchKey) {
|
|
// Key lives server-side: show it as configured, never expose it.
|
|
searchKeyInput.value = "";
|
|
searchKeyInput.placeholder = "•••••••• · provided by the server";
|
|
searchKeyInput.disabled = true;
|
|
toolWebHint.textContent = "Ready. The search key is held server-side and never sent to your browser.";
|
|
} else {
|
|
searchKeyInput.disabled = false;
|
|
searchKeyInput.value = userSearchKey;
|
|
searchKeyInput.placeholder = "Paste a Serper key to enable web search";
|
|
toolWebHint.textContent = userSearchKey
|
|
? "Using your key — stored in this browser only."
|
|
: "No server key configured. Add your own Serper key to enable web search.";
|
|
}
|
|
|
|
const count = agentToolDefs.length;
|
|
if (!count) {
|
|
agentToolsHint.textContent = "No MCP tools loaded from the remote agent profile.";
|
|
} else if (searchMcpEnabled && lastUserQuery && rankedAgentTools.length) {
|
|
agentToolsHint.textContent = `${count} MCP tools loaded. Top matches for "${truncateError(lastUserQuery)}".`;
|
|
} else if (searchMcpEnabled && lastUserQuery) {
|
|
agentToolsHint.textContent = `${count} MCP tools loaded. No description match found for "${truncateError(lastUserQuery)}".`;
|
|
} else {
|
|
agentToolsHint.textContent = `${count} MCP tool${count === 1 ? "" : "s"} loaded from the remote agent profile.`;
|
|
}
|
|
agentToolsList.innerHTML = renderAgentToolsHtml();
|
|
for (const toolName of openToolNames) {
|
|
const item = agentToolsList.querySelector(`details[data-mcp-tool-item="${CSS.escape(toolName)}"]`);
|
|
if (item instanceof HTMLDetailsElement) item.open = true;
|
|
}
|
|
}
|
|
|
|
/** @returns {string} */
|
|
function renderAgentToolsHtml() {
|
|
if (!agentToolDefs.length) return '<div class="agent-tool-empty">No MCP tools available.</div>';
|
|
const suggested = topSuggestedAgentTools(3);
|
|
const suggestedNames = new Set(suggested.map((entry) => entry.tool.name));
|
|
let html = "";
|
|
if (searchMcpEnabled && lastUserQuery) {
|
|
html += suggested.length
|
|
? `
|
|
<div class="agent-tool-match-banner">
|
|
<div class="agent-tool-section-title">Suggested For Current Request</div>
|
|
<div class="agent-tool-match-text">${escHtml(lastUserQuery)}</div>
|
|
</div>
|
|
`
|
|
: `
|
|
<div class="agent-tool-empty">
|
|
No MCP tool description matches the current request.
|
|
</div>
|
|
`;
|
|
}
|
|
const orderedTools = suggested
|
|
.map((entry) => entry.tool)
|
|
.concat(agentToolDefs.filter((tool) => !suggestedNames.has(tool.name)));
|
|
html += orderedTools.map((tool) => {
|
|
const paramRows = describeToolParams(tool.parameters);
|
|
const toolState = getMcpToolCallState(tool.name);
|
|
const suggestedIndex = suggested.findIndex((entry) => entry.tool.name === tool.name);
|
|
const badge = suggestedIndex >= 0
|
|
? `<span class="agent-tool-badge suggested">Suggested #${suggestedIndex + 1}</span>`
|
|
: '<span class="agent-tool-badge">MCP</span>';
|
|
return `
|
|
<details class="agent-tool-item" data-mcp-tool-item="${escHtml(tool.name)}">
|
|
<summary class="agent-tool-summary">
|
|
<div class="agent-tool-head">
|
|
<span class="tool-name">${escHtml(tool.name)}</span>
|
|
${badge}
|
|
</div>
|
|
<span class="tool-desc">${escHtml(tool.description || "No description provided.")}</span>
|
|
</summary>
|
|
<div class="agent-tool-body">
|
|
<div class="agent-tool-section-title">Parameters</div>
|
|
${paramRows || '<div class="agent-tool-empty small">No parameters.</div>'}
|
|
<div class="agent-tool-section-title">Manual Call</div>
|
|
<label class="agent-tool-call-label">
|
|
<span class="agent-tool-call-copy">JSON arguments</span>
|
|
<textarea
|
|
class="agent-tool-call-args"
|
|
data-mcp-tool-args="${escHtml(tool.name)}"
|
|
spellcheck="false"
|
|
rows="4"
|
|
>${escHtml(toolState.argsText)}</textarea>
|
|
</label>
|
|
<div class="agent-tool-call-actions">
|
|
<button
|
|
type="button"
|
|
class="btn primary agent-tool-call-btn"
|
|
data-mcp-call-btn="${escHtml(tool.name)}"
|
|
${toolState.status === "running" ? "disabled" : ""}
|
|
>${toolState.status === "running" ? "Calling..." : "Call method"}</button>
|
|
<button
|
|
type="button"
|
|
class="btn ghost agent-tool-call-btn"
|
|
data-mcp-send-btn="${escHtml(tool.name)}"
|
|
${toolState.status === "running" ? "disabled" : ""}
|
|
>Call and send</button>
|
|
</div>
|
|
${renderMcpToolCallResultHtml(tool.name)}
|
|
</div>
|
|
</details>
|
|
`;
|
|
}).join("");
|
|
return html;
|
|
}
|
|
|
|
/** @param {number} limit */
|
|
function topSuggestedAgentTools(limit) {
|
|
return rankedAgentTools.slice(0, limit);
|
|
}
|
|
|
|
/**
|
|
* @param {string} query
|
|
* @param {{ partial?: boolean, itemId?: string }} [options]
|
|
*/
|
|
function updateMcpToolSuggestions(query, options = {}) {
|
|
lastUserQuery = query.trim();
|
|
console.debug(
|
|
`[MCP RANK] updateMcpToolSuggestions ENTRY partial=${!!options.partial} itemId=${options.itemId || "-"} ` +
|
|
`query=${JSON.stringify(lastUserQuery)} searchMcpEnabled=${searchMcpEnabled}`,
|
|
);
|
|
rankedAgentTools = rankAgentTools(lastUserQuery);
|
|
// #region debug-point A:ranked-mcp-suggestions
|
|
reportDebug("A", "main.js:updateMcpToolSuggestions", "ranked MCP tools for user query", {
|
|
partial: !!options.partial,
|
|
query: lastUserQuery,
|
|
ranked: rankedAgentTools.map((entry) => ({ name: entry.tool.name, score: entry.score })),
|
|
searchMcpEnabled,
|
|
});
|
|
// #endregion
|
|
const rankedNames = rankedAgentTools.map((entry) => `${entry.tool.name}:${entry.score}`).join(", ");
|
|
console.debug(
|
|
`[MCP RANK] updateMcpToolSuggestions RESULTS partial=${!!options.partial} ` +
|
|
`matchedTools=${rankedAgentTools.length} ranked=${rankedNames || "none"}`,
|
|
);
|
|
if (!options.partial && rankedAgentTools.length) {
|
|
const top3 = rankedAgentTools
|
|
.slice(0, 3)
|
|
.map((entry, index) => `${index + 1}:${entry.tool.name}:${entry.score}`)
|
|
.join(" | ");
|
|
console.debug(`[MCP RANK] ✦ FINAL top3 query=${JSON.stringify(lastUserQuery)} top3=${top3 || "none"}`);
|
|
}
|
|
syncToolsUi();
|
|
if (client && LIVE_STATES.has(currentState)) {
|
|
const instructions = effectiveInstructions();
|
|
if (options.partial && rankedAgentTools.length) {
|
|
pushLiveToolPolicy();
|
|
pushLiveInstructions(instructions);
|
|
} else if (!options.partial) {
|
|
pushLiveToolPolicy();
|
|
pushLiveInstructions(instructions);
|
|
maybeAutoRunSuggestedMcpTool(lastUserQuery, options.itemId || "");
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {string} query
|
|
* @param {string} itemId
|
|
*/
|
|
function maybeAutoRunSuggestedMcpTool(query, itemId) {
|
|
if (!client || !LIVE_STATES.has(currentState) || !searchMcpEnabled || autoMcpInFlight) return;
|
|
const normalized = normalizeMcpSearchText(query);
|
|
const top = rankedAgentTools[0];
|
|
if (!normalized || !top) return;
|
|
const second = rankedAgentTools[1];
|
|
const topNameText = normalizeMcpSearchText(top.tool.name);
|
|
const topDescText = normalizeMcpSearchText(top.tool.description || "");
|
|
const topRequiredParams = requiredToolParams(top.tool.parameters);
|
|
const relaxedDailyReportMatch = top.score >= AUTO_MCP_RELAXED_REPORT_SCORE &&
|
|
!topRequiredParams.length &&
|
|
(
|
|
((topNameText.includes("day") || topNameText.includes("daily") || topNameText.includes("today")) &&
|
|
topNameText.includes("report")) ||
|
|
((topDescText.includes("day") || topDescText.includes("daily") || topDescText.includes("today")) &&
|
|
topDescText.includes("report"))
|
|
);
|
|
const strongByScore = top.score >= AUTO_MCP_MIN_SCORE;
|
|
const strongByLead = top.score >= AUTO_MCP_MIN_SCORE_WITH_LEAD &&
|
|
(!second || (top.score - second.score) >= AUTO_MCP_MIN_LEAD);
|
|
const strongMatch = top.score >= AUTO_MCP_MIN_SCORE ||
|
|
(top.score >= AUTO_MCP_MIN_SCORE_WITH_LEAD && (!second || (top.score - second.score) >= AUTO_MCP_MIN_LEAD)) ||
|
|
relaxedDailyReportMatch;
|
|
if (!strongMatch) {
|
|
console.debug(`[Vocero MCP AUTO] skip weak match query=${JSON.stringify(query)} top=${top.tool.name}:${top.score}`);
|
|
return;
|
|
}
|
|
const turnKey = `${itemId || normalized}::${top.tool.name}`;
|
|
if (turnKey === lastAutoMcpTurnKey) return;
|
|
lastAutoMcpTurnKey = turnKey;
|
|
clearPendingAutoMcpFallback();
|
|
const fastSafeMatch = !topRequiredParams.length && (top.score >= 40 || relaxedDailyReportMatch || strongByLead);
|
|
const autoDelayMs = fastSafeMatch ? AUTO_MCP_FAST_DELAY_MS : AUTO_MCP_FALLBACK_DELAY_MS;
|
|
reportDebug("A", "main.js:maybeAutoRunSuggestedMcpTool", "scheduled MCP fallback for matched user query", {
|
|
itemId,
|
|
query,
|
|
toolName: top.tool.name,
|
|
score: top.score,
|
|
nextScore: second?.score ?? null,
|
|
autoDelayMs,
|
|
});
|
|
console.debug(
|
|
`[Vocero MCP AUTO] scheduled query=${JSON.stringify(query)} itemId=${JSON.stringify(itemId)} ` +
|
|
`tool=${top.tool.name} score=${top.score} reason=${strongByScore ? "strict" : strongByLead ? "lead" : "relaxed-report"} ` +
|
|
`delayMs=${autoDelayMs}`
|
|
);
|
|
pendingAutoMcpTimer = window.setTimeout(() => {
|
|
pendingAutoMcpTimer = 0;
|
|
if (!client || !LIVE_STATES.has(currentState) || autoMcpInFlight) return;
|
|
autoMcpInFlight = true;
|
|
console.debug(
|
|
`[Vocero MCP AUTO] trigger query=${JSON.stringify(query)} itemId=${JSON.stringify(itemId)} ` +
|
|
`tool=${top.tool.name} score=${top.score}`
|
|
);
|
|
client.suppressAssistantAudio?.();
|
|
chat.onToolCall(top.tool.name);
|
|
void callAgentToolFromPanel(top.tool.name, true, query).finally(() => {
|
|
autoMcpInFlight = false;
|
|
});
|
|
}, autoDelayMs);
|
|
}
|
|
|
|
/**
|
|
* @param {string} query
|
|
* @returns {{ tool: import("./ws/s2s-ws-client.js").ToolDef, score: number }[]}
|
|
*/
|
|
function rankAgentTools(query) {
|
|
console.debug(`[MCP RANK] ════════════════════════════════════════`);
|
|
console.debug(`[MCP RANK] rankAgentTools START query=${JSON.stringify(query)} searchMcpEnabled=${searchMcpEnabled} agentToolDefsCount=${agentToolDefs.length}`);
|
|
const searchTexts = buildMcpSearchTexts(query);
|
|
const normalized = searchTexts[0] || "";
|
|
if (!searchMcpEnabled || !normalized || !agentToolDefs.length) {
|
|
const reasonParts = [];
|
|
if (!searchMcpEnabled) reasonParts.push("searchMcpEnabled=false — da /api/config: MCP Aquachile feature OFF oppure S2S_TOOL_EXECUTION_MCP_URL non configurato");
|
|
if (!normalized) reasonParts.push("query-normalized-empty");
|
|
if (!agentToolDefs.length) reasonParts.push("agentToolDefs=0 — catalogo tool MCP vuoto: profilo agente non ha tools in agentProfile via Aquachile /api/agents/profile");
|
|
console.debug(
|
|
`[MCP RANK] → EARLY EXIT reason="${reasonParts.join("; ")}" ` +
|
|
`searchMcpEnabled=${searchMcpEnabled} normalized=${!!normalized} agentToolDefs=${agentToolDefs.length}`,
|
|
);
|
|
console.debug(`[MCP RANK] ════════════════════════════════════════ (empty result)`);
|
|
return [];
|
|
}
|
|
const tokens = [...new Set(searchTexts.flatMap((text) => text.split(" ").filter((token) => token.length >= 3)))];
|
|
if (!tokens.length) {
|
|
console.debug(
|
|
`[MCP RANK] → EARLY EXIT reason="nessun token >= 3 chars dopo split" searchTexts=${JSON.stringify(searchTexts)}`,
|
|
);
|
|
console.debug(`[MCP RANK] ════════════════════════════════════════ (empty result)`);
|
|
return [];
|
|
}
|
|
const dailyReportIntent = searchTexts.some((text) =>
|
|
/\b(day|daily|today)\b/.test(text) && /\breport\b/.test(text),
|
|
);
|
|
console.debug(
|
|
`[MCP RANK] searchTexts=${JSON.stringify(searchTexts)} ` +
|
|
`tokensCount=${tokens.length} tokens=${JSON.stringify(tokens)} ` +
|
|
`dailyReportIntent=${dailyReportIntent}`,
|
|
);
|
|
const scored = agentToolDefs
|
|
.map((tool) => {
|
|
const nameText = normalizeMcpSearchText(tool.name);
|
|
const descText = normalizeMcpSearchText(tool.description || "");
|
|
const paramText = normalizeMcpSearchText(extractToolParamSearchText(tool.parameters));
|
|
const requiredParams = requiredToolParams(tool.parameters);
|
|
const breakdown = {
|
|
nameHitFull: 0,
|
|
descHitFull: 0,
|
|
paramHitFull: 0,
|
|
nameHitTokens: 0,
|
|
descHitTokens: 0,
|
|
paramHitTokens: 0,
|
|
allFoundBonus: 0,
|
|
noRequiredParamsBonus: 0,
|
|
dailyReportMatchBoost: 0,
|
|
dailyReportPenalty: 0,
|
|
};
|
|
let score = 0;
|
|
for (const searchText of searchTexts) {
|
|
if (!searchText) continue;
|
|
if (descText.includes(searchText)) {
|
|
const add = searchText === normalized ? 20 : 16;
|
|
score += add;
|
|
breakdown.descHitFull += add;
|
|
}
|
|
if (nameText.includes(searchText)) {
|
|
const add = searchText === normalized ? 24 : 18;
|
|
score += add;
|
|
breakdown.nameHitFull += add;
|
|
}
|
|
if (paramText.includes(searchText)) {
|
|
const add = searchText === normalized ? 6 : 4;
|
|
score += add;
|
|
breakdown.paramHitFull += add;
|
|
}
|
|
}
|
|
const tokenMatchDetails = [];
|
|
for (const token of tokens) {
|
|
const onName = nameText.includes(token);
|
|
const onDesc = descText.includes(token);
|
|
const onParam = paramText.includes(token);
|
|
if (!onName && !onDesc && !onParam) continue;
|
|
const entry = { token };
|
|
if (onName) { score += 8; breakdown.nameHitTokens += 8; entry.name = true; }
|
|
if (onDesc) { score += 5; breakdown.descHitTokens += 5; entry.desc = true; }
|
|
if (onParam) { score += 2; breakdown.paramHitTokens += 2; entry.param = true; }
|
|
tokenMatchDetails.push(entry);
|
|
}
|
|
const allFound = searchTexts.some((searchText) =>
|
|
searchText.split(" ").filter((token) => token.length >= 3).every((token) =>
|
|
nameText.includes(token) || descText.includes(token) || paramText.includes(token),
|
|
),
|
|
);
|
|
if (allFound) { score += 10; breakdown.allFoundBonus = 10; }
|
|
if (!requiredParams.length) { score += 3; breakdown.noRequiredParamsBonus = 3; }
|
|
if (dailyReportIntent) {
|
|
if ((nameText.includes("day report") || nameText.includes("daily report") || nameText.includes("today")) &&
|
|
(descText.includes("day report") || descText.includes("daily report") || descText.includes("today"))) {
|
|
score += 24;
|
|
breakdown.dailyReportMatchBoost = 24;
|
|
} else if (requiredParams.length) {
|
|
score -= 10;
|
|
breakdown.dailyReportPenalty = -10;
|
|
}
|
|
}
|
|
console.debug(
|
|
`[MCP RANK] ▸ tool=${JSON.stringify(tool.name)} score=${score} ` +
|
|
`requiredParams=${requiredParams.length} allFound=${allFound}\n` +
|
|
`[MCP RANK] ↳ fullMatches: name=${breakdown.nameHitFull} desc=${breakdown.descHitFull} params=${breakdown.paramHitFull}\n` +
|
|
`[MCP RANK] ↳ tokenMatches(${tokenMatchDetails.length}): ${tokenMatchDetails.length ? JSON.stringify(tokenMatchDetails) : "-"}\n` +
|
|
`[MCP RANK] ↳ bonuses: allFound=${breakdown.allFoundBonus} noRequiredParams=${breakdown.noRequiredParamsBonus} ` +
|
|
`dailyBoost=${breakdown.dailyReportMatchBoost} dailyPenalty=${breakdown.dailyReportPenalty}\n` +
|
|
`[MCP RANK] ↳ nameTokens=${JSON.stringify(nameText.slice(0, 100))}\n` +
|
|
`[MCP RANK] ↳ descTokens=${JSON.stringify(descText.slice(0, 200))}\n` +
|
|
`[MCP RANK] ↳ paramTokens=${JSON.stringify(paramText.slice(0, 150))}`,
|
|
);
|
|
return { tool, score };
|
|
});
|
|
const filtered = scored.filter((entry) => entry.score > 0);
|
|
const sorted = filtered.sort((a, b) => b.score - a.score || a.tool.name.localeCompare(b.tool.name));
|
|
console.debug(
|
|
`[MCP RANK] DONE scoredCount=${scored.length} matchedCount=${filtered.length} ` +
|
|
`top5=${JSON.stringify(sorted.slice(0, 5).map((e) => ({ t: e.tool.name, s: e.score })))}`,
|
|
);
|
|
console.debug(`[MCP RANK] ════════════════════════════════════════`);
|
|
return sorted;
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @returns {string}
|
|
*/
|
|
function translateMcpSearchTextToEnglish(text) {
|
|
const before = normalizeMcpSearchText(text);
|
|
let translated = before;
|
|
const appliedPhrases = [];
|
|
for (const [pattern, replacement] of MCP_SEARCH_TRANSLATION_PHRASES) {
|
|
const previous = translated;
|
|
translated = translated.replace(pattern, replacement);
|
|
if (translated !== previous) {
|
|
appliedPhrases.push({ pattern: String(pattern), to: replacement });
|
|
}
|
|
}
|
|
const beforeTokens = translated;
|
|
const appliedTokens = [];
|
|
translated = translated
|
|
.split(" ")
|
|
.map((token) => {
|
|
const mapped = MCP_SEARCH_TRANSLATION_TOKENS[token];
|
|
if (mapped && mapped !== token) {
|
|
appliedTokens.push({ from: token, to: mapped });
|
|
return mapped;
|
|
}
|
|
return token;
|
|
})
|
|
.join(" ");
|
|
translated = normalizeMcpSearchText(translated);
|
|
console.debug(
|
|
`[MCP RANK] translate input=${JSON.stringify(before)} phrasesApplied=${appliedPhrases.length} ` +
|
|
`tokensApplied=${appliedTokens.length} output=${JSON.stringify(translated)}` +
|
|
(appliedPhrases.length ? ` phrases=${JSON.stringify(appliedPhrases)}` : "") +
|
|
(appliedTokens.length ? ` tokens=${JSON.stringify(appliedTokens)}` : ""),
|
|
);
|
|
return translated;
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @returns {string[]}
|
|
*/
|
|
function buildMcpSearchTexts(text) {
|
|
const normalized = normalizeMcpSearchText(text);
|
|
const translated = translateMcpSearchTextToEnglish(text);
|
|
const result = [...new Set([normalized, translated].filter(Boolean))];
|
|
console.debug(
|
|
`[MCP RANK] buildSearchTexts raw=${JSON.stringify(text)} ` +
|
|
`normalized=${JSON.stringify(normalized)} translated=${JSON.stringify(translated)} ` +
|
|
`count=${result.length} final=${JSON.stringify(result)}`,
|
|
);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @returns {string}
|
|
*/
|
|
function normalizeMcpSearchText(text) {
|
|
return text
|
|
.toLowerCase()
|
|
.normalize("NFD")
|
|
.replace(/[\u0300-\u036f]/g, "")
|
|
.replace(/[^a-z0-9]+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
/**
|
|
* @param {any} schema
|
|
* @returns {string}
|
|
*/
|
|
function extractToolParamSearchText(schema) {
|
|
const props = schema && typeof schema === "object" && schema.properties && typeof schema.properties === "object"
|
|
? schema.properties
|
|
: null;
|
|
if (!props) return "";
|
|
/** @type {string[]} */
|
|
const chunks = [];
|
|
for (const [name, raw] of Object.entries(props)) {
|
|
chunks.push(name);
|
|
if (raw && typeof raw === "object" && typeof raw.description === "string") {
|
|
chunks.push(raw.description);
|
|
}
|
|
}
|
|
return chunks.join(" ");
|
|
}
|
|
|
|
/**
|
|
* @param {any} schema
|
|
* @returns {string}
|
|
*/
|
|
function describeToolParams(schema) {
|
|
const props = schema && typeof schema === "object" && schema.properties && typeof schema.properties === "object"
|
|
? schema.properties
|
|
: null;
|
|
if (!props) return "";
|
|
/** @type {string[]} */
|
|
const rows = [];
|
|
const required = Array.isArray(schema.required) ? new Set(schema.required) : new Set();
|
|
for (const [name, raw] of Object.entries(props)) {
|
|
const spec = raw && typeof raw === "object" ? raw : {};
|
|
const type = paramTypeLabel(spec);
|
|
const def = "default" in spec ? ` · default: ${formatParamValue(spec.default)}` : "";
|
|
const req = required.has(name) ? " required" : "";
|
|
rows.push(
|
|
`<div class="agent-param-row">` +
|
|
`<span class="agent-param-name">${escHtml(name)}</span>` +
|
|
`<span class="agent-param-type">${escHtml(type + req + def)}</span>` +
|
|
`</div>`,
|
|
);
|
|
}
|
|
return rows.join("");
|
|
}
|
|
|
|
/**
|
|
* @param {any} spec
|
|
* @returns {string}
|
|
*/
|
|
function paramTypeLabel(spec) {
|
|
if (Array.isArray(spec.anyOf)) {
|
|
/** @type {string[]} */
|
|
const labels = [];
|
|
for (const item of spec.anyOf) {
|
|
labels.push((item && typeof item === "object" && typeof item.type === "string") ? item.type : "unknown");
|
|
}
|
|
return labels.join(" | ");
|
|
}
|
|
if (typeof spec.type === "string") return spec.type;
|
|
return "unknown";
|
|
}
|
|
|
|
/**
|
|
* @param {unknown} value
|
|
* @returns {string}
|
|
*/
|
|
function formatParamValue(value) {
|
|
if (value === null) return "null";
|
|
if (typeof value === "string") return value;
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
/**
|
|
* @param {any} schema
|
|
* @returns {Record<string, any>}
|
|
*/
|
|
function schemaProperties(schema) {
|
|
return schema && typeof schema === "object" && schema.properties && typeof schema.properties === "object"
|
|
? schema.properties
|
|
: {};
|
|
}
|
|
|
|
/**
|
|
* @param {any} schema
|
|
* @returns {string[]}
|
|
*/
|
|
function requiredToolParams(schema) {
|
|
if (!Array.isArray(schema?.required)) return [];
|
|
/** @type {string[]} */
|
|
const required = [];
|
|
for (const name of schema.required) {
|
|
if (typeof name === "string") required.push(name);
|
|
}
|
|
return required;
|
|
}
|
|
|
|
/**
|
|
* @param {any} spec
|
|
* @param {string} type
|
|
* @returns {boolean}
|
|
*/
|
|
function schemaSupportsType(spec, type) {
|
|
if (!spec || typeof spec !== "object") return false;
|
|
if (spec.type === type) return true;
|
|
if (Array.isArray(spec.anyOf)) {
|
|
for (const item of spec.anyOf) {
|
|
if (item && typeof item === "object" && item.type === type) return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* @param {any} spec
|
|
* @returns {boolean}
|
|
*/
|
|
function isStringArraySchema(spec) {
|
|
if (!spec || typeof spec !== "object") return false;
|
|
if (spec.type === "array") {
|
|
return !spec.items || spec.items.type === "string";
|
|
}
|
|
if (Array.isArray(spec.anyOf)) {
|
|
for (const item of spec.anyOf) {
|
|
if (item && typeof item === "object" && item.type === "array" && (!item.items || item.items.type === "string")) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* @param {string} name
|
|
* @returns {string[]}
|
|
*/
|
|
function paramAliases(name) {
|
|
const aliases = new Set([name, name.replace(/_/g, " "), name.replace(/_/g, "-")]);
|
|
if (/phone|mobile|whatsapp/i.test(name)) {
|
|
aliases.add("phone");
|
|
aliases.add("phone number");
|
|
aliases.add("mobile");
|
|
aliases.add("mobile number");
|
|
aliases.add("whatsapp");
|
|
aliases.add("whatsapp number");
|
|
aliases.add("number");
|
|
}
|
|
return [...aliases].filter(Boolean);
|
|
}
|
|
|
|
const DIGIT_WORDS = {
|
|
zero: "0",
|
|
oh: "0",
|
|
one: "1",
|
|
two: "2",
|
|
three: "3",
|
|
four: "4",
|
|
five: "5",
|
|
six: "6",
|
|
seven: "7",
|
|
eight: "8",
|
|
nine: "9",
|
|
};
|
|
|
|
/**
|
|
* @param {string} name
|
|
* @returns {boolean}
|
|
*/
|
|
function isPhoneLikeParam(name) {
|
|
return /phone|mobile|whatsapp/i.test(name);
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @returns {string}
|
|
*/
|
|
function normalizePhoneText(text) {
|
|
const trimmed = text.trim();
|
|
if (!trimmed) return "";
|
|
const hasLeadingPlus = /^\s*\+/.test(trimmed);
|
|
const compact = trimmed
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9+\s-]/g, " ")
|
|
.split(/\s+/)
|
|
.filter(Boolean);
|
|
/** @type {string[]} */
|
|
const digits = [];
|
|
for (const token of compact) {
|
|
if (token === "+" && !digits.length && hasLeadingPlus) continue;
|
|
if (/^\d+$/.test(token)) {
|
|
digits.push(token);
|
|
continue;
|
|
}
|
|
if (Object.prototype.hasOwnProperty.call(DIGIT_WORDS, token)) {
|
|
digits.push(DIGIT_WORDS[/** @type {keyof typeof DIGIT_WORDS} */ (token)]);
|
|
}
|
|
}
|
|
if (!digits.length) return "";
|
|
return `${hasLeadingPlus ? "+" : ""}${digits.join("")}`;
|
|
}
|
|
|
|
/**
|
|
* @param {string} query
|
|
* @returns {string}
|
|
*/
|
|
function extractPhoneValue(query) {
|
|
const patterns = [
|
|
/\b(?:my\s+)?(?:phone|mobile|whatsapp|number)\s*(?:number)?\s*(?:is|=|:)?\s*([^.!,;\n]+)/i,
|
|
/\bcall me at\s*([^.!,;\n]+)/i,
|
|
/\breach me at\s*([^.!,;\n]+)/i,
|
|
];
|
|
for (const pattern of patterns) {
|
|
const match = query.match(pattern);
|
|
const normalized = normalizePhoneText(match?.[1] || "");
|
|
if (normalized) return normalized;
|
|
}
|
|
return normalizePhoneText(query);
|
|
}
|
|
|
|
/**
|
|
* @param {string} query
|
|
* @param {string} name
|
|
* @returns {string}
|
|
*/
|
|
function inferStringParamValue(query, name) {
|
|
if (isPhoneLikeParam(name)) return extractPhoneValue(query);
|
|
return "";
|
|
}
|
|
|
|
/**
|
|
* @param {string} query
|
|
* @param {string[]} aliases
|
|
* @returns {string}
|
|
*/
|
|
function matchParamValueText(query, aliases) {
|
|
for (const alias of aliases) {
|
|
const escaped = alias.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
const patterns = [
|
|
new RegExp(`\\b${escaped}\\b\\s*(?:=|:|is|to)?\\s*"([^"]+)"`, "i"),
|
|
new RegExp(`\\b${escaped}\\b\\s*(?:=|:|is|to)?\\s*'([^']+)'`, "i"),
|
|
new RegExp(`\\b${escaped}\\b\\s*(?:=|:|is|to)?\\s*([^,;\\n]+)`, "i"),
|
|
];
|
|
for (const pattern of patterns) {
|
|
const match = query.match(pattern);
|
|
if (match?.[1]) return match[1].trim();
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
|
|
/**
|
|
* @param {string} query
|
|
* @param {any} schema
|
|
* @returns {Record<string, unknown>}
|
|
*/
|
|
function inferToolArgsFromQuery(query, schema) {
|
|
const props = schemaProperties(schema);
|
|
/** @type {Record<string, unknown>} */
|
|
const inferred = {};
|
|
for (const [name, raw] of Object.entries(props)) {
|
|
const spec = raw && typeof raw === "object" ? raw : {};
|
|
const valueText = matchParamValueText(query, paramAliases(name));
|
|
const fallbackText = !valueText && schemaSupportsType(spec, "string")
|
|
? inferStringParamValue(query, name)
|
|
: "";
|
|
const resolvedText = valueText || fallbackText;
|
|
if (!resolvedText) continue;
|
|
if (schemaSupportsType(spec, "number")) {
|
|
const num = Number(resolvedText);
|
|
if (!Number.isNaN(num)) inferred[name] = num;
|
|
continue;
|
|
}
|
|
if (schemaSupportsType(spec, "boolean")) {
|
|
const normalized = resolvedText.toLowerCase();
|
|
if (["true", "yes", "1", "on"].includes(normalized)) inferred[name] = true;
|
|
else if (["false", "no", "0", "off"].includes(normalized)) inferred[name] = false;
|
|
continue;
|
|
}
|
|
if (isStringArraySchema(spec)) {
|
|
inferred[name] = resolvedText
|
|
.split(",")
|
|
.map((item) => item.trim())
|
|
.filter(Boolean);
|
|
continue;
|
|
}
|
|
if (schemaSupportsType(spec, "string")) {
|
|
inferred[name] = resolvedText;
|
|
}
|
|
}
|
|
return inferred;
|
|
}
|
|
|
|
/**
|
|
* @param {Record<string, unknown>} args
|
|
* @param {any} schema
|
|
* @returns {Record<string, unknown>}
|
|
*/
|
|
function applyToolArgDefaults(args, schema) {
|
|
const props = schemaProperties(schema);
|
|
/** @type {Record<string, unknown>} */
|
|
const merged = { ...args };
|
|
for (const [name, raw] of Object.entries(props)) {
|
|
const spec = raw && typeof raw === "object" ? raw : {};
|
|
if (!(name in merged) && Object.prototype.hasOwnProperty.call(spec, "default")) {
|
|
merged[name] = spec.default;
|
|
}
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
/**
|
|
* @param {Record<string, unknown>} args
|
|
* @param {any} schema
|
|
* @returns {string[]}
|
|
*/
|
|
function missingRequiredToolArgs(args, schema) {
|
|
return requiredToolParams(schema).filter((name) => {
|
|
const value = args[name];
|
|
return value === undefined || value === null || value === "";
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param {string} toolName
|
|
* @returns {{ argsText: string, status: "idle" | "running" | "success" | "error", resultText: string, lastAction: "call" | "send" }}
|
|
*/
|
|
function getMcpToolCallState(toolName) {
|
|
if (!mcpToolCallState[toolName]) {
|
|
mcpToolCallState[toolName] = {
|
|
argsText: "{}",
|
|
status: "idle",
|
|
resultText: "",
|
|
lastAction: "call",
|
|
};
|
|
}
|
|
return mcpToolCallState[toolName];
|
|
}
|
|
|
|
/**
|
|
* @param {string} toolName
|
|
* @returns {import("./ws/s2s-ws-client.js").ToolDef | undefined}
|
|
*/
|
|
function findAgentToolDef(toolName) {
|
|
return agentToolDefs.find((tool) => tool.name === toolName);
|
|
}
|
|
|
|
/**
|
|
* @param {string} toolName
|
|
* @returns {string}
|
|
*/
|
|
function renderMcpToolCallResultHtml(toolName) {
|
|
const state = getMcpToolCallState(toolName);
|
|
if (state.status === "idle" && !state.resultText) return "";
|
|
const cls = state.status === "error"
|
|
? "agent-tool-call-result error"
|
|
: state.status === "success"
|
|
? "agent-tool-call-result success"
|
|
: "agent-tool-call-result running";
|
|
const label = state.status === "running"
|
|
? (state.lastAction === "send" ? "Calling tool and sending to conversation..." : "Calling tool...")
|
|
: state.status === "error"
|
|
? "Call failed"
|
|
: (state.lastAction === "send" ? "Sent to conversation" : "Call result");
|
|
return `
|
|
<div class="${cls}">
|
|
<div class="agent-tool-call-result-title">${escHtml(label)}</div>
|
|
<pre class="agent-tool-call-output">${escHtml(state.resultText || "")}</pre>
|
|
</div>
|
|
`;
|
|
}
|
|
|
|
/**
|
|
* @param {string} toolName
|
|
* @param {boolean} [sendToConversation]
|
|
* @param {string} [sourceQuery]
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async function callAgentToolFromPanel(toolName, sendToConversation = false, sourceQuery = "") {
|
|
const tool = findAgentToolDef(toolName);
|
|
if (!tool) return;
|
|
const state = getMcpToolCallState(toolName);
|
|
state.lastAction = sendToConversation ? "send" : "call";
|
|
let args = /** @type {Record<string, unknown>} */ ({});
|
|
try {
|
|
args = state.argsText.trim() ? JSON.parse(state.argsText) : {};
|
|
} catch (err) {
|
|
const msg = err instanceof Error ? err.message : String(err);
|
|
state.status = "error";
|
|
state.resultText = `Invalid JSON arguments: ${msg}`;
|
|
syncToolsUi();
|
|
console.warn(`[Vocero MCP PANEL] invalid args name=${toolName} error=${msg}`);
|
|
return;
|
|
}
|
|
if (!args || typeof args !== "object" || Array.isArray(args)) {
|
|
state.status = "error";
|
|
state.resultText = "Arguments must be a JSON object.";
|
|
syncToolsUi();
|
|
console.warn(`[Vocero MCP PANEL] invalid args shape name=${toolName}`);
|
|
return;
|
|
}
|
|
const inferredArgs = sourceQuery.trim() ? inferToolArgsFromQuery(sourceQuery, tool.parameters) : {};
|
|
args = applyToolArgDefaults({ ...inferredArgs, ...args }, tool.parameters);
|
|
const missingArgs = missingRequiredToolArgs(args, tool.parameters);
|
|
if (missingArgs.length) {
|
|
state.status = "error";
|
|
state.resultText = `Missing required arguments: ${missingArgs.join(", ")}`;
|
|
syncToolsUi();
|
|
console.warn(
|
|
`[Vocero MCP PANEL] missing required args name=${toolName} missing=${missingArgs.join(",")} ` +
|
|
`sourceQuery=${JSON.stringify(sourceQuery)}`
|
|
);
|
|
return;
|
|
}
|
|
state.status = "running";
|
|
state.resultText = "";
|
|
syncToolsUi();
|
|
reportDebug("E", "main.js:callAgentToolFromPanel", "manual MCP tool call from tools panel", {
|
|
name: toolName,
|
|
args,
|
|
sendToConversation,
|
|
sourceQuery,
|
|
});
|
|
console.debug(
|
|
`[Vocero MCP PANEL] call name=${toolName} sendToConversation=${sendToConversation} ` +
|
|
`sourceQuery=${JSON.stringify(sourceQuery)} inferred=${JSON.stringify(inferredArgs)} args=${JSON.stringify(args)}`
|
|
);
|
|
try {
|
|
const execStartedAt = performance.now();
|
|
const { rawOutput, speakableOutput } = await execRemoteMcpTool(toolName, args);
|
|
const output = rawOutput;
|
|
reportDebug("A", "main.js:callAgentToolFromPanel", "MCP tool execution finished", {
|
|
name: toolName,
|
|
sendToConversation,
|
|
outputLength: output.length,
|
|
speakableOutputLength: speakableOutput.length,
|
|
durationMs: Math.round(performance.now() - execStartedAt),
|
|
});
|
|
if (sendToConversation) {
|
|
if (!client || !LIVE_STATES.has(currentState)) {
|
|
throw new Error("No live conversation is active.");
|
|
}
|
|
const promptBuildStartedAt = performance.now();
|
|
const modelOutput = speakableOutput || summarizeMcpOutputForConversation(output);
|
|
const prompt = (
|
|
sourceQuery.trim()
|
|
? `User request: ${sourceQuery.trim()}\n\n`
|
|
: ""
|
|
) + `Tool ${toolName} returned the following internal facts:\n\n${modelOutput}\n\n` +
|
|
"Answer the user naturally in their language. Do not read JSON keys, field names, or raw URLs aloud unless the user explicitly asks for them.";
|
|
reportDebug("A", "main.js:callAgentToolFromPanel", "built live prompt from MCP output", {
|
|
name: toolName,
|
|
outputLength: output.length,
|
|
modelOutputLength: modelOutput.length,
|
|
promptLength: prompt.length,
|
|
buildMs: Math.round(performance.now() - promptBuildStartedAt),
|
|
});
|
|
const chatRenderStartedAt = performance.now();
|
|
chat.onToolResult(toolName, JSON.stringify(args), output);
|
|
reportDebug("C", "main.js:callAgentToolFromPanel", "rendered MCP tool result in chat", {
|
|
name: toolName,
|
|
outputLength: output.length,
|
|
renderMs: Math.round(performance.now() - chatRenderStartedAt),
|
|
});
|
|
chat.onTranscript({
|
|
role: "user",
|
|
text: prompt,
|
|
partial: false,
|
|
itemId: `manual-mcp-${toolName}-${Date.now()}`,
|
|
});
|
|
reportDebug("B", "main.js:callAgentToolFromPanel", "sending MCP prompt back to live conversation", {
|
|
name: toolName,
|
|
promptLength: prompt.length,
|
|
outputLength: output.length,
|
|
});
|
|
client.sendUserText(prompt);
|
|
client.requestResponse();
|
|
console.debug(`[Vocero MCP PANEL] sent to conversation name=${toolName} length=${prompt.length}`);
|
|
}
|
|
state.status = "success";
|
|
state.resultText = output;
|
|
console.debug(`[Vocero MCP PANEL] result name=${toolName} output=${output.slice(0, 500)}`);
|
|
} catch (err) {
|
|
const msg = err instanceof Error ? err.message : String(err);
|
|
state.status = "error";
|
|
state.resultText = msg;
|
|
console.warn(`[Vocero MCP PANEL] error name=${toolName} error=${msg}`);
|
|
}
|
|
syncToolsUi();
|
|
}
|
|
|
|
toolsBtn.addEventListener("click", () => { syncToolsUi(); toolsModal.showModal(); });
|
|
toolsClose.addEventListener("click", () => toolsModal.close());
|
|
toolsModal.addEventListener("click", (e) => {
|
|
if (e.target === toolsModal) toolsModal.close();
|
|
});
|
|
agentToolsList.addEventListener("input", (event) => {
|
|
const target = /** @type {HTMLElement | null} */ (event.target instanceof HTMLElement ? event.target : null);
|
|
if (!(target instanceof HTMLTextAreaElement)) return;
|
|
const toolName = target.dataset.mcpToolArgs;
|
|
if (!toolName) return;
|
|
const state = getMcpToolCallState(toolName);
|
|
state.argsText = target.value;
|
|
});
|
|
agentToolsList.addEventListener("click", (event) => {
|
|
const target = /** @type {HTMLElement | null} */ (event.target instanceof HTMLElement ? event.target : null);
|
|
const button = target?.closest("[data-mcp-call-btn]");
|
|
if (button instanceof HTMLButtonElement) {
|
|
const toolName = button.dataset.mcpCallBtn;
|
|
if (!toolName) return;
|
|
void callAgentToolFromPanel(toolName, false);
|
|
return;
|
|
}
|
|
const sendButton = target?.closest("[data-mcp-send-btn]");
|
|
if (!(sendButton instanceof HTMLButtonElement)) return;
|
|
const toolName = sendButton.dataset.mcpSendBtn;
|
|
if (!toolName) return;
|
|
void callAgentToolFromPanel(toolName, true);
|
|
});
|
|
|
|
toolWebSwitch.addEventListener("change", () => {
|
|
if (toolWebSwitch.checked && !searchAvailable()) {
|
|
toolWebSwitch.checked = false; // guard: can't enable without a key
|
|
return;
|
|
}
|
|
toolsEnabled.web_search = toolWebSwitch.checked;
|
|
saveTools();
|
|
pushToolsToSession();
|
|
});
|
|
|
|
searchKeyInput.addEventListener("input", () => {
|
|
if (serverSearchKey) return;
|
|
userSearchKey = searchKeyInput.value.trim();
|
|
if (userSearchKey) localStorage.setItem(STORAGE_KEYS.searchKey, userSearchKey);
|
|
else localStorage.removeItem(STORAGE_KEYS.searchKey);
|
|
|
|
const avail = searchAvailable();
|
|
toolWebSwitch.disabled = !avail;
|
|
toolWebRow.classList.toggle("disabled", !avail);
|
|
// Losing the key disables a previously-enabled tool.
|
|
if (!avail && toolsEnabled.web_search) {
|
|
toolsEnabled.web_search = false;
|
|
toolWebSwitch.checked = false;
|
|
saveTools();
|
|
pushToolsToSession();
|
|
}
|
|
toolWebHint.textContent = userSearchKey
|
|
? "Using your key — stored in this browser only."
|
|
: "No server key configured. Add your own Serper key to enable web search.";
|
|
});
|
|
|
|
// ── Tool executor ─────────────────────────────────────────────────────────
|
|
// Runs the function the model called, returns the result, and asks for a
|
|
// response so the model speaks it. Errors come back as the tool output too, so
|
|
// the model can recover gracefully instead of the turn stalling.
|
|
|
|
/**
|
|
* Run the function the model called, return its result to the backend, and ask
|
|
* for a follow-up response. We also hand the result back to the caller so it
|
|
* can be shown in the conversation once the tool has actually run.
|
|
* @param {string} name @param {string} argsJson @param {string} callId
|
|
* @returns {Promise<{ output: string, image?: string }>}
|
|
*/
|
|
async function runTool(name, argsJson, callId) {
|
|
if (!client) return { output: "" };
|
|
let args = /** @type {Record<string, unknown>} */ ({});
|
|
try { args = JSON.parse(argsJson || "{}"); } catch { /* keep {} */ }
|
|
|
|
// #region debug-point D:run-tool-entry
|
|
reportDebug("D", "main.js:runTool", "tool execution started", {
|
|
name,
|
|
callId,
|
|
args,
|
|
knownMcpTool: agentToolDefs.some((tool) => tool.name === name),
|
|
});
|
|
// #endregion
|
|
|
|
if (DEBUG) console.debug(`[tool] run name=${name} callId=${JSON.stringify(callId)} args=${argsJson}`);
|
|
if (!callId) console.warn("[tool] empty call_id — the backend didn't tag the call, can't return a function_call_output");
|
|
|
|
/** @type {{ output: string, image?: string }} */
|
|
let result = { output: "" };
|
|
try {
|
|
if (name === "web_search") {
|
|
const query = typeof args.query === "string" ? args.query : "";
|
|
result.output = await execWebSearch(query);
|
|
// Return the result and let the bare response.create (below) trigger the
|
|
// spoken answer.
|
|
client.sendToolOutput(callId, result.output);
|
|
} else if (agentToolDefs.some((tool) => tool.name === name)) {
|
|
// #region debug-point E:mcp-proxy-before-fetch
|
|
reportDebug("E", "main.js:runTool", "calling MCP proxy", {
|
|
name,
|
|
callId,
|
|
args,
|
|
});
|
|
// #endregion
|
|
const mcpResult = await execRemoteMcpTool(name, args);
|
|
result.output = mcpResult.rawOutput;
|
|
console.debug(`[Vocero MCP OUT] name=${name} callId=${JSON.stringify(callId)} output=${result.output}`);
|
|
// #region debug-point E:mcp-proxy-after-fetch
|
|
reportDebug("E", "main.js:runTool", "MCP proxy returned result", {
|
|
name,
|
|
callId,
|
|
outputPreview: result.output.slice(0, 300),
|
|
speakableOutputLength: mcpResult.speakableOutput.length,
|
|
});
|
|
// #endregion
|
|
client.sendToolOutput(callId, mcpResult.speakableOutput || summarizeMcpOutputForConversation(result.output));
|
|
} else {
|
|
result.output = `Unknown tool: ${name}`;
|
|
client.sendToolOutput(callId, result.output);
|
|
}
|
|
} catch (err) {
|
|
const msg = err instanceof Error ? err.message : String(err);
|
|
result.output = `Tool failed: ${msg}`;
|
|
// #region debug-point E:mcp-proxy-error
|
|
reportDebug("E", "main.js:runTool", "tool execution failed", {
|
|
name,
|
|
callId,
|
|
error: msg,
|
|
});
|
|
// #endregion
|
|
client.sendToolOutput(callId, result.output);
|
|
}
|
|
if (DEBUG) console.debug(`[tool] requesting model response after ${name}`);
|
|
client.requestResponse(result.image ? { image: result.image } : undefined);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Execute an MCP-backed tool through the app's same-origin proxy.
|
|
* @param {string} name
|
|
* @param {Record<string, unknown>} args
|
|
* @returns {Promise<{ rawOutput: string, speakableOutput: string }>}
|
|
*/
|
|
async function execRemoteMcpTool(name, args) {
|
|
const startedAt = performance.now();
|
|
if (DEBUG) {
|
|
console.debug(`[Vocero MCP HTTP] request name=${name} url=/api/mcp/tool args=${JSON.stringify(args || {})}`);
|
|
}
|
|
const res = await fetch("api/mcp/tool", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ name, arguments: args || {} }),
|
|
});
|
|
if (DEBUG) {
|
|
console.debug(`[Vocero MCP HTTP] response name=${name} status=${res.status} ok=${res.ok}`);
|
|
}
|
|
if (!res.ok) {
|
|
let detail = String(res.status);
|
|
try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {}
|
|
console.warn(`[Vocero MCP HTTP] error name=${name} detail=${detail}`);
|
|
throw new Error(`mcp error (${detail})`);
|
|
}
|
|
const json = await res.json();
|
|
reportDebug("A", "main.js:execRemoteMcpTool", "received MCP HTTP response body", {
|
|
name,
|
|
durationMs: Math.round(performance.now() - startedAt),
|
|
resultJsonLength: JSON.stringify(json?.result ?? null).length,
|
|
});
|
|
if (DEBUG) {
|
|
const preview = JSON.stringify(json?.result ?? null).slice(0, 300);
|
|
console.debug(`[Vocero MCP HTTP] payload name=${name} preview=${preview}`);
|
|
}
|
|
const rawOutput = formatMcpToolResult(json.result);
|
|
const speakableOutput = formatSpeakableMcpToolResult(json.result, rawOutput);
|
|
return { rawOutput, speakableOutput };
|
|
}
|
|
|
|
/**
|
|
* Turn an MCP tool result into plain text the model can read back.
|
|
* @param {any} result
|
|
* @returns {string}
|
|
*/
|
|
function formatMcpToolResult(result) {
|
|
if (!result) return "Tool returned no result.";
|
|
if (result.structuredContent) {
|
|
return JSON.stringify(result.structuredContent, null, 2);
|
|
}
|
|
/** @type {{ type?: string, text?: string }[]} */
|
|
const parts = Array.isArray(result.content) ? result.content : [];
|
|
/** @type {string[]} */
|
|
const textParts = [];
|
|
for (const item of parts) {
|
|
if (item && item.type === "text" && typeof item.text === "string") {
|
|
const text = item.text.trim();
|
|
if (text) textParts.push(text);
|
|
}
|
|
}
|
|
if (textParts.length) return textParts.join("\n\n");
|
|
return JSON.stringify(result, null, 2);
|
|
}
|
|
|
|
/**
|
|
* Turn an MCP tool result into compact speech-ready facts for the live model/TTS path.
|
|
* @param {any} result
|
|
* @param {string} [rawOutput]
|
|
* @returns {string}
|
|
*/
|
|
function formatSpeakableMcpToolResult(result, rawOutput = "") {
|
|
if (!result) return "Tool returned no result.";
|
|
if (result.structuredContent != null) {
|
|
const summary = buildSpeakableMcpFacts(result.structuredContent);
|
|
if (summary) return summary.length > 1200 ? `${summary.slice(0, 1200).trim()}\n...` : summary;
|
|
}
|
|
/** @type {{ type?: string, text?: string }[]} */
|
|
const parts = Array.isArray(result.content) ? result.content : [];
|
|
/** @type {string[]} */
|
|
const textParts = [];
|
|
for (const item of parts) {
|
|
if (item && item.type === "text" && typeof item.text === "string") {
|
|
const text = item.text.trim();
|
|
if (text) textParts.push(text);
|
|
}
|
|
}
|
|
if (textParts.length) {
|
|
return summarizeMcpOutputForConversation(textParts.join("\n\n"));
|
|
}
|
|
return summarizeMcpOutputForConversation(rawOutput || JSON.stringify(result, null, 2));
|
|
}
|
|
|
|
const SPEAKABLE_PRIORITY_KEYS = new Set([
|
|
"name",
|
|
"title",
|
|
"summary",
|
|
"description",
|
|
"message",
|
|
"status",
|
|
"role",
|
|
"date",
|
|
"reportId",
|
|
"report_id",
|
|
"center",
|
|
"priority",
|
|
"audience",
|
|
"briefing",
|
|
"value",
|
|
"label",
|
|
"text",
|
|
]);
|
|
|
|
const SKIP_SPEAKABLE_KEYS = new Set([
|
|
"id",
|
|
"userId",
|
|
"user_id",
|
|
"openapiUrl",
|
|
"docsUrl",
|
|
"mcpPath",
|
|
"url",
|
|
"urls",
|
|
"image",
|
|
"images",
|
|
"raw",
|
|
"json",
|
|
]);
|
|
|
|
/**
|
|
* @param {string} text
|
|
* @param {number} maxLen
|
|
* @returns {string}
|
|
*/
|
|
function trimSpeakableText(text, maxLen) {
|
|
const clean = String(text || "").replace(/\s+/g, " ").trim();
|
|
if (!clean) return "";
|
|
return clean.length > maxLen ? `${clean.slice(0, maxLen).trim()}...` : clean;
|
|
}
|
|
|
|
/**
|
|
* @param {string} key
|
|
* @returns {string}
|
|
*/
|
|
function humanizeFactKey(key) {
|
|
return key
|
|
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
.replace(/[_-]+/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim()
|
|
.replace(/^./, (c) => c.toUpperCase());
|
|
}
|
|
|
|
/**
|
|
* @param {any} value
|
|
* @param {number} [maxLen]
|
|
* @returns {string}
|
|
*/
|
|
function pickSpeakableSnippet(value, maxLen = 180) {
|
|
if (value == null) return "";
|
|
if (typeof value === "string") return trimSpeakableText(value, maxLen);
|
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
if (Array.isArray(value)) {
|
|
const parts = value
|
|
.slice(0, 3)
|
|
.map((item) => pickSpeakableSnippet(item, 80))
|
|
.filter(Boolean);
|
|
return trimSpeakableText(parts.join("; "), maxLen);
|
|
}
|
|
if (typeof value === "object") {
|
|
const obj = /** @type {Record<string, any>} */ (value);
|
|
for (const key of SPEAKABLE_PRIORITY_KEYS) {
|
|
if (key in obj) {
|
|
const text = pickSpeakableSnippet(obj[key], maxLen);
|
|
if (text) return text;
|
|
}
|
|
}
|
|
for (const [key, item] of Object.entries(obj)) {
|
|
if (SKIP_SPEAKABLE_KEYS.has(key)) continue;
|
|
const text = pickSpeakableSnippet(item, maxLen);
|
|
if (text) return text;
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
|
|
/**
|
|
* @param {Record<string, any>} obj
|
|
* @returns {string[]}
|
|
*/
|
|
function collectSpeakableObjectFacts(obj) {
|
|
/** @type {string[]} */
|
|
const facts = [];
|
|
for (const [key, value] of Object.entries(obj)) {
|
|
if (SKIP_SPEAKABLE_KEYS.has(key) || value == null) continue;
|
|
if (typeof value === "string") {
|
|
const text = trimSpeakableText(value, 220);
|
|
if (text) facts.push(`${humanizeFactKey(key)}: ${text}`);
|
|
continue;
|
|
}
|
|
if (typeof value === "number" || typeof value === "boolean") {
|
|
facts.push(`${humanizeFactKey(key)}: ${String(value)}`);
|
|
continue;
|
|
}
|
|
if (Array.isArray(value)) {
|
|
if (!value.length) continue;
|
|
const first = pickSpeakableSnippet(value[0], 160);
|
|
if (first) {
|
|
facts.push(`${humanizeFactKey(key)}: ${first}`);
|
|
if (value.length > 1) facts.push(`${humanizeFactKey(key)} count: ${value.length}`);
|
|
}
|
|
continue;
|
|
}
|
|
if (typeof value === "object") {
|
|
const nested = pickSpeakableSnippet(value, 180);
|
|
if (nested) facts.push(`${humanizeFactKey(key)}: ${nested}`);
|
|
}
|
|
}
|
|
return facts;
|
|
}
|
|
|
|
/**
|
|
* @param {any} parsed
|
|
* @returns {string}
|
|
*/
|
|
function buildSpeakableMcpFacts(parsed) {
|
|
if (parsed == null) return "No useful facts returned by the tool.";
|
|
if (typeof parsed === "string") return trimSpeakableText(parsed, 900);
|
|
if (typeof parsed === "number" || typeof parsed === "boolean") return String(parsed);
|
|
if (Array.isArray(parsed)) {
|
|
const items = parsed
|
|
.slice(0, 4)
|
|
.map((item, index) => {
|
|
const text = pickSpeakableSnippet(item, 180);
|
|
return text ? `- Item ${index + 1}: ${text}` : "";
|
|
})
|
|
.filter(Boolean);
|
|
if (parsed.length > 4) items.push(`- Additional items not shown: ${parsed.length - 4}`);
|
|
return items.join("\n") || "The tool returned a list with no speakable details.";
|
|
}
|
|
const obj = /** @type {Record<string, any>} */ (parsed);
|
|
const facts = collectSpeakableObjectFacts(obj).slice(0, 8);
|
|
if (facts.length) return facts.map((line) => `- ${line}`).join("\n");
|
|
const fallback = pickSpeakableSnippet(obj, 900);
|
|
return fallback || "The tool returned structured data with no speakable summary.";
|
|
}
|
|
|
|
/**
|
|
* @param {string} output
|
|
* @returns {string}
|
|
*/
|
|
function summarizeMcpOutputForConversation(output) {
|
|
const text = (output || "").trim();
|
|
if (!text) return "Tool returned no result.";
|
|
try {
|
|
const parsed = JSON.parse(text);
|
|
const summary = buildSpeakableMcpFacts(parsed);
|
|
return summary.length > 1200 ? `${summary.slice(0, 1200).trim()}\n...` : summary;
|
|
} catch {
|
|
return trimSpeakableText(text, 1200);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Normalize agent profile tools into OpenAI-compatible function tool defs.
|
|
* @param {any[]} tools
|
|
* @returns {import("./ws/s2s-ws-client.js").ToolDef[]}
|
|
*/
|
|
function normalizeAgentTools(tools) {
|
|
if (!Array.isArray(tools)) return [];
|
|
return tools
|
|
.filter((tool) => tool && tool.enabled !== false && typeof tool.name === "string")
|
|
.map((tool) => ({
|
|
type: "function",
|
|
name: tool.name,
|
|
description: typeof tool.description === "string" ? tool.description : "",
|
|
parameters: tool.parameters && typeof tool.parameters === "object"
|
|
? tool.parameters
|
|
: { type: "object", properties: {}, required: [] },
|
|
}));
|
|
}
|
|
|
|
/** @param {string} query @returns {Promise<string>} */
|
|
async function execWebSearch(query) {
|
|
if (!query) return "No query provided.";
|
|
/** @type {Record<string, string>} */
|
|
const body = { query };
|
|
// Only send a user key when there's no server key (server prefers its own).
|
|
if (!serverSearchKey && userSearchKey) body.key = userSearchKey;
|
|
|
|
const res = await fetch("api/search", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify(body),
|
|
});
|
|
if (!res.ok) {
|
|
let detail = String(res.status);
|
|
try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {}
|
|
throw new Error(`search error (${detail})`);
|
|
}
|
|
const json = await res.json();
|
|
// Date-stamp the header so the model treats these as fresh realtime facts
|
|
// rather than its (older) training knowledge.
|
|
const today = new Date().toISOString().slice(0, 10);
|
|
/** @type {string[]} */
|
|
const lines = [`Google search result from ${today}:`];
|
|
if (json.answer) lines.push(`Answer: ${json.answer}`);
|
|
for (const r of json.results || []) {
|
|
lines.push(`- ${r.title}: ${r.snippet} (${r.url})`);
|
|
}
|
|
return lines.length > 1 ? lines.join("\n") : `${lines[0]}\nNo results found.`;
|
|
}
|
|
|
|
async function fetchVoceroUserProfile() {
|
|
if (!voceroUserPhoneNumber) return;
|
|
try {
|
|
const res = await fetch(`api/user/phone?phoneNumber=${encodeURIComponent(voceroUserPhoneNumber)}`);
|
|
if (!res.ok) {
|
|
const message = await res.text();
|
|
throw new Error(message || `HTTP ${res.status}`);
|
|
}
|
|
const json = await res.json();
|
|
voceroUserProfile = json && typeof json === "object" && json.result && typeof json.result === "object"
|
|
? json.result
|
|
: null;
|
|
voceroUserProfileLoaded = true;
|
|
console.debug(
|
|
`[Vocero USER AUTH] phoneNumber=${JSON.stringify(voceroUserPhoneNumber)} ` +
|
|
`loaded=${!!voceroUserProfile} keys=${voceroUserProfile ? Object.keys(voceroUserProfile).join(",") : "none"}`
|
|
);
|
|
syncVoceroHeaderUser();
|
|
if (client && LIVE_STATES.has(currentState)) {
|
|
pushLiveInstructions(effectiveInstructions());
|
|
}
|
|
} catch (err) {
|
|
voceroUserProfile = null;
|
|
voceroUserProfileLoaded = false;
|
|
syncVoceroHeaderUser();
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
console.warn(`[Vocero USER AUTH] phoneNumber=${JSON.stringify(voceroUserPhoneNumber)} error=${message}`);
|
|
}
|
|
}
|
|
|
|
/** Learn server config (search key + connection target), then refresh the UI. */
|
|
async function fetchConfig() {
|
|
try {
|
|
const res = await fetch("api/config");
|
|
if (res.ok) {
|
|
const json = await res.json();
|
|
serverSearchKey = !!json.search;
|
|
lbMode = !!json.lb;
|
|
reasoningType = ["OPENAI","NODE_RED"].includes((json.reasoningType || "").toUpperCase())
|
|
? (json.reasoningType || "").toUpperCase()
|
|
: "PROXY";
|
|
// Lock to LB mode only when the deploy reports a load balancer.
|
|
allowDirect = json.allowDirect ?? !lbMode;
|
|
// Deploy-pinned direct URL (overrides the LB server-side already).
|
|
pinnedUrl = (json.s2sUrl || "").trim();
|
|
pinnedPrompt = (json.primaryPrompt || "").trim();
|
|
pinnedPromptSource = (json.agentProfile || "").trim();
|
|
agentToolDefs = normalizeAgentTools(json.agentTools || []);
|
|
searchMcpEnabled = !!json.searchMcp;
|
|
rankedAgentTools = rankAgentTools(lastUserQuery);
|
|
// #region debug-point B:config-loaded
|
|
reportDebug("B", "main.js:fetchConfig", "loaded config from api/config", {
|
|
searchMcpEnabled,
|
|
reasoningType,
|
|
agentToolNames: agentToolDefs.map((tool) => tool.name),
|
|
agentToolCount: agentToolDefs.length,
|
|
pinnedPromptLength: pinnedPrompt.length,
|
|
});
|
|
// #endregion
|
|
// The conversation-time limiter rides on the LB being present.
|
|
limiterOn = lbMode;
|
|
}
|
|
// Non-OK response: leave the fail-open default (allowDirect = true).
|
|
} catch {
|
|
// Config endpoint unreachable (e.g. static hosting): keep direct entry.
|
|
}
|
|
if (DEBUG) console.debug(`[ui] config: allowDirect=${allowDirect} lbMode=${lbMode}`);
|
|
// Login chip + remaining-budget (no-op / hidden when the limiter is off).
|
|
void account.refresh();
|
|
syncToolsUi();
|
|
syncConnectionUi();
|
|
}
|
|
|
|
/**
|
|
* Resolve where to connect, per the deploy's mode:
|
|
* • LB mode -> `{ sessionUrl }`, the client POSTs the same-origin /api/session
|
|
* proxy and the server forwards to the LB (its address stays server-side).
|
|
* • direct -> `{ directUrl }`, connect straight to the s2s WebSocket.
|
|
* Throws a user-facing error if direct mode is on but no URL was entered.
|
|
* @returns {{ sessionUrl: string } | { directUrl: string }}
|
|
*/
|
|
function connectionTarget() {
|
|
if (!allowDirect) {
|
|
return { sessionUrl: "api/session" };
|
|
}
|
|
const directUrl = buildDirectWsUrl(pinnedUrl || settings.directUrl);
|
|
if (!directUrl) {
|
|
throw new Error("Enter a speech-to-speech server URL in Settings.");
|
|
}
|
|
return { directUrl };
|
|
}
|
|
|
|
/**
|
|
* Normalise a user-typed server address into a realtime WebSocket URL.
|
|
* Accepts bare hosts (`localhost:8080`), http(s) URLs, or ws(s) URLs, and adds
|
|
* the `/v1/realtime` path when none is given. A full connect URL (with path
|
|
* and/or query) is preserved as-is.
|
|
* @param {string} raw @returns {string}
|
|
*/
|
|
function buildDirectWsUrl(raw) {
|
|
let s = (raw || "").trim();
|
|
if (!s) return "";
|
|
if (!/^wss?:\/\//i.test(s)) {
|
|
if (/^https?:\/\//i.test(s)) {
|
|
s = s.replace(/^http/i, "ws"); // http→ws, https→wss
|
|
} else {
|
|
const isLocal = /^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(s);
|
|
s = (isLocal ? "ws://" : "wss://") + s;
|
|
}
|
|
}
|
|
try {
|
|
const u = new URL(s);
|
|
if (u.pathname === "" || u.pathname === "/") u.pathname = "/v1/realtime";
|
|
return u.toString();
|
|
} catch {
|
|
return s;
|
|
}
|
|
}
|
|
|
|
/** Create + resume an AudioContext synchronously (must run inside the user
|
|
* gesture so iOS lets it start). Returns null if construction fails. */
|
|
function createResumedAudioContext() {
|
|
try {
|
|
const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext;
|
|
const ctx = new Ctx({ latencyHint: "interactive" });
|
|
if (ctx.state === "suspended") void ctx.resume().catch(() => {});
|
|
return /** @type {AudioContext} */ (ctx);
|
|
} catch (err) {
|
|
console.warn("[main] AudioContext init failed:", err);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Read the editable settings out of the form. The URL field is only honoured
|
|
* in free direct mode — in LB mode it's hidden, and when the deploy pins a
|
|
* URL it's read-only, so the user's saved URL survives either way. */
|
|
function readSettingsFromForm() {
|
|
return {
|
|
directUrl: allowDirect && !pinnedUrl ? inputLbUrl.value.trim() : settings.directUrl,
|
|
voice: inputVoice.value || DEFAULT_VOICE,
|
|
instructions: pinnedPrompt ? settings.instructions : inputInstructions.value.trim() || DEFAULT_INSTRUCTIONS,
|
|
noiseGate: readGateThreshold(),
|
|
};
|
|
}
|
|
|
|
/** Gate threshold (dBFS) currently shown on the slider, clamped to range. */
|
|
function readGateThreshold() {
|
|
const v = Math.round(Number(inputNoiseGate.value));
|
|
if (!Number.isFinite(v)) return GATE_OFF_DB;
|
|
return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, v));
|
|
}
|
|
|
|
/** Adapt the connection field to the mode learned from /api/config. */
|
|
function syncConnectionUi() {
|
|
if (pinnedUrl) {
|
|
// Deploy-pinned URL: show it, but locked — the deployment owns it.
|
|
connField.hidden = false;
|
|
inputLbUrl.value = pinnedUrl;
|
|
inputLbUrl.readOnly = true;
|
|
connHint.classList.remove("error");
|
|
connHint.textContent = "Speech-to-speech server URL pinned by this deployment.";
|
|
} else if (allowDirect) {
|
|
// Direct mode: the user sets their own s2s server URL.
|
|
connField.hidden = false;
|
|
inputLbUrl.value = settings.directUrl;
|
|
inputLbUrl.readOnly = false;
|
|
inputLbUrl.placeholder = "http://localhost:port";
|
|
connHint.classList.remove("error");
|
|
connHint.textContent =
|
|
"URL of your speech-to-speech server, e.g. http://localhost:8080 (the app adds /v1/realtime).";
|
|
} else {
|
|
// LB mode: the load balancer URL is deployment-owned — hide it entirely so
|
|
// its address is never exposed in Settings.
|
|
connField.hidden = true;
|
|
}
|
|
}
|
|
|
|
/** True when the user must supply a server URL before connecting (direct mode
|
|
* with nothing set). */
|
|
function missingServerUrl() {
|
|
return allowDirect && !pinnedUrl && !buildDirectWsUrl(settings.directUrl);
|
|
}
|
|
|
|
/** Open Settings and point the user at the empty server-URL field. */
|
|
function promptServerUrl() {
|
|
if (settingsModal.open) syncConnectionUi();
|
|
else openSettings();
|
|
connHint.textContent = "Set the speech-to-speech server URL to start.";
|
|
connHint.classList.add("error");
|
|
inputLbUrl.focus();
|
|
}
|
|
|
|
settingsForm.addEventListener("submit", (event) => {
|
|
const submitter = /** @type {HTMLButtonElement | null} */ ((/** @type {SubmitEvent} */ (event)).submitter);
|
|
if (submitter?.value !== "save") return;
|
|
|
|
settings = readSettingsFromForm();
|
|
saveSettings(settings);
|
|
|
|
// Voice + instructions can apply to a live session without reconnecting; a
|
|
// changed connection URL only takes effect on the next restart.
|
|
if (client && LIVE_STATES.has(currentState)) {
|
|
client.updateSession({ voice: settings.voice, instructions: effectiveInstructions() });
|
|
}
|
|
});
|
|
|
|
// The noise gate applies live (worklet param), so tune it without a restart:
|
|
// update the label/marker, persist, and push straight to the running client.
|
|
inputNoiseGate.addEventListener("input", () => {
|
|
setGateThreshold(readGateThreshold());
|
|
});
|
|
|
|
restartBtn.addEventListener("click", async () => {
|
|
if (currentState === "connecting") return; // a connect is already underway
|
|
settings = readSettingsFromForm();
|
|
saveSettings(settings);
|
|
if (missingServerUrl()) { promptServerUrl(); return; } // keep settings open
|
|
settingsModal.close();
|
|
// Grab the AudioContext NOW, inside the click gesture — teardown() awaits, and
|
|
// creating it afterwards would fall outside the gesture (silent on iOS).
|
|
const audioContext = createResumedAudioContext();
|
|
try {
|
|
if (client) await teardown();
|
|
await doStart(audioContext);
|
|
} catch (err) {
|
|
await handleStartError(err);
|
|
}
|
|
});
|
|
|
|
circleBtn.addEventListener("click", async () => {
|
|
try {
|
|
if (currentState === "idle" || currentState === "error") {
|
|
if (missingServerUrl()) { promptServerUrl(); return; }
|
|
await doStart();
|
|
}
|
|
} catch (err) {
|
|
await handleStartError(err);
|
|
}
|
|
});
|
|
|
|
/** A failed start is either the daily limit (show the modal, return to idle) or
|
|
* a real fault (surface it). doStart already closed any orphan AudioContext.
|
|
* @param {any} err */
|
|
async function handleStartError(err) {
|
|
if (err && err.code === "limit") {
|
|
await teardown();
|
|
account.showLimit(err.tier);
|
|
return;
|
|
}
|
|
// The user left the queue (close() aborted the wait): teardown already reset
|
|
// the UI to idle, so there's nothing to report.
|
|
if (err && err.code === "aborted") return;
|
|
// The whole waiting line is full: a warm, reassuring modal rather than an error.
|
|
if (err && err.code === "queue-full") {
|
|
await teardown();
|
|
account.showBusy();
|
|
return;
|
|
}
|
|
// Our place lapsed (ticket reaped, or the join window ran out). Recoverable, not
|
|
// a fault: land on the retry state with a kind, plain-language reason.
|
|
if (err && (err.code === "queue-expired" || err.code === "join-expired")) {
|
|
await teardown();
|
|
setState("error");
|
|
setCaption(
|
|
err.code === "join-expired"
|
|
? "Your spot expired. Tap to rejoin."
|
|
: "That took a while. Tap to rejoin.",
|
|
"error",
|
|
);
|
|
return;
|
|
}
|
|
onFatalError(err);
|
|
}
|
|
|
|
micBtn.addEventListener("click", () => {
|
|
if (!micStream || !client) return;
|
|
micMuted = !micMuted;
|
|
for (const track of micStream.getAudioTracks()) {
|
|
track.enabled = !micMuted;
|
|
}
|
|
client.setMuted(micMuted);
|
|
micBtn.classList.toggle("muted", micMuted);
|
|
micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute");
|
|
micBtn.title = micMuted ? "Unmute" : "Mute";
|
|
});
|
|
|
|
stopBtn.addEventListener("click", async () => {
|
|
await teardown();
|
|
});
|
|
|
|
// "Leave queue": tear down the pending connect (aborts the poll wait) and drop
|
|
// our place in line. Same teardown path as stopping a live call.
|
|
leaveQueueBtn.addEventListener("click", async () => {
|
|
await teardown();
|
|
});
|
|
|
|
// "Join now": accept the held slot. The click is a user gesture, so the client
|
|
// re-resumes the AudioContext here (iOS) before dialing.
|
|
joinQueueBtn.addEventListener("click", () => {
|
|
stopJoinCountdown();
|
|
if (client) client.join();
|
|
});
|
|
|
|
const MIC_CONSTRAINTS = {
|
|
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
|
};
|
|
|
|
/** Prompt for mic permission up front, then immediately release the tracks so no
|
|
* recording indicator lingers during a queue wait. Throws a friendly error if the
|
|
* user denies. */
|
|
async function primeMicPermission() {
|
|
try {
|
|
const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
|
|
for (const track of s.getTracks()) track.stop();
|
|
} catch (err) {
|
|
throw new Error(
|
|
`Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/** Acquire the live capture stream once a slot is granted. Permission was primed
|
|
* in the tap gesture, so this is silent. Stored module-side for mute + teardown. */
|
|
async function acquireMicStream() {
|
|
micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
|
|
return micStream;
|
|
}
|
|
|
|
/** @param {number} position Update the queued caption ("You're #N in line"). */
|
|
function onQueuePosition(position) {
|
|
const n = Number(position) || 0;
|
|
setCaption(n > 0 ? `You're #${n} in line` : "Finding you a spot…", "muted");
|
|
}
|
|
|
|
// ── "Your turn" join countdown ──────────────────────────────────────────────
|
|
// While a slot is held for us, show how long is left to accept it. The client's
|
|
// join gate expires just before the load balancer reclaims the slot.
|
|
let joinCountdownTimer = 0;
|
|
|
|
/** @param {number} sec */
|
|
function startJoinCountdown(sec) {
|
|
stopJoinCountdown();
|
|
let left = Math.max(0, Math.floor(sec));
|
|
const paint = () => {
|
|
joinQueueBtn.textContent = left > 0 ? `Join now (${left}s)` : "Join now";
|
|
};
|
|
paint();
|
|
joinCountdownTimer = window.setInterval(() => {
|
|
left -= 1;
|
|
if (left <= 0) {
|
|
stopJoinCountdown();
|
|
joinQueueBtn.textContent = "Join now";
|
|
return;
|
|
}
|
|
paint();
|
|
}, 1000);
|
|
}
|
|
|
|
function stopJoinCountdown() {
|
|
if (joinCountdownTimer) {
|
|
clearInterval(joinCountdownTimer);
|
|
joinCountdownTimer = 0;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start a conversation. Pass a pre-created AudioContext when the caller already
|
|
* made one inside the tap/click gesture (required on iOS); otherwise one is
|
|
* created here, which is still inside the gesture for a direct orb tap.
|
|
* @param {AudioContext | null} [audioContext]
|
|
*/
|
|
async function doStart(audioContext = null) {
|
|
// Resolve the target before touching mic/audio so a misconfiguration (e.g.
|
|
// direct mode with no URL) fails fast with a clear message.
|
|
const target = connectionTarget();
|
|
|
|
chat.clear();
|
|
chat.reset();
|
|
setState("connecting");
|
|
setCaption("Asking for mic…", "muted");
|
|
|
|
// Create + resume the AudioContext SYNCHRONOUSLY, still inside the gesture.
|
|
// iOS Safari only starts an AudioContext from a user gesture; if we waited
|
|
// until after the getUserMedia / session-creation awaits below, it would stay
|
|
// suspended and the whole pipeline would be silent.
|
|
if (!audioContext) audioContext = createResumedAudioContext();
|
|
|
|
// Prime the mic permission now (get the prompt out of the way up front), then
|
|
// release it. The real capture stream is acquired only once a slot is granted
|
|
// (see acquireMicStream), so the mic 'in use' indicator never lights while we
|
|
// sit in the queue. Permission persists, so the later acquire is silent.
|
|
try {
|
|
await primeMicPermission();
|
|
} catch (err) {
|
|
if (audioContext) void audioContext.close().catch(() => {});
|
|
throw err;
|
|
}
|
|
|
|
const startInstructions = effectiveInstructions();
|
|
const startTools = activeToolDefs();
|
|
// #region debug-point B:client-construction
|
|
reportDebug("B", "main.js:doStart", "constructing realtime client", {
|
|
toolNames: startTools.map((tool) => tool.name),
|
|
toolCount: startTools.length,
|
|
instructionLength: startInstructions.length,
|
|
searchMcpEnabled,
|
|
});
|
|
// #endregion
|
|
const c = new S2sWsRealtimeClient({
|
|
...target,
|
|
voice: settings.voice,
|
|
instructions: startInstructions,
|
|
acquireMic: acquireMicStream,
|
|
tools: startTools,
|
|
noiseGate: gateParams(settings.noiseGate),
|
|
...(audioContext ? { audioContext } : {}),
|
|
});
|
|
client = c;
|
|
liveInstructionsSnapshot = startInstructions;
|
|
|
|
c.addEventListener("queue", (e) => {
|
|
const { position, queueId } = /** @type {CustomEvent<{ position: number; queueId: string }>} */ (e).detail;
|
|
if (queueId) queuedTicketId = queueId;
|
|
onQueuePosition(position);
|
|
});
|
|
|
|
c.addEventListener("ready-to-join", (e) => {
|
|
const { info, expiresSec } = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo; expiresSec: number }>} */ (e).detail;
|
|
// A slot is held for us. We're out of the queue now, so drop the ticket ref.
|
|
// Track the granted session id already so that leaving (or letting the timer
|
|
// lapse) refunds the budget the server reserved at claim, even before we dial.
|
|
queuedTicketId = "";
|
|
if (info?.sessionId) {
|
|
trackedSessionId = info.sessionId;
|
|
trackedTier = info.tier || "anon";
|
|
}
|
|
startJoinCountdown(expiresSec);
|
|
});
|
|
|
|
c.addEventListener("status", (e) => {
|
|
const detail = /** @type {CustomEvent<{ status: string }>} */ (e).detail;
|
|
_kickIdleTimer(`client-status:${detail.status}`);
|
|
onClientStatus(detail.status);
|
|
});
|
|
c.addEventListener("transcript", (e) => {
|
|
const d = /** @type {CustomEvent<{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }>} */ (e).detail;
|
|
_kickIdleTimer(`transcript-${d.role}${d.partial ? ":partial" : ""}`);
|
|
chat.onTranscript(d);
|
|
if (d.role === "user") {
|
|
updateMcpToolSuggestions(d.text || "", { partial: d.partial, itemId: d.itemId });
|
|
if (!d.partial) void maybeRunOpenAiReasoning(d.text || "", d.itemId || "");
|
|
}
|
|
if (d.role === "assistant" && !d.partial) maybeRunDeclaredAssistantTool(d.text || "", d.responseId || "");
|
|
});
|
|
|
|
c.addEventListener("response-finished", (e) => {
|
|
const detail = /** @type {CustomEvent<{ responseId: string; status: string; audible?: boolean; transcript?: string }>} */ (e).detail;
|
|
_kickIdleTimer(`response-finished:${detail.status || "done"}`);
|
|
chat.onResponseFinished(detail);
|
|
maybeRunDeclaredAssistantTool(detail.transcript || "", detail.responseId || "");
|
|
});
|
|
|
|
c.addEventListener("toolcall", (e) => {
|
|
const { name, arguments: args, callId } = /** @type {CustomEvent<{ name: string; arguments: string; callId: string }>} */ (e).detail;
|
|
_kickIdleTimer(`toolcall:${name}`);
|
|
clearPendingAutoMcpFallback();
|
|
// #region debug-point D:toolcall-event
|
|
reportDebug("D", "main.js:client.toolcall", "toolcall event received by main app", {
|
|
name,
|
|
callId,
|
|
args,
|
|
knownMcpTool: agentToolDefs.some((tool) => tool.name === name),
|
|
});
|
|
// #endregion
|
|
if (agentToolDefs.some((tool) => tool.name === name)) {
|
|
console.debug(`[Vocero MCP CALL] name=${name} callId=${JSON.stringify(callId)} args=${args}`);
|
|
}
|
|
chat.onToolCall(name);
|
|
// Execute the tool, then push it to the conversation once the result is in,
|
|
// so the toggle shows both the call input and its output together.
|
|
void runTool(name, args, callId).then(({ output, image }) => {
|
|
_kickIdleTimer(`tool-result:${name}`);
|
|
chat.onToolResult(name, args, output, image);
|
|
});
|
|
});
|
|
c.addEventListener("error", (e) => {
|
|
const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail;
|
|
onFatalError(detail.error);
|
|
});
|
|
c.addEventListener("server-error", (e) => {
|
|
// Non-fatal: the backend reported an error mid-session. Log it, keep the
|
|
// socket and the conversation alive (the model can recover on its own).
|
|
const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail;
|
|
const msg = detail.error instanceof Error ? detail.error.message : String(detail.error);
|
|
console.warn("[main] server error (non-fatal):", msg);
|
|
});
|
|
c.addEventListener("session", (e) => {
|
|
const info = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo }>} */ (e).detail.info;
|
|
console.log("[ws] session created:", info.sessionId);
|
|
// A slot was granted — we're out of the queue; drop the ticket reference so
|
|
// teardown doesn't try to leave a line we already left.
|
|
queuedTicketId = "";
|
|
// A metered tier (anon / free): heartbeat so the server can extend the
|
|
// reservation and tell us when the daily budget runs out. PRO isn't limited.
|
|
if (info.limited && info.sessionId) {
|
|
trackedSessionId = info.sessionId;
|
|
trackedTier = info.tier || "anon";
|
|
startHeartbeat(info.heartbeatSec || 5);
|
|
}
|
|
});
|
|
c.addEventListener("input-level", (e) => {
|
|
const { rms } = /** @type {CustomEvent<{ rms: number }>} */ (e).detail;
|
|
paintInputLevel(rms);
|
|
// Proxy for "user is speaking" before VAD/STT emits the first transcript
|
|
// delta. RMS > ~-52 dBFS (≈ 0.0025 linear) consistently means actual
|
|
// speech or close enough to a real utterance that the 10s idle window
|
|
// should restart. Threshold is generous (≈ GATE_DEFAULT_DB - 2dB) so even
|
|
// quiet speakers keep the session alive; linear fast path avoids log10
|
|
// on the hot 50 Hz level path.
|
|
if (rms > 0.0025) _kickIdleTimer("input-level");
|
|
});
|
|
|
|
try {
|
|
await c.connect();
|
|
} catch (err) {
|
|
// The grant can be refused (402 → limit) or the dial can fail. In LB mode
|
|
// the AudioContext hasn't been adopted by the client yet (the session POST
|
|
// runs first), so close the one we created here to avoid leaking it.
|
|
if (audioContext) void audioContext.close().catch(() => {});
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── Conversation-time heartbeat ─────────────────────────────────────────────
|
|
|
|
/** Ping the server every `sec` seconds so it can meter the live session; when
|
|
* it reports the daily budget is spent, cut the call and show the limit modal.
|
|
* @param {number} sec */
|
|
function startHeartbeat(sec) {
|
|
stopHeartbeat();
|
|
heartbeatTimer = window.setInterval(async () => {
|
|
if (!trackedSessionId) return;
|
|
try {
|
|
const res = await fetch("api/session/heartbeat", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ sessionId: trackedSessionId }),
|
|
keepalive: true,
|
|
});
|
|
const json = await res.json().catch(() => ({}));
|
|
if (json.expired) await onLimitReached();
|
|
} catch (err) {
|
|
// A transient network blip shouldn't kill the call; the next tick retries.
|
|
if (DEBUG) console.debug("[ui] heartbeat failed:", err);
|
|
}
|
|
}, Math.max(1, sec) * 1000);
|
|
}
|
|
|
|
function stopHeartbeat() {
|
|
if (heartbeatTimer) {
|
|
clearInterval(heartbeatTimer);
|
|
heartbeatTimer = 0;
|
|
}
|
|
}
|
|
|
|
/** The server cut the live session: tear down and explain why. */
|
|
async function onLimitReached() {
|
|
const tier = trackedTier;
|
|
stopHeartbeat();
|
|
await teardown();
|
|
account.showLimit(tier);
|
|
}
|
|
|
|
/** Tell the server a session ended so it reconciles + refunds the unused chunk.
|
|
* Uses sendBeacon so it still fires when the tab is closing. */
|
|
function endTrackedSession() {
|
|
if (!trackedSessionId) return;
|
|
const body = JSON.stringify({ sessionId: trackedSessionId });
|
|
try {
|
|
const blob = new Blob([body], { type: "application/json" });
|
|
if (!navigator.sendBeacon("api/session/end", blob)) {
|
|
void fetch("api/session/end", {
|
|
method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true,
|
|
}).catch(() => {});
|
|
}
|
|
} catch {
|
|
// Best-effort; the server sweep reaps the session anyway.
|
|
}
|
|
trackedSessionId = "";
|
|
trackedTier = "";
|
|
}
|
|
|
|
/** Leave the waiting queue so the LB frees our place. sendBeacon so it still
|
|
* fires on tab close; the LB also reaps the ticket on TTL as a backstop. */
|
|
function endQueueTicket() {
|
|
if (!queuedTicketId) return;
|
|
const body = JSON.stringify({ queueId: queuedTicketId });
|
|
try {
|
|
const blob = new Blob([body], { type: "application/json" });
|
|
if (!navigator.sendBeacon("api/queue/end", blob)) {
|
|
void fetch("api/queue/end", {
|
|
method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true,
|
|
}).catch(() => {});
|
|
}
|
|
} catch {
|
|
// Best-effort; the LB reaps the ticket on TTL anyway.
|
|
}
|
|
queuedTicketId = "";
|
|
}
|
|
|
|
/** @param {string} status */
|
|
function onClientStatus(status) {
|
|
switch (status) {
|
|
case "creating-session":
|
|
case "connecting":
|
|
setState("connecting");
|
|
break;
|
|
case "queued":
|
|
setState("queued");
|
|
break;
|
|
case "your-turn":
|
|
setState("your-turn");
|
|
break;
|
|
case "connected":
|
|
setState("listening");
|
|
// Wake word confirmation: if this session was triggered by the "Hola
|
|
// Vocero" hot-word, inject the phrase into the conversation exactly as
|
|
// though the user had spoken it, then ask the model to reply so the
|
|
// assistant greets back instead of silently waiting for the next turn.
|
|
if (pendingWakeWordConfirmationText && client) {
|
|
const phrase = pendingWakeWordConfirmationText.trim() || "hola vocero";
|
|
pendingWakeWordConfirmationText = "";
|
|
console.info(`[WAKE WORD] 📤 Injecting confirmation phrase to ai-server: ${JSON.stringify(phrase)}`);
|
|
try {
|
|
client.sendUserText(phrase);
|
|
client.requestResponse();
|
|
} catch (err) {
|
|
console.warn("[WAKE WORD] failed to inject confirmation phrase:", err && err instanceof Error ? err.message : err);
|
|
}
|
|
}
|
|
break;
|
|
case "user-speaking":
|
|
setState("user-speaking");
|
|
break;
|
|
case "processing":
|
|
setState("processing");
|
|
break;
|
|
case "ai-speaking":
|
|
setState("ai-speaking");
|
|
break;
|
|
case "closed":
|
|
// teardown() will move us to idle
|
|
break;
|
|
case "error":
|
|
setState("error");
|
|
break;
|
|
}
|
|
}
|
|
|
|
async function teardown() {
|
|
clearPendingAutoMcpFallback();
|
|
stopHeartbeat();
|
|
stopJoinCountdown();
|
|
endTrackedSession();
|
|
endQueueTicket();
|
|
chat.reset({ dismiss: true });
|
|
updateMcpToolSuggestions("");
|
|
liveInstructionsSnapshot = "";
|
|
liveToolPolicySnapshot = "";
|
|
// If teardown fires *before* the first "connected" status (e.g. failed dial,
|
|
// user left queue, limit exceeded) the pending wake-word confirmation would
|
|
// be re-injected on the next session. Clear it so we never double-fire.
|
|
pendingWakeWordConfirmationText = "";
|
|
if (client) {
|
|
try {
|
|
await client.close();
|
|
} catch (err) {
|
|
console.warn("[main] error closing client:", err);
|
|
}
|
|
client = null;
|
|
}
|
|
if (micStream) {
|
|
for (const track of micStream.getTracks()) track.stop();
|
|
micStream = null;
|
|
}
|
|
micMuted = false;
|
|
micBtn.classList.remove("muted");
|
|
setState("idle");
|
|
// Refresh the chip's remaining-today after the budget moved.
|
|
if (limiterOn) void account.refresh();
|
|
}
|
|
|
|
/** @param {unknown} err */
|
|
function onFatalError(err) {
|
|
console.error("[main] fatal:", err);
|
|
setState("error");
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
setCaption(truncateError(message), "error");
|
|
void teardown().catch(() => {
|
|
setState("error");
|
|
setCaption(truncateError(message), "error");
|
|
});
|
|
}
|
|
|
|
setState("idle");
|
|
chat.renderEmptyState();
|
|
initGateArc();
|
|
void fetchConfig();
|
|
void fetchVoceroUserProfile();
|
|
(() => {
|
|
const toolCamRow = document.getElementById("tool-cam-row");
|
|
if (toolCamRow) toolCamRow.setAttribute("hidden", "");
|
|
if (toolCamHint) toolCamHint.setAttribute("hidden", "");
|
|
if (camPip) camPip.setAttribute("hidden", "");
|
|
if (camVideo) camVideo.setAttribute("hidden", "");
|
|
})();
|
|
|
|
// Reconcile a live session if the tab is closed/hidden mid-call (no teardown).
|
|
window.addEventListener("pagehide", () => { endTrackedSession(); endQueueTicket(); });
|
|
|
|
requestAnimationFrame(() => {
|
|
document.body.classList.remove("booting");
|
|
});
|