// @ts-check /** * Minimal voice conversation app, talking to a Hugging Face speech-to-speech * backend over **WebSocket** (drop-in alternative to the WebRTC variant). * * Click the orb -> we ask for the mic, POST a session on the LB, open a * WebSocket on the routed compute endpoint, push session.update + mic * audio, play back the TTS audio. The orb visually reflects the live * state (idle, connecting, listening, user-speaking, processing, * ai-speaking). * * The only meaningful difference vs. the WebRTC main.js is that the * client owns its own AudioContext (no `attachOutputTrack`), so we hand * it the MediaStream directly. * * @typedef {"idle" | "connecting" | "queued" | "your-turn" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState */ import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js?v=20260826001"; import { $, escHtml, truncateError, DEBUG } from "./ui/dom.js?v=20260826001"; import { ChatView } from "./ui/chat.js?v=20260826001"; import { Account } from "./ui/account.js?v=20260826001"; const DEFAULT_VOICE = "Aiden"; const DEFAULT_INSTRUCTIONS = "You are a friendly voice assistant. " + "Keep replies short, warm, and spoken. Avoid long monologues."; // Appended to the user's instructions whenever at least one tool is enabled. // Stops the model from announcing capabilities ("Yes, I can search") and then // idling for the next turn — it should act immediately in the same response. const TOOL_USE_HINT = " When the user's request calls for one of your tools, do not describe your " + "capabilities or say you can do it and wait for another turn. Instead, say " + 'a brief acknowledgement like "Let me search for that..." and call the tool ' + "right away in the same response."; const MCP_SEARCH_HINT = " When the user asks for information, review the full MCP tool catalog, choose " + "the single best tool for the request, and call it immediately when it can solve " + "the task. If required parameters are missing, ask a short follow-up question. " + "If no MCP tool fits, answer normally or use the other available tools."; const OPENAI_RENDER_MARKER = "[VOCERO_RENDER_REPLY]"; const OPENAI_RENDER_HINT = ` Do not answer ordinary user speech directly. Stay silent unless the latest user message starts with ` + `${OPENAI_RENDER_MARKER}. When it does, speak only the content after the marker naturally, without adding, ` + `removing, or changing facts. Do not call tools while rendering that message.`; const VOCERO_USER_CONTEXT_HINT = " Caller profile resolved from the page phoneNumber: "; /** @type {[RegExp, string][]} */ const MCP_SEARCH_TRANSLATION_PHRASES = [ [/informaciones del usuario/g, "user information"], [/informacion del usuario/g, "user information"], [/informazioni utente/g, "user information"], [/informazione utente/g, "user information"], [/report del dia/g, "day report"], [/reporte del dia/g, "day report"], [/report di oggi/g, "today report"], [/reporte de hoy/g, "today report"], [/report giornaliero/g, "daily report"], [/reporte diario/g, "daily report"], [/reporte de ayer/g, "yesterday report"], [/reporte del día anterior/g, "previous day report"], [/report di ieri/g, "yesterday report"], [/report del giorno prima/g, "previous day report"], [/último reporte/g, "latest report"], [/ultimo reporte/g, "latest report"], [/ultimo report/g, "latest report"], [/report più recente/g, "most recent report"], [/reporte más reciente/g, "most recent report"], [/reporte mas reciente/g, "most recent report"], [/más reciente/g, "most recent"], [/mas reciente/g, "most recent"], [/più recente/g, "most recent"], [/listado de reportes/g, "list reports report list"], [/lista di report/g, "list reports report list"], [/reportes recientes/g, "recent reports list"], [/report recenti/g, "recent reports list"], [/mi numero es/g, "my number is"], [/il mio numero e/g, "my number is"], [/numero di telefono/g, "phone number"], [/numero telefonico/g, "phone number"], [/numero de telefono/g, "phone number"], [/numero de whatsapp/g, "whatsapp number"], [/informacion api/g, "api information"], [/informazioni api/g, "api information"], ]; /** @type {Record} */ const MCP_SEARCH_TRANSLATION_TOKENS = { ayuda: "help", api: "api", ayer: "yesterday", ieri: "yesterday", anterior: "previous", precedente: "previous", dami: "give", dame: "give", dammi: "give", buscar: "search", busca: "search", busqueda: "search", call: "call", catalogo: "catalogue", catalogue: "catalogue", cerca: "search", chiama: "call", cliente: "customer", clientes: "customers", customer: "customer", detalle: "detail", detalles: "details", dettagli: "details", dia: "day", diario: "daily", dio: "day", documento: "document", documentos: "documents", elenco: "list", email: "email", estado: "status", estadoo: "status", historial: "history", hoy: "today", informacion: "information", informazioni: "information", informe: "report", informes: "reports", lista: "list", listado: "list", liste: "lists", mostrar: "show", mostra: "show", numero: "number", numeros: "numbers", obtener: "get", ottieni: "get", perfil: "profile", profilo: "profile", reciente: "recent", recientes: "recent", recente: "recent", recenti: "recent", reporte: "report", reportes: "reports", reporto: "report", stato: "status", telefono: "phone", telefonoo: "phone", usuario: "user", utenti: "users", utente: "user", ultimo: "latest", ultimi: "latest", último: "latest", últimos: "latest", whatsapp: "whatsapp", }; const STORAGE_KEYS = { directUrl: "s2s.ws.directUrl", voice: "s2s.ws.voice", instructions: "s2s.ws.instructions", tools: "s2s.ws.tools", searchKey: "s2s.ws.searchKey", noiseGate: "s2s.ws.noiseGate", }; // ── Wake Word ("hola vocero") ───────────────────────────────────────────── // Local browser Web Speech API (SpeechRecognition) with a tight JSGF grammar // restricted to the trigger phrase, continuous/interim listening while the app // is idle/error, and auto-restart on end so it keeps arming across Chrome's // ~60s silence timeouts. On unsupported browsers (Firefox, Safari desktop <16) // this module is a silent no-op and the caption stays actionable via tap only. const WAKE_WORD_LANG = "es-ES"; const WAKE_WORD_PHRASES = [ "hola vocero", "ola vocero", "hola vocero", "hola vocerò", "ola vocerò", "olá vocero", "oye vocero", "hola vocerro", "hola vo zero", "ola vo zero", ]; const WAKE_WORD_JSGF_GRAMMAR = "#JSGF V1.0; grammar wake_word; public = hola vocero | ola vocero | hola vocerò | ola vocerò | olá vocero | oye vocero | hola vocerro ;"; const WAKE_WORD_RESTART_DELAY_MS = 250; const WAKE_WORD_MAX_INTERIM_LEN = 64; /** @typedef {any & { continuous: boolean, interimResults: boolean, lang: string, grammars: any, maxAlternatives: number, start: () => void, stop: () => void, abort: () => void, onresult: any, onerror: any, onend: any, onstart: any, onaudioend: any }} WakeRec */ /** @type {WakeRec | null} */ let wakeWordEngine = null; let wakeWordArmed = false; let wakeWordRestartTimer = 0; let pendingWakeWordConfirmationText = ""; /** @param {string} s @returns {string} */ function _normalizeWakeWord(s) { return (s || "") .toLowerCase() .normalize("NFD") .replace(/[\u0300-\u036f]/g, "") .replace(/[^a-z0-9\s]/g, " ") .replace(/\s+/g, " ") .trim(); } /** @param {string} transcript */ function _matchesWakeWord(transcript) { const hay = _normalizeWakeWord(transcript); if (!hay) return false; // Strict: contains a whole-phrase substring match of any variant. for (const raw of WAKE_WORD_PHRASES) { const needle = _normalizeWakeWord(raw); if (needle && hay.includes(needle)) return true; } // Loose fallback for off-by-one phonemes: allow "vocero" preceded by any // 3-5 char greeting-like word (e.g. "ola", "hola", "oye", "uy") to catch // accent variations the grammar misses. if (/\b(vocero|vocerro|vocerò|vo.?zero)\b/.test(hay) && /\b(hola|ola|oye|uy|holi|oli)\b/.test(hay)) { return true; } return false; } /** @param {WakeRec} engine */ function _installWakeWordHandlers(engine) { engine.onresult = (/** @type {any} */ event) => { if (!wakeWordArmed) return; let aggregate = ""; for (let i = event.resultIndex; i < event.results.length; i += 1) { const res = event.results[i]; const alt = res[0]?.transcript || ""; aggregate += ` ${alt}`; if (aggregate.length > WAKE_WORD_MAX_INTERIM_LEN) break; if (_matchesWakeWord(alt) || _matchesWakeWord(aggregate)) { if (DEBUG) console.debug(`[WAKE WORD] MATCH transcript=${JSON.stringify(alt + (aggregate !== ` ${alt}` ? ` aggregate=${JSON.stringify(aggregate)}` : ""))}`); console.info(`[WAKE WORD] ✓ Trigger fired — starting conversation`); // Disarm *before* click: stop sharing the mic with SpeechRecognition // while we're about to open getUserMedia for the realtime session. wakeWordArmed = false; try { engine.stop(); } catch { /* ignore */ } if (wakeWordRestartTimer) { clearTimeout(wakeWordRestartTimer); wakeWordRestartTimer = 0; } // Remember the exact phrase so onClientStatus(connected) can inject it // into the live conversation and ask the model to reply to it (instead // of silently waiting for the next user utterance). const spoken = (alt || "").trim() || (aggregate || "").trim() || "hola vocero"; pendingWakeWordConfirmationText = spoken; console.info(`[WAKE WORD] 🔁 Pending confirmation text: ${JSON.stringify(pendingWakeWordConfirmationText)}`); // Pretend the user tapped the orb. If the AudioContext fails to // resume (iOS Safari needs a real tap), doStart() will surface the // error with the normal "Tap to retry" affordance. circleBtn.click(); return; } } if (DEBUG) { const cleaned = aggregate.trim(); if (cleaned) console.debug(`[WAKE WORD] interim transcript=${JSON.stringify(cleaned.slice(0, WAKE_WORD_MAX_INTERIM_LEN))}`); } }; engine.onerror = (/** @type {any} */ event) => { const err = event?.error || String(event || "unknown"); // "no-speech" / "audio-capture" / "aborted" are benign. if (["no-speech", "aborted"].includes(err)) return; console.warn(`[WAKE WORD] engine error: ${err}${event?.message ? ` (${event.message})` : ""}`); // "not-allowed" or "service-not-allowed": the user/OS blocked the mic for // the recognizer — permanently disarm so we don't thrash the permission. if (err === "not-allowed" || err === "service-not-allowed") { console.warn("[WAKE WORD] ✋ Permission denied — permanently disarming wake word (tap the orb to start)"); wakeWordArmed = false; try { engine.stop(); } catch { /* ignore */ } if (wakeWordRestartTimer) { clearTimeout(wakeWordRestartTimer); wakeWordRestartTimer = 0; } } }; engine.onend = () => { if (!wakeWordArmed) return; // Chrome terminates continuous recognition after ~60s of silence. // Re-arm on the next tick so a single failed start doesn't loop. if (wakeWordRestartTimer) clearTimeout(wakeWordRestartTimer); wakeWordRestartTimer = window.setTimeout(() => { if (!wakeWordArmed) return; try { engine.start(); if (DEBUG) console.debug("[WAKE WORD] 🔁 restarted continuous listener"); } catch (err) { // Already-started is harmless; anything else we swallow briefly. if (DEBUG) console.debug("[WAKE WORD] restart skipped:", err && err instanceof Error ? err.message : err); } }, WAKE_WORD_RESTART_DELAY_MS); }; } function startWakeWordListener() { if (wakeWordArmed) return; // Only in Chromium-derived and recent Safari WebViews with SpeechRecognition. // @ts-ignore - vendor prefixes are intentional. const Ctor = window.SpeechRecognition || window.webkitSpeechRecognition || window.mozSpeechRecognition; if (!Ctor) { if (DEBUG) console.debug("[WAKE WORD] ⚠️ Web Speech Recognition unavailable on this browser — tap the orb to start"); return; } try { /** @type {WakeRec} */ // @ts-ignore const engine = new Ctor(); engine.continuous = true; engine.interimResults = true; engine.lang = WAKE_WORD_LANG; engine.maxAlternatives = 1; // Narrow the grammar so interim results are biased toward the phrase. try { // @ts-ignore const list = window.SpeechGrammarList || window.webkitSpeechGrammarList; if (list) { // @ts-ignore const gl = new list(); gl.addFromString(WAKE_WORD_JSGF_GRAMMAR, 1.0); engine.grammars = gl; } } catch { // Grammar support is optional; fallback to pure transcript matching. } _installWakeWordHandlers(engine); engine.start(); wakeWordEngine = engine; wakeWordArmed = true; console.info("[WAKE WORD] 🎯 Armed — listening for \"Hola Vocero\""); } catch (err) { console.warn("[WAKE WORD] failed to start arming:", err && err instanceof Error ? err.message : err); wakeWordArmed = false; } } function stopWakeWordListener() { if (!wakeWordArmed && !wakeWordEngine && !pendingWakeWordConfirmationText) return; wakeWordArmed = false; if (wakeWordRestartTimer) { clearTimeout(wakeWordRestartTimer); wakeWordRestartTimer = 0; } if (wakeWordEngine) { try { wakeWordEngine.abort(); } catch { /* ignore */ } try { wakeWordEngine.stop(); } catch { /* ignore */ } wakeWordEngine = null; } if (DEBUG) console.debug("[WAKE WORD] ⏸️ Disarmed for live session"); } // ── Noise gate ────────────────────────────────────────────────────────────── // The Settings cursor sets the gate's open threshold in dBFS. Its leftmost // position is an OFF detent (gate disabled, pure passthrough); the rest of the // travel is the active threshold. The cursor shares the meter's dB axis, so the // handle sits on the level bar — raise it until room noise stops lighting it up. // The slider range IS the shared axis: the live meter fill and the threshold // thumb both map across [GATE_OFF_DB, GATE_MAX_DB], so the thumb sits exactly // where the gate cuts on the same scale as the level bar. const GATE_OFF_DB = -66; // slider minimum = off / bottom of the meter axis const GATE_MAX_DB = -3; // slider maximum = most aggressive / top of the meter axis const GATE_DEFAULT_DB = -50; // first-run default: a gentle gate, enabled /** @param {number} thresholdDb @returns {import("./ws/s2s-ws-client.js").NoiseGate} */ function gateParams(thresholdDb) { return { enabled: thresholdDb > GATE_OFF_DB, thresholdDb }; } // ── Tools ───────────────────────────────────────────────────────────────── // Function tools we declare to the backend. The model decides when to call // one; the executor below runs it and returns the result (see runTool). /** @type {Record} */ const TOOL_DEFS = { web_search: { type: "function", name: "web_search", description: "Search the web for current or factual information you don't already know " + "(news, prices, facts, documentation). Returns the top results with titles, " + "snippets and URLs.", parameters: { type: "object", properties: { query: { type: "string", description: "The search query." } }, required: ["query"], }, }, }; /** @type {import("./ws/s2s-ws-client.js").ToolDef[]} */ let agentToolDefs = []; function loadSettings() { return { directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "", voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE, instructions: localStorage.getItem(STORAGE_KEYS.instructions) || DEFAULT_INSTRUCTIONS, noiseGate: loadGateThreshold(), }; } /** Stored gate threshold (dBFS), clamped to the slider range. Defaults to a * gentle enabled gate (GATE_DEFAULT_DB) when the user hasn't set one yet. */ function loadGateThreshold() { const stored = localStorage.getItem(STORAGE_KEYS.noiseGate); // getItem returns null when unset, and Number(null) === 0 (finite!), so guard // the missing/empty case explicitly before coercing — otherwise the default // never fires and 0 clamps to the slider max. if (stored === null || stored === "") return GATE_DEFAULT_DB; const raw = Number(stored); if (!Number.isFinite(raw)) return GATE_DEFAULT_DB; return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(raw))); } /** @param {ReturnType} s */ function saveSettings(s) { localStorage.setItem(STORAGE_KEYS.directUrl, s.directUrl); localStorage.setItem(STORAGE_KEYS.voice, s.voice); localStorage.setItem(STORAGE_KEYS.instructions, s.instructions); localStorage.setItem(STORAGE_KEYS.noiseGate, String(s.noiseGate)); } /** @returns {{ web_search: boolean }} */ function loadTools() { try { const raw = JSON.parse(localStorage.getItem(STORAGE_KEYS.tools) || "{}"); return { web_search: raw.web_search ?? true, }; } catch { return { web_search: true }; } } function saveTools() { localStorage.setItem(STORAGE_KEYS.tools, JSON.stringify(toolsEnabled)); } /** @type {Record} */ const STATE_VIEWS = { idle: { caption: "Tap or say \"Hola Vocero\"", disabled: false }, connecting: { caption: "Connecting", disabled: true }, queued: { caption: "Finding you a spot…", disabled: true }, "your-turn": { caption: "You're up! 🎉", disabled: true }, listening: { caption: "", disabled: false }, "user-speaking": { caption: "", disabled: false }, processing: { caption: "", disabled: false }, "ai-speaking": { caption: "", disabled: false }, error: { caption: "Tap to retry", disabled: false }, }; /** @type {Record} */ const STATE_CLASS = { idle: "state-idle", connecting: "state-connecting", queued: "state-queued", "your-turn": "state-your-turn", listening: "state-listening", "user-speaking": "state-user-speaking", processing: "state-processing", "ai-speaking": "state-ai-speaking", error: "state-error", }; /** @type {ReadonlySet} */ const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]); /** @type {HTMLButtonElement} */ const circleBtn = $("#main-circle"); /** @type {HTMLParagraphElement} */ const circleCaption = $("#circle-caption"); /** @type {HTMLParagraphElement} */ const circleSubcaption = $("#circle-subcaption"); /** @type {HTMLElement} */ const orbWrap = $(".orb-wrap"); /** @type {HTMLButtonElement} */ const micBtn = $("#mic-btn"); /** @type {HTMLButtonElement} */ const stopBtn = $("#stop-btn"); /** @type {HTMLElement} */ const queueActions = $("#queue-actions"); /** @type {HTMLButtonElement} */ const joinQueueBtn = $("#join-queue-btn"); /** @type {HTMLButtonElement} */ const leaveQueueBtn = $("#leave-queue-btn"); /** @type {HTMLButtonElement} */ const settingsBtn = $("#settings-btn"); /** @type {HTMLDialogElement} */ const settingsModal = $("#settings-modal"); /** @type {HTMLButtonElement | null} */ const aboutBtn = document.querySelector("#about-btn"); /** @type {HTMLDialogElement} */ const aboutModal = $("#about-modal"); /** @type {HTMLButtonElement} */ const aboutClose = $("#about-close"); /** @type {HTMLButtonElement} */ const toolsBtn = $("#tools-btn"); /** @type {HTMLDialogElement} */ const toolsModal = $("#tools-modal"); /** @type {HTMLButtonElement} */ const toolsClose = $("#tools-close"); /** @type {HTMLInputElement} */ const toolWebSwitch = $("#tool-web"); /** @type {HTMLElement} */ const toolWebRow = $("#tool-web-row"); /** @type {HTMLElement} */ const toolWebHint = $("#tool-web-hint"); /** @type {HTMLInputElement} */ const toolCamSwitch = $("#tool-cam"); /** @type {HTMLElement} */ const toolCamHint = $("#tool-cam-hint"); /** @type {HTMLElement} */ const camPip = $("#cam-pip"); /** @type {HTMLVideoElement} */ const camVideo = $("#cam-video"); /** @type {HTMLElement} */ const agentToolsHint = $("#agent-tools-hint"); /** @type {HTMLElement} */ const agentToolsList = $("#agent-tools-list"); /** @type {HTMLInputElement} */ const searchKeyInput = $("#search-key"); /** @type {HTMLInputElement} */ const inputLbUrl = $("#lb-url"); /** @type {HTMLElement} */ const connField = $("#conn-field"); /** @type {HTMLElement} */ const connHint = $("#conn-hint"); /** @type {HTMLSelectElement} */ const inputVoice = $("#voice"); /** @type {HTMLTextAreaElement} */ const inputInstructions = $("#instructions"); /** @type {HTMLElement} */ const primaryPromptMeta = $("#primary-prompt-meta"); /** @type {HTMLElement} */ const identUserName = $("#ident-user-name"); /** @type {HTMLInputElement} */ const inputNoiseGate = $("#noise-gate"); /** @type {HTMLElement} */ const gateValue = $("#gate-value"); /** @type {HTMLElement} */ const gateMeterFill = $("#gate-meter-fill"); /** @type {HTMLElement} */ const micGate = $("#mic-gate"); const mgaArc = /** @type {SVGSVGElement} */ (document.querySelector("#mic-gate-arc")); const mgaTrack = /** @type {SVGPathElement} */ (document.querySelector("#mga-track")); const mgaFill = /** @type {SVGPathElement} */ (document.querySelector("#mga-fill")); const mgaHit = /** @type {SVGPathElement} */ (document.querySelector("#mga-hit")); const mgaHandle = /** @type {SVGCircleElement} */ (document.querySelector("#mga-handle")); /** @type {HTMLButtonElement} */ const restartBtn = $("#restart-conversation"); /** @type {HTMLElement} */ const restartHint = $("#restart-hint"); const settingsForm = /** @type {HTMLFormElement} */ (settingsModal.querySelector("form")); /** @type {AppState} */ let currentState = "idle"; let settings = loadSettings(); // ── Connection target ──────────────────────────────────────────────────────── // Three modes, decided by the deploy via /api/config: // • SPEECH_TO_SPEECH_URL set -> direct mode pinned by the deploy: the browser // connects straight to that URL, shown read-only in Settings. Overrides the // load balancer entirely. // • LOAD_BALANCER_URL set -> original flow: POST the same-origin /api/session // proxy (the server forwards to the LB; the LB address is never sent here). // • neither (allowDirect) -> the user sets a speech-to-speech server URL and // the browser connects to it directly (no load balancer, no /session). let lbMode = false; let reasoningType = "PROXY"; // Fail open: direct entry is allowed unless /api/config reports an LB URL. This // way a missing/unreachable config (e.g. static hosting) leaves the field // usable rather than locked. let allowDirect = true; // Deploy-pinned s2s URL (SPEECH_TO_SPEECH_URL). Non-empty -> locked direct // mode: the field displays it read-only and the saved user URL is untouched. let pinnedUrl = ""; // Deploy-pinned primary prompt (S2S_PRIMARY_PROMPT). Non-empty -> locked prompt. let pinnedPrompt = ""; let pinnedPromptSource = ""; // ── Tool state ────────────────────────────────────────────────────────────── let toolsEnabled = loadTools(); // Whether the server holds a Serper key (learned from /api/config on load). let serverSearchKey = false; // A user-supplied key (fallback when the deploy has none). localStorage only. let userSearchKey = localStorage.getItem(STORAGE_KEYS.searchKey) || ""; let searchMcpEnabled = false; let lastUserQuery = ""; /** @type {{ tool: import("./ws/s2s-ws-client.js").ToolDef, score: number }[]} */ let rankedAgentTools = []; let liveInstructionsSnapshot = ""; let liveToolPolicySnapshot = ""; /** @type {Record} */ let mcpToolCallState = Object.create(null); const pageParams = new URLSearchParams(window.location.search); const voceroUserPhoneNumber = (pageParams.get("phoneNumber") || "").trim(); /** @type {Record | null} */ let voceroUserProfile = null; let voceroUserProfileLoaded = false; const AUTO_MCP_MIN_SCORE = 20; const AUTO_MCP_MIN_SCORE_WITH_LEAD = 12; const AUTO_MCP_MIN_LEAD = 4; const AUTO_MCP_RELAXED_REPORT_SCORE = 15; const AUTO_MCP_FAST_DELAY_MS = 150; const AUTO_MCP_FALLBACK_DELAY_MS = 1200; let autoMcpInFlight = false; let lastAutoMcpTurnKey = ""; let pendingAutoMcpTimer = 0; let lastDeclaredAssistantToolKey = ""; const processedReasoningTurns = new Set(); /** * @param {string} hypothesisId * @param {string} location * @param {string} msg * @param {Record} [data] */ function reportDebug(hypothesisId, location, msg, data = {}) { void hypothesisId; void location; void msg; void data; } /** Search is usable if the server has a key or the user supplied one. */ function searchAvailable() { return serverSearchKey || !!userSearchKey; } /** Tool definitions for the currently-enabled (and usable) tools. */ function activeToolDefs() { const defs = []; if (toolsEnabled.web_search && searchAvailable()) defs.push(TOOL_DEFS.web_search); return defs.concat(agentToolDefs); } /** @returns {string} */ function voceroUserContextText() { if (!voceroUserProfileLoaded || !voceroUserProfile || !voceroUserPhoneNumber) return ""; let serialized = JSON.stringify(voceroUserProfile); if (serialized.length > 1600) serialized = `${serialized.slice(0, 1600)}...`; return `${VOCERO_USER_CONTEXT_HINT}${serialized}. Use this as trusted caller context when relevant.`; } /** * @param {Record | null} profile * @returns {string} */ function extractVoceroUserName(profile) { if (!profile) return ""; const directCandidates = [ profile.name, profile.fullName, profile.full_name, profile.displayName, profile.display_name, profile.username, ]; for (const value of directCandidates) { if (typeof value === "string" && value.trim()) return value.trim(); } const firstName = typeof profile.firstName === "string" ? profile.firstName.trim() : ""; const lastName = typeof profile.lastName === "string" ? profile.lastName.trim() : ""; if (firstName || lastName) return `${firstName} ${lastName}`.trim(); const nestedKeys = ["user", "customer", "profile", "contact"]; for (const key of nestedKeys) { const nested = profile[key]; if (nested && typeof nested === "object" && !Array.isArray(nested)) { const nestedName = extractVoceroUserName(/** @type {Record} */ (nested)); if (nestedName) return nestedName; } } return ""; } function syncVoceroHeaderUser() { const name = extractVoceroUserName(voceroUserProfile); identUserName.hidden = !name; identUserName.textContent = name ? `Utente: ${name}` : ""; } /** * @param {import("./ws/s2s-ws-client.js").ToolDef[]} tools * @returns {string} */ function buildMcpToolCatalogSummary(tools) { return tools .map((tool) => { const required = requiredToolParams(tool.parameters); const requiredText = required.length ? ` required: ${required.join(", ")}` : " required: none"; const desc = (tool.description || "No description provided.") .replace(/\s+/g, " ") .trim(); return `${tool.name} - ${desc}.${requiredText}.`; }) .join(" "); } /** Instructions used by the reasoning engine. */ function effectiveReasoningInstructions() { const base = pinnedPrompt || settings.instructions; let instructions = base; if (activeToolDefs().length) instructions += TOOL_USE_HINT; if (searchMcpEnabled && agentToolDefs.length) { instructions += MCP_SEARCH_HINT; instructions += ` Available MCP tools: ${buildMcpToolCatalogSummary(agentToolDefs)}`; } const suggested = topSuggestedAgentTools(3); if (searchMcpEnabled && suggested.length) { const lines = suggested .map((entry, index) => `${index + 1}. ${entry.tool.name}: ${entry.tool.description || "No description provided."}`) .join(" "); instructions += ` Current user request: "${lastUserQuery}". Client-side ranking hint only: ${lines}. Use this hint only as a secondary signal after reviewing the full MCP catalog.`; } const userContext = voceroUserContextText(); if (userContext) instructions += userContext; return instructions; } /** Instructions pushed to the realtime ASR/TTS backend. */ function effectiveInstructions() { const instructions = effectiveReasoningInstructions(); return reasoningType === "OPENAI" ? `${instructions}${OPENAI_RENDER_HINT}` : instructions; } /** * @param {string} answer * @returns {string} */ function buildRenderOnlyPrompt(answer) { const clean = String(answer || "").trim(); return clean ? `${OPENAI_RENDER_MARKER}\n${clean}` : ""; } /** * @returns {string} */ function preferredResponseLanguage() { const lang = (navigator.language || document.documentElement.lang || "").trim(); return lang || ""; } /** * @param {string} transcript * @returns {Promise<{ answer: string, log_only: boolean, raw: any }>} */ async function fetchOpenAiReasoningAnswer(transcript) { const res = await fetch("api/reasoning", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ transcript, instructions: effectiveReasoningInstructions(), language: preferredResponseLanguage(), user_context: voceroUserContextText(), phoneNumber: voceroUserPhoneNumber || null, }), }); if (!res.ok) { let detail = String(res.status); try { const json = await res.json(); if (json?.detail) detail = String(json.detail); } catch {} throw new Error(`reasoning error (${detail})`); } const json = await res.json(); const answer = typeof json?.answer === "string" ? json.answer.trim() : ""; const log_only = Boolean(json && json.log_only === true); // Log-only mode: può tornare answer="" — non è un errore. if (!answer && !log_only) throw new Error("reasoning returned no answer"); return { answer, log_only, raw: json }; } /** * @param {string} transcript * @param {string} itemId */ async function maybeRunOpenAiReasoning(transcript, itemId) { if (!["OPENAI", "NODE_RED"].includes(reasoningType)) return; const text = transcript.trim(); const turnKey = itemId || text; if (!text || !turnKey || processedReasoningTurns.has(turnKey)) return; processedReasoningTurns.add(turnKey); if (!client || !LIVE_STATES.has(currentState)) return; clearPendingAutoMcpFallback(); reportDebug("R", "main.js:maybeRunOpenAiReasoning", "starting external reasoning turn", { itemId, transcriptLength: text.length, currentState, }); let result; try { result = await fetchOpenAiReasoningAnswer(text); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.warn(`[Vocero REASONING] error itemId=${JSON.stringify(itemId)} error=${msg}`); reportDebug("R", "main.js:maybeRunOpenAiReasoning", "external reasoning failed", { itemId, error: msg, }); return; } // ── LOG-ONLY MODE (FASE 1) ───────────────────────────────────────────── // Node-RED si limita a loggare e torna log_only=true. // NON cancelliamo la risposta backend, NON iniettiamo testo, NON parliamo. // Il flusso resta COMPLETAMENTE a carico del backend S2S nativo. if (result && result.log_only) { reportDebug("R", "main.js:maybeRunOpenAiReasoning", "external reasoning in LOG_ONLY mode — pass through to native S2S backend", { itemId, reasoningType, intent: (result.raw && result.raw.intent) || "log_only", }); console.debug( `[Vocero REASONING LOG_ONLY] mode=log_only engine=${reasoningType} ` + `transcript=${JSON.stringify(text.slice(0, 160))} ` + `intent=${JSON.stringify((result.raw && result.raw.intent) || "log_only")}` ); return; } // ── MODE STANDARD (fasi successive) ──────────────────────────────────── // Il reasoning esterno vince: annulla la risposta nativa, inserisci il // testo dell'assistente e parla via SpeechSynthesis. const answer = (result && result.answer) || ""; if (!answer) { console.warn(`[Vocero REASONING] empty non-log-only answer itemId=${JSON.stringify(itemId)}`); return; } if (typeof client.cancelActiveResponse === "function") { client.cancelActiveResponse(); } else if (typeof client.suppressAssistantAudio === "function") { client.suppressAssistantAudio(); } reportDebug("R", "main.js:maybeRunOpenAiReasoning", "received external reasoning answer", { itemId, answerLength: answer.length, }); if (typeof client.appendAssistantText === "function") { client.appendAssistantText(answer); } else { const renderPrompt = buildRenderOnlyPrompt(answer); if (renderPrompt) { client.sendUserText(renderPrompt); client.requestResponse(); } } try { if ("speechSynthesis" in window) { const utter = new SpeechSynthesisUtterance(answer); const want = (preferredResponseLanguage() || "en").slice(0, 2).toLowerCase(); const voices = window.speechSynthesis.getVoices ? window.speechSynthesis.getVoices() : []; const match = Array.isArray(voices) && voices.length > 0 ? (voices.find(v => (v.lang || "").slice(0, 2).toLowerCase() === want) || voices[0]) : null; if (match) utter.voice = match; if (want) utter.lang = want; utter.rate = 1.0; utter.pitch = 1.0; window.speechSynthesis.cancel(); window.speechSynthesis.speak(utter); } } catch (ttsErr) { console.warn("[Vocero REASONING] speechSynthesis failed:", ttsErr); } } /** @param {string} instructions */ function pushLiveInstructions(instructions) { if (!client || !LIVE_STATES.has(currentState)) return; if (instructions === liveInstructionsSnapshot) return; liveInstructionsSnapshot = instructions; // #region debug-point C:push-live-instructions reportDebug("C", "main.js:pushLiveInstructions", "pushing computed instructions to live session", { instructionLength: instructions.length, lastUserQuery, suggestedTools: topSuggestedAgentTools(3).map((entry) => entry.tool.name), }); // #endregion client.updateSession({ instructions }); } function pushLiveToolPolicy() { if (!client || !LIVE_STATES.has(currentState)) return; const tools = activeToolDefs(); const toolChoice = tools.length ? "auto" : "none"; const snapshot = JSON.stringify({ toolChoice, toolNames: tools.map((tool) => tool.name) }); if (snapshot === liveToolPolicySnapshot) return; liveToolPolicySnapshot = snapshot; // #region debug-point B:push-live-tool-policy reportDebug("B", "main.js:pushLiveToolPolicy", "pushing live tool policy", { toolChoice, toolNames: tools.map((tool) => tool.name), lastUserQuery, searchMcpEnabled, }); // #endregion client.setTools(tools, toolChoice); } function clearPendingAutoMcpFallback() { if (!pendingAutoMcpTimer) return; window.clearTimeout(pendingAutoMcpTimer); pendingAutoMcpTimer = 0; } /** * @param {string} text * @returns {import("./ws/s2s-ws-client.js").ToolDef | undefined} */ function extractDeclaredAssistantTool(text) { const match = text.match(/<\s*calls?\s+tool\s*:\s*([a-z0-9_.-]+)\s*>/i); if (!match?.[1]) return undefined; const toolName = match[1].trim(); return findAgentToolDef(toolName); } /** * @param {string} text * @param {string} responseId */ function maybeRunDeclaredAssistantTool(text, responseId = "") { if (!client || !LIVE_STATES.has(currentState) || autoMcpInFlight) return; const tool = extractDeclaredAssistantTool(text); if (!tool) return; const dedupeBase = responseId || normalizeMcpSearchText(text); const dedupeKey = `${dedupeBase}::${tool.name}`; if (dedupeKey === lastDeclaredAssistantToolKey) return; lastDeclaredAssistantToolKey = dedupeKey; clearPendingAutoMcpFallback(); autoMcpInFlight = true; console.debug( `[Vocero MCP DECLARED] trigger responseId=${JSON.stringify(responseId)} tool=${tool.name} ` + `query=${JSON.stringify(lastUserQuery)}` ); client.suppressAssistantAudio?.(); chat.onToolCall(tool.name); void callAgentToolFromPanel(tool.name, true, lastUserQuery).finally(() => { autoMcpInFlight = false; }); } /** Push the active tool set to a live session so toggles apply mid-call. */ function pushToolsToSession() { if (!client || !LIVE_STATES.has(currentState)) return; const tools = activeToolDefs(); // #region debug-point B:push-tools-session reportDebug("B", "main.js:pushToolsToSession", "pushing tools to live session", { state: currentState, toolNames: tools.map((tool) => tool.name), toolCount: tools.length, searchMcpEnabled, }); // #endregion liveToolPolicySnapshot = ""; client.setTools(tools, tools.length ? "auto" : "none"); // The hidden tool-use hint depends on whether any tool is active, so refresh // instructions alongside the tool set. const instructions = effectiveInstructions(); // #region debug-point C:push-instructions-session reportDebug("C", "main.js:pushToolsToSession", "updating live session instructions", { instructionLength: instructions.length, hasSuggestedTools: topSuggestedAgentTools(3).length > 0, lastUserQuery, }); // #endregion pushLiveInstructions(instructions); } // ── Chat view ─────────────────────────────────────────────────────────────── // Owns the history panel, the ephemeral bubbles, and all transcript/tool // streaming state. The client's events are forwarded to its on* methods. const chat = new ChatView(); // ── Account / limiter ───────────────────────────────────────────────────── // Login chip + daily-limit modal (inert unless the deploy is in LB mode). The // server meters conversation time; the client just heartbeats a live session // and tears down when the server reports the budget is spent. const account = new Account(); let limiterOn = false; let heartbeatTimer = 0; let trackedSessionId = ""; let trackedTier = ""; // The waiting-queue ticket id while we're in line (else ""). Used to leave the // queue on teardown / tab-close so we don't hold a phantom place. let queuedTicketId = ""; // ── Idle auto-close (30s of silence, never during AI speech) ───────────── // The user asked the voice assistant to go to sleep silently after 30s of no // activity. "Activity" is any meaningful event on either side: the user // started speaking (VAD), the assistant started speaking (TTS), a partial or // final transcript arrived, a tool call was requested or returned, or the // client moved out of a transient pre-live state (connecting / queue / // your-turn). Once the session is live, every tick of activity resets the // countdown back to 30s; if 30s expire without a single event we close the // socket, refund any reserved budget, and drop back to the idle orb so the // microphone stops listening and the daily budget isn't eaten by a forgotten // tab. The countdown is intentionally disabled in the pre-live states (we // don't want to close while the user is still waiting in queue) and during // `ai-speaking` — if the *system* is still talking we NEVER close the // session mid-reply, even on very long TTS turns (the TTS engine itself will // fire transcript/response-finished events that re-arm the timer afterward). const IDLE_TIMEOUT_MS = 30 * 1000; let idleCloseTimer = 0; let idleCloseLastActivityAtMs = 0; /** @type {Set} - States where a live session is open and the user could be silent. `ai-speaking` is intentionally excluded so we never kill a reply mid-TTS. */ const IDLE_COUNT_STATES = new Set(["listening", "user-speaking", "processing"]); function _kickIdleTimer(reason = "activity") { idleCloseLastActivityAtMs = Date.now(); if (!IDLE_COUNT_STATES.has(currentState)) { if (idleCloseTimer) { clearTimeout(idleCloseTimer); idleCloseTimer = 0; } return; } if (idleCloseTimer) clearTimeout(idleCloseTimer); idleCloseTimer = window.setTimeout(() => { const staleMs = Date.now() - idleCloseLastActivityAtMs; if (staleMs + 500 < IDLE_TIMEOUT_MS) { // Race with a very recent kick: re-arm rather than close. if (DEBUG) console.debug(`[IDLE] fire skipped (staleMs=${staleMs} < ${IDLE_TIMEOUT_MS}), re-arming`); _kickIdleTimer("re-arm"); return; } console.info(`[IDLE] ⏰ Closing session after ${(staleMs / 1000).toFixed(1)}s of inactivity (timeout=${IDLE_TIMEOUT_MS / 1000}s) — reason=${reason}`); idleCloseTimer = 0; // Friendly caption flash right before teardown moves us back to the idle // copy (so the user understands *why* the call ended). setCaption("Chiuso per inattività", "muted"); void (async () => { try { await teardown(); } catch (err) { console.warn("[IDLE] teardown failed during auto-close:", err && err instanceof Error ? err.message : err); } // teardown() calls setState("idle") which restores the normal idle // caption. Leave a visible breadcrumb on the subcaption briefly so the // user can tell what happened instead of wondering why the orb reset. circleSubcaption.textContent = "30 sec sil."; circleSubcaption.hidden = false; window.setTimeout(() => { circleSubcaption.hidden = true; }, 8000); })(); }, IDLE_TIMEOUT_MS); if (DEBUG) { console.debug(`[IDLE] 🔄 reset timer (${IDLE_TIMEOUT_MS / 1000}s) reason=${reason} state=${currentState}`); } } function _cancelIdleTimer() { if (idleCloseTimer) { clearTimeout(idleCloseTimer); idleCloseTimer = 0; } idleCloseLastActivityAtMs = 0; if (DEBUG) console.debug(`[IDLE] ⛔ cancelled timer state=${currentState}`); } /** @type {S2sWsRealtimeClient | null} */ let client = null; /** @type {MediaStream | null} */ let micStream = null; let micMuted = false; /** @param {AppState} next */ function setState(next) { currentState = next; const view = STATE_VIEWS[next]; circleBtn.disabled = view.disabled; circleBtn.className = `circle ${STATE_CLASS[next]}`; if (next !== "error") setCaption(view.caption); const live = LIVE_STATES.has(next); orbWrap.classList.toggle("live", live); micBtn.setAttribute("aria-hidden", live ? "false" : "true"); stopBtn.setAttribute("aria-hidden", live ? "false" : "true"); micBtn.tabIndex = live ? 0 : -1; stopBtn.tabIndex = live ? 0 : -1; // Queue affordances: "Leave queue" whenever we're in line; "Join now" only once // it's our turn (a slot is held for us). Both live under #queue-actions. const yourTurn = next === "your-turn"; const inLine = next === "queued" || yourTurn; queueActions.hidden = !inLine; joinQueueBtn.hidden = !yourTurn; joinQueueBtn.tabIndex = yourTurn ? 0 : -1; leaveQueueBtn.hidden = !inLine; leaveQueueBtn.tabIndex = inLine ? 0 : -1; if (!yourTurn) stopJoinCountdown(); // Warm reassurance under the terse position, only while waiting in line. if (next === "queued") { circleSubcaption.textContent = "Sorry, we overhugged! 🤗 Every slot is busy, so we saved you a spot. Hang tight, you're moving up."; circleSubcaption.hidden = false; } else { circleSubcaption.hidden = true; } // Wake word: armed only when idle/error (no live mic contention with the // realtime session), and disarmed the moment we move into any other state. if (next === "idle" || next === "error") { startWakeWordListener(); } else { stopWakeWordListener(); } // Idle auto-close: arm the countdown whenever we land in a live conversing // state (user/assistant turn, processing), and cancel it whenever we leave // those states (idle, queue, error, teardown). Moving *to* a live state is // itself counted as activity so the 10s window starts fresh. if (IDLE_COUNT_STATES.has(next)) { _kickIdleTimer(`state→${next}`); } else { _cancelIdleTimer(); } updateRestartAvailability(); } function updateRestartAvailability() { // Restart works from any settled state — it tears down a live call (if any) // and reconnects with the current settings. Only block while mid-connect or // while waiting in the queue (restarting from there would just re-queue). restartBtn.disabled = currentState === "connecting" || currentState === "queued" || currentState === "your-turn"; restartHint.hidden = false; restartHint.textContent = LIVE_STATES.has(currentState) ? "Reconnects now with the settings above." : "Starts a conversation with the settings above."; } /** * @param {string} text * @param {"" | "error" | "muted"} [kind] */ function setCaption(text, kind = "") { const trimmed = text.trim(); circleCaption.textContent = trimmed; circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`; } function openSettings() { syncConnectionUi(); inputVoice.value = settings.voice; inputInstructions.value = pinnedPrompt || settings.instructions; inputInstructions.readOnly = !!pinnedPrompt; primaryPromptMeta.hidden = !pinnedPrompt; primaryPromptMeta.textContent = pinnedPrompt ? pinnedPromptSource ? `Primary prompt from ${pinnedPromptSource}` : "Primary prompt pinned by this deployment." : ""; syncGateUi(); updateRestartAvailability(); settingsModal.showModal(); } /** dB position (clamped to the slider axis) as a 0..1 fraction of the track. * @param {number} db */ function dbToFraction(db) { const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db)); return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB); } /** @param {number} f @returns {number} dB at a 0..1 position on the gate axis. */ function fractionToDb(f) { const clamped = Math.min(1, Math.max(0, f)); return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB)); } // ── Radial gate arc (around the mic button, live during a call) ───────────── // A 270° arc with the gap facing the orb (right). Fraction 0 (=Off) sits at the // bottom-ish start; 1 (=max) at the top-ish end. The level fill and the // threshold handle ride this same axis, mirroring the Settings widget. const ARC_R = 40; // A ~200° arc centred on the left (180°) so the wide gap faces the orb (right). const ARC_SPAN_DEG = 200; const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // lower-left start; Off end /** Point at fraction f (0..1) and radius r, in the 0..100 viewBox. * @param {number} f @param {number} [r] */ function arcPoint(f, r = ARC_R) { const deg = ARC_START_DEG + f * ARC_SPAN_DEG; const rad = (deg * Math.PI) / 180; return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) }; } /** SVG path `d` for the full 0..1 arc (clockwise). */ function fullArcD() { const a = arcPoint(0); const b = arcPoint(1); const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0; return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`; } /** One-time geometry: track, fill (dash-revealed) and the transparent hit band. */ function initGateArc() { const d = fullArcD(); mgaTrack.setAttribute("d", d); mgaFill.setAttribute("d", d); mgaHit.setAttribute("d", d); // pathLength 100 lets us reveal the fill by fraction via dashoffset. mgaFill.setAttribute("pathLength", "100"); mgaFill.style.strokeDasharray = "100 100"; mgaFill.style.strokeDashoffset = "100"; // empty until levels arrive renderGateHandle(); } /** Place the threshold bead on the arc at the stored threshold; flag off state. */ function renderGateHandle() { const off = settings.noiseGate <= GATE_OFF_DB; const p = arcPoint(dbToFraction(settings.noiseGate)); mgaHandle.setAttribute("cx", String(p.x)); mgaHandle.setAttribute("cy", String(p.y)); micGate.classList.toggle("gate-off", off); } /** Paint a 0..1 live level onto the arc fill (and the Settings meter if open). * Brightens the tick when the level crosses the threshold — i.e. the gate is * actually open — but only when gating is enabled. * @param {number} rms */ function paintInputLevel(rms) { const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB; const f = dbToFraction(db); mgaFill.style.strokeDashoffset = String(100 * (1 - f)); if (settingsModal.open) gateMeterFill.style.width = `${f * 100}%`; const enabled = settings.noiseGate > GATE_OFF_DB; micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(settings.noiseGate)); } /** The single place that commits a new gate threshold: updates both controls, * persists, and applies live to the running session. * @param {number} db */ function setGateThreshold(db) { settings.noiseGate = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db))); const off = settings.noiseGate <= GATE_OFF_DB; inputNoiseGate.value = String(settings.noiseGate); gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; renderGateHandle(); localStorage.setItem(STORAGE_KEYS.noiseGate, String(settings.noiseGate)); if (client && LIVE_STATES.has(currentState)) { client.setNoiseGate(gateParams(settings.noiseGate)); } } /** Reflect the stored gate threshold into the slider, label and arc handle. */ function syncGateUi() { inputNoiseGate.value = String(settings.noiseGate); const off = settings.noiseGate <= GATE_OFF_DB; gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; renderGateHandle(); } // Drag along the arc band to set the threshold (a tap on the glyph still mutes). let gateDragging = false; /** @param {PointerEvent} e */ function gatePointerToDb(e) { const rect = mgaArc.getBoundingClientRect(); const cx = rect.left + rect.width / 2; const cy = rect.top + rect.height / 2; let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI; if (deg < 0) deg += 360; // Map the on-arc angle to a fraction; angles in the right-side gap fall // outside [0,1] and fractionToDb clamps them to the nearest end (just-below // start -> Off, just-past end -> max). const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG; return fractionToDb(f); } mgaHit.addEventListener("pointerdown", (e) => { gateDragging = true; mgaHit.setPointerCapture(e.pointerId); setGateThreshold(gatePointerToDb(e)); }); mgaHit.addEventListener("pointermove", (e) => { if (gateDragging) setGateThreshold(gatePointerToDb(e)); }); const endGateDrag = (/** @type {PointerEvent} */ e) => { if (!gateDragging) return; gateDragging = false; try { mgaHit.releasePointerCapture(e.pointerId); } catch {} }; mgaHit.addEventListener("pointerup", endGateDrag); mgaHit.addEventListener("pointercancel", endGateDrag); settingsBtn.addEventListener("click", openSettings); // About panel: native , Esc closes for free; also close on the X and // on a click in the backdrop (a click whose target is the dialog itself). aboutBtn?.addEventListener("click", () => aboutModal.showModal()); // Mobile twin of the (i), living in the right-hand control cluster. $("#about-btn-m").addEventListener("click", () => aboutModal.showModal()); aboutClose.addEventListener("click", () => aboutModal.close()); aboutModal.addEventListener("click", (e) => { if (e.target === aboutModal) aboutModal.close(); }); // ── Tools panel ─────────────────────────────────────────────────────────── /** Reflect the current tool state into the panel controls. */ function syncToolsUi() { const openToolNames = new Set( Array.from(agentToolsList.querySelectorAll("details[open][data-mcp-tool-item]")) .map((item) => item.getAttribute("data-mcp-tool-item") || "") .filter(Boolean), ); const avail = searchAvailable(); toolWebSwitch.checked = toolsEnabled.web_search && avail; toolWebSwitch.disabled = !avail; toolWebRow.classList.toggle("disabled", !avail); toolCamSwitch.closest("div")?.setAttribute("hidden", ""); toolCamHint.setAttribute("hidden", ""); camPip.setAttribute("hidden", ""); if (serverSearchKey) { // Key lives server-side: show it as configured, never expose it. searchKeyInput.value = ""; searchKeyInput.placeholder = "•••••••• · provided by the server"; searchKeyInput.disabled = true; toolWebHint.textContent = "Ready. The search key is held server-side and never sent to your browser."; } else { searchKeyInput.disabled = false; searchKeyInput.value = userSearchKey; searchKeyInput.placeholder = "Paste a Serper key to enable web search"; toolWebHint.textContent = userSearchKey ? "Using your key — stored in this browser only." : "No server key configured. Add your own Serper key to enable web search."; } const count = agentToolDefs.length; if (!count) { agentToolsHint.textContent = "No MCP tools loaded from the remote agent profile."; } else if (searchMcpEnabled && lastUserQuery && rankedAgentTools.length) { agentToolsHint.textContent = `${count} MCP tools loaded. Top matches for "${truncateError(lastUserQuery)}".`; } else if (searchMcpEnabled && lastUserQuery) { agentToolsHint.textContent = `${count} MCP tools loaded. No description match found for "${truncateError(lastUserQuery)}".`; } else { agentToolsHint.textContent = `${count} MCP tool${count === 1 ? "" : "s"} loaded from the remote agent profile.`; } agentToolsList.innerHTML = renderAgentToolsHtml(); for (const toolName of openToolNames) { const item = agentToolsList.querySelector(`details[data-mcp-tool-item="${CSS.escape(toolName)}"]`); if (item instanceof HTMLDetailsElement) item.open = true; } } /** @returns {string} */ function renderAgentToolsHtml() { if (!agentToolDefs.length) return '
No MCP tools available.
'; const suggested = topSuggestedAgentTools(3); const suggestedNames = new Set(suggested.map((entry) => entry.tool.name)); let html = ""; if (searchMcpEnabled && lastUserQuery) { html += suggested.length ? `
Suggested For Current Request
${escHtml(lastUserQuery)}
` : `
No MCP tool description matches the current request.
`; } const orderedTools = suggested .map((entry) => entry.tool) .concat(agentToolDefs.filter((tool) => !suggestedNames.has(tool.name))); html += orderedTools.map((tool) => { const paramRows = describeToolParams(tool.parameters); const toolState = getMcpToolCallState(tool.name); const suggestedIndex = suggested.findIndex((entry) => entry.tool.name === tool.name); const badge = suggestedIndex >= 0 ? `Suggested #${suggestedIndex + 1}` : 'MCP'; return `
${escHtml(tool.name)} ${badge}
${escHtml(tool.description || "No description provided.")}
Parameters
${paramRows || '
No parameters.
'}
Manual Call
${renderMcpToolCallResultHtml(tool.name)}
`; }).join(""); return html; } /** @param {number} limit */ function topSuggestedAgentTools(limit) { return rankedAgentTools.slice(0, limit); } /** * @param {string} query * @param {{ partial?: boolean, itemId?: string }} [options] */ function updateMcpToolSuggestions(query, options = {}) { lastUserQuery = query.trim(); console.debug( `[MCP RANK] updateMcpToolSuggestions ENTRY partial=${!!options.partial} itemId=${options.itemId || "-"} ` + `query=${JSON.stringify(lastUserQuery)} searchMcpEnabled=${searchMcpEnabled}`, ); rankedAgentTools = rankAgentTools(lastUserQuery); // #region debug-point A:ranked-mcp-suggestions reportDebug("A", "main.js:updateMcpToolSuggestions", "ranked MCP tools for user query", { partial: !!options.partial, query: lastUserQuery, ranked: rankedAgentTools.map((entry) => ({ name: entry.tool.name, score: entry.score })), searchMcpEnabled, }); // #endregion const rankedNames = rankedAgentTools.map((entry) => `${entry.tool.name}:${entry.score}`).join(", "); console.debug( `[MCP RANK] updateMcpToolSuggestions RESULTS partial=${!!options.partial} ` + `matchedTools=${rankedAgentTools.length} ranked=${rankedNames || "none"}`, ); if (!options.partial && rankedAgentTools.length) { const top3 = rankedAgentTools .slice(0, 3) .map((entry, index) => `${index + 1}:${entry.tool.name}:${entry.score}`) .join(" | "); console.debug(`[MCP RANK] ✦ FINAL top3 query=${JSON.stringify(lastUserQuery)} top3=${top3 || "none"}`); } syncToolsUi(); if (client && LIVE_STATES.has(currentState)) { const instructions = effectiveInstructions(); if (options.partial && rankedAgentTools.length) { pushLiveToolPolicy(); pushLiveInstructions(instructions); } else if (!options.partial) { pushLiveToolPolicy(); pushLiveInstructions(instructions); maybeAutoRunSuggestedMcpTool(lastUserQuery, options.itemId || ""); } } } /** * @param {string} query * @param {string} itemId */ function maybeAutoRunSuggestedMcpTool(query, itemId) { if (!client || !LIVE_STATES.has(currentState) || !searchMcpEnabled || autoMcpInFlight) return; const normalized = normalizeMcpSearchText(query); const top = rankedAgentTools[0]; if (!normalized || !top) return; const second = rankedAgentTools[1]; const topNameText = normalizeMcpSearchText(top.tool.name); const topDescText = normalizeMcpSearchText(top.tool.description || ""); const topRequiredParams = requiredToolParams(top.tool.parameters); const relaxedDailyReportMatch = top.score >= AUTO_MCP_RELAXED_REPORT_SCORE && !topRequiredParams.length && ( ((topNameText.includes("day") || topNameText.includes("daily") || topNameText.includes("today")) && topNameText.includes("report")) || ((topDescText.includes("day") || topDescText.includes("daily") || topDescText.includes("today")) && topDescText.includes("report")) ); const strongByScore = top.score >= AUTO_MCP_MIN_SCORE; const strongByLead = top.score >= AUTO_MCP_MIN_SCORE_WITH_LEAD && (!second || (top.score - second.score) >= AUTO_MCP_MIN_LEAD); const strongMatch = top.score >= AUTO_MCP_MIN_SCORE || (top.score >= AUTO_MCP_MIN_SCORE_WITH_LEAD && (!second || (top.score - second.score) >= AUTO_MCP_MIN_LEAD)) || relaxedDailyReportMatch; if (!strongMatch) { console.debug(`[Vocero MCP AUTO] skip weak match query=${JSON.stringify(query)} top=${top.tool.name}:${top.score}`); return; } const turnKey = `${itemId || normalized}::${top.tool.name}`; if (turnKey === lastAutoMcpTurnKey) return; lastAutoMcpTurnKey = turnKey; clearPendingAutoMcpFallback(); const fastSafeMatch = !topRequiredParams.length && (top.score >= 40 || relaxedDailyReportMatch || strongByLead); const autoDelayMs = fastSafeMatch ? AUTO_MCP_FAST_DELAY_MS : AUTO_MCP_FALLBACK_DELAY_MS; reportDebug("A", "main.js:maybeAutoRunSuggestedMcpTool", "scheduled MCP fallback for matched user query", { itemId, query, toolName: top.tool.name, score: top.score, nextScore: second?.score ?? null, autoDelayMs, }); console.debug( `[Vocero MCP AUTO] scheduled query=${JSON.stringify(query)} itemId=${JSON.stringify(itemId)} ` + `tool=${top.tool.name} score=${top.score} reason=${strongByScore ? "strict" : strongByLead ? "lead" : "relaxed-report"} ` + `delayMs=${autoDelayMs}` ); pendingAutoMcpTimer = window.setTimeout(() => { pendingAutoMcpTimer = 0; if (!client || !LIVE_STATES.has(currentState) || autoMcpInFlight) return; autoMcpInFlight = true; console.debug( `[Vocero MCP AUTO] trigger query=${JSON.stringify(query)} itemId=${JSON.stringify(itemId)} ` + `tool=${top.tool.name} score=${top.score}` ); client.suppressAssistantAudio?.(); chat.onToolCall(top.tool.name); void callAgentToolFromPanel(top.tool.name, true, query).finally(() => { autoMcpInFlight = false; }); }, autoDelayMs); } /** * @param {string} query * @returns {{ tool: import("./ws/s2s-ws-client.js").ToolDef, score: number }[]} */ function rankAgentTools(query) { console.debug(`[MCP RANK] ════════════════════════════════════════`); console.debug(`[MCP RANK] rankAgentTools START query=${JSON.stringify(query)} searchMcpEnabled=${searchMcpEnabled} agentToolDefsCount=${agentToolDefs.length}`); const searchTexts = buildMcpSearchTexts(query); const normalized = searchTexts[0] || ""; if (!searchMcpEnabled || !normalized || !agentToolDefs.length) { const reasonParts = []; if (!searchMcpEnabled) reasonParts.push("searchMcpEnabled=false — da /api/config: MCP Aquachile feature OFF oppure S2S_TOOL_EXECUTION_MCP_URL non configurato"); if (!normalized) reasonParts.push("query-normalized-empty"); if (!agentToolDefs.length) reasonParts.push("agentToolDefs=0 — catalogo tool MCP vuoto: profilo agente non ha tools in agentProfile via Aquachile /api/agents/profile"); console.debug( `[MCP RANK] → EARLY EXIT reason="${reasonParts.join("; ")}" ` + `searchMcpEnabled=${searchMcpEnabled} normalized=${!!normalized} agentToolDefs=${agentToolDefs.length}`, ); console.debug(`[MCP RANK] ════════════════════════════════════════ (empty result)`); return []; } const tokens = [...new Set(searchTexts.flatMap((text) => text.split(" ").filter((token) => token.length >= 3)))]; if (!tokens.length) { console.debug( `[MCP RANK] → EARLY EXIT reason="nessun token >= 3 chars dopo split" searchTexts=${JSON.stringify(searchTexts)}`, ); console.debug(`[MCP RANK] ════════════════════════════════════════ (empty result)`); return []; } const dailyReportIntent = searchTexts.some((text) => /\b(day|daily|today)\b/.test(text) && /\breport\b/.test(text), ); console.debug( `[MCP RANK] searchTexts=${JSON.stringify(searchTexts)} ` + `tokensCount=${tokens.length} tokens=${JSON.stringify(tokens)} ` + `dailyReportIntent=${dailyReportIntent}`, ); const scored = agentToolDefs .map((tool) => { const nameText = normalizeMcpSearchText(tool.name); const descText = normalizeMcpSearchText(tool.description || ""); const paramText = normalizeMcpSearchText(extractToolParamSearchText(tool.parameters)); const requiredParams = requiredToolParams(tool.parameters); const breakdown = { nameHitFull: 0, descHitFull: 0, paramHitFull: 0, nameHitTokens: 0, descHitTokens: 0, paramHitTokens: 0, allFoundBonus: 0, noRequiredParamsBonus: 0, dailyReportMatchBoost: 0, dailyReportPenalty: 0, }; let score = 0; for (const searchText of searchTexts) { if (!searchText) continue; if (descText.includes(searchText)) { const add = searchText === normalized ? 20 : 16; score += add; breakdown.descHitFull += add; } if (nameText.includes(searchText)) { const add = searchText === normalized ? 24 : 18; score += add; breakdown.nameHitFull += add; } if (paramText.includes(searchText)) { const add = searchText === normalized ? 6 : 4; score += add; breakdown.paramHitFull += add; } } const tokenMatchDetails = []; for (const token of tokens) { const onName = nameText.includes(token); const onDesc = descText.includes(token); const onParam = paramText.includes(token); if (!onName && !onDesc && !onParam) continue; const entry = { token }; if (onName) { score += 8; breakdown.nameHitTokens += 8; entry.name = true; } if (onDesc) { score += 5; breakdown.descHitTokens += 5; entry.desc = true; } if (onParam) { score += 2; breakdown.paramHitTokens += 2; entry.param = true; } tokenMatchDetails.push(entry); } const allFound = searchTexts.some((searchText) => searchText.split(" ").filter((token) => token.length >= 3).every((token) => nameText.includes(token) || descText.includes(token) || paramText.includes(token), ), ); if (allFound) { score += 10; breakdown.allFoundBonus = 10; } if (!requiredParams.length) { score += 3; breakdown.noRequiredParamsBonus = 3; } if (dailyReportIntent) { if ((nameText.includes("day report") || nameText.includes("daily report") || nameText.includes("today")) && (descText.includes("day report") || descText.includes("daily report") || descText.includes("today"))) { score += 24; breakdown.dailyReportMatchBoost = 24; } else if (requiredParams.length) { score -= 10; breakdown.dailyReportPenalty = -10; } } console.debug( `[MCP RANK] ▸ tool=${JSON.stringify(tool.name)} score=${score} ` + `requiredParams=${requiredParams.length} allFound=${allFound}\n` + `[MCP RANK] ↳ fullMatches: name=${breakdown.nameHitFull} desc=${breakdown.descHitFull} params=${breakdown.paramHitFull}\n` + `[MCP RANK] ↳ tokenMatches(${tokenMatchDetails.length}): ${tokenMatchDetails.length ? JSON.stringify(tokenMatchDetails) : "-"}\n` + `[MCP RANK] ↳ bonuses: allFound=${breakdown.allFoundBonus} noRequiredParams=${breakdown.noRequiredParamsBonus} ` + `dailyBoost=${breakdown.dailyReportMatchBoost} dailyPenalty=${breakdown.dailyReportPenalty}\n` + `[MCP RANK] ↳ nameTokens=${JSON.stringify(nameText.slice(0, 100))}\n` + `[MCP RANK] ↳ descTokens=${JSON.stringify(descText.slice(0, 200))}\n` + `[MCP RANK] ↳ paramTokens=${JSON.stringify(paramText.slice(0, 150))}`, ); return { tool, score }; }); const filtered = scored.filter((entry) => entry.score > 0); const sorted = filtered.sort((a, b) => b.score - a.score || a.tool.name.localeCompare(b.tool.name)); console.debug( `[MCP RANK] DONE scoredCount=${scored.length} matchedCount=${filtered.length} ` + `top5=${JSON.stringify(sorted.slice(0, 5).map((e) => ({ t: e.tool.name, s: e.score })))}`, ); console.debug(`[MCP RANK] ════════════════════════════════════════`); return sorted; } /** * @param {string} text * @returns {string} */ function translateMcpSearchTextToEnglish(text) { const before = normalizeMcpSearchText(text); let translated = before; const appliedPhrases = []; for (const [pattern, replacement] of MCP_SEARCH_TRANSLATION_PHRASES) { const previous = translated; translated = translated.replace(pattern, replacement); if (translated !== previous) { appliedPhrases.push({ pattern: String(pattern), to: replacement }); } } const beforeTokens = translated; const appliedTokens = []; translated = translated .split(" ") .map((token) => { const mapped = MCP_SEARCH_TRANSLATION_TOKENS[token]; if (mapped && mapped !== token) { appliedTokens.push({ from: token, to: mapped }); return mapped; } return token; }) .join(" "); translated = normalizeMcpSearchText(translated); console.debug( `[MCP RANK] translate input=${JSON.stringify(before)} phrasesApplied=${appliedPhrases.length} ` + `tokensApplied=${appliedTokens.length} output=${JSON.stringify(translated)}` + (appliedPhrases.length ? ` phrases=${JSON.stringify(appliedPhrases)}` : "") + (appliedTokens.length ? ` tokens=${JSON.stringify(appliedTokens)}` : ""), ); return translated; } /** * @param {string} text * @returns {string[]} */ function buildMcpSearchTexts(text) { const normalized = normalizeMcpSearchText(text); const translated = translateMcpSearchTextToEnglish(text); const result = [...new Set([normalized, translated].filter(Boolean))]; console.debug( `[MCP RANK] buildSearchTexts raw=${JSON.stringify(text)} ` + `normalized=${JSON.stringify(normalized)} translated=${JSON.stringify(translated)} ` + `count=${result.length} final=${JSON.stringify(result)}`, ); return result; } /** * @param {string} text * @returns {string} */ function normalizeMcpSearchText(text) { return text .toLowerCase() .normalize("NFD") .replace(/[\u0300-\u036f]/g, "") .replace(/[^a-z0-9]+/g, " ") .trim(); } /** * @param {any} schema * @returns {string} */ function extractToolParamSearchText(schema) { const props = schema && typeof schema === "object" && schema.properties && typeof schema.properties === "object" ? schema.properties : null; if (!props) return ""; /** @type {string[]} */ const chunks = []; for (const [name, raw] of Object.entries(props)) { chunks.push(name); if (raw && typeof raw === "object" && typeof raw.description === "string") { chunks.push(raw.description); } } return chunks.join(" "); } /** * @param {any} schema * @returns {string} */ function describeToolParams(schema) { const props = schema && typeof schema === "object" && schema.properties && typeof schema.properties === "object" ? schema.properties : null; if (!props) return ""; /** @type {string[]} */ const rows = []; const required = Array.isArray(schema.required) ? new Set(schema.required) : new Set(); for (const [name, raw] of Object.entries(props)) { const spec = raw && typeof raw === "object" ? raw : {}; const type = paramTypeLabel(spec); const def = "default" in spec ? ` · default: ${formatParamValue(spec.default)}` : ""; const req = required.has(name) ? " required" : ""; rows.push( `
` + `${escHtml(name)}` + `${escHtml(type + req + def)}` + `
`, ); } return rows.join(""); } /** * @param {any} spec * @returns {string} */ function paramTypeLabel(spec) { if (Array.isArray(spec.anyOf)) { /** @type {string[]} */ const labels = []; for (const item of spec.anyOf) { labels.push((item && typeof item === "object" && typeof item.type === "string") ? item.type : "unknown"); } return labels.join(" | "); } if (typeof spec.type === "string") return spec.type; return "unknown"; } /** * @param {unknown} value * @returns {string} */ function formatParamValue(value) { if (value === null) return "null"; if (typeof value === "string") return value; return JSON.stringify(value); } /** * @param {any} schema * @returns {Record} */ function schemaProperties(schema) { return schema && typeof schema === "object" && schema.properties && typeof schema.properties === "object" ? schema.properties : {}; } /** * @param {any} schema * @returns {string[]} */ function requiredToolParams(schema) { if (!Array.isArray(schema?.required)) return []; /** @type {string[]} */ const required = []; for (const name of schema.required) { if (typeof name === "string") required.push(name); } return required; } /** * @param {any} spec * @param {string} type * @returns {boolean} */ function schemaSupportsType(spec, type) { if (!spec || typeof spec !== "object") return false; if (spec.type === type) return true; if (Array.isArray(spec.anyOf)) { for (const item of spec.anyOf) { if (item && typeof item === "object" && item.type === type) return true; } } return false; } /** * @param {any} spec * @returns {boolean} */ function isStringArraySchema(spec) { if (!spec || typeof spec !== "object") return false; if (spec.type === "array") { return !spec.items || spec.items.type === "string"; } if (Array.isArray(spec.anyOf)) { for (const item of spec.anyOf) { if (item && typeof item === "object" && item.type === "array" && (!item.items || item.items.type === "string")) { return true; } } } return false; } /** * @param {string} name * @returns {string[]} */ function paramAliases(name) { const aliases = new Set([name, name.replace(/_/g, " "), name.replace(/_/g, "-")]); if (/phone|mobile|whatsapp/i.test(name)) { aliases.add("phone"); aliases.add("phone number"); aliases.add("mobile"); aliases.add("mobile number"); aliases.add("whatsapp"); aliases.add("whatsapp number"); aliases.add("number"); } return [...aliases].filter(Boolean); } const DIGIT_WORDS = { zero: "0", oh: "0", one: "1", two: "2", three: "3", four: "4", five: "5", six: "6", seven: "7", eight: "8", nine: "9", }; /** * @param {string} name * @returns {boolean} */ function isPhoneLikeParam(name) { return /phone|mobile|whatsapp/i.test(name); } /** * @param {string} text * @returns {string} */ function normalizePhoneText(text) { const trimmed = text.trim(); if (!trimmed) return ""; const hasLeadingPlus = /^\s*\+/.test(trimmed); const compact = trimmed .toLowerCase() .replace(/[^a-z0-9+\s-]/g, " ") .split(/\s+/) .filter(Boolean); /** @type {string[]} */ const digits = []; for (const token of compact) { if (token === "+" && !digits.length && hasLeadingPlus) continue; if (/^\d+$/.test(token)) { digits.push(token); continue; } if (Object.prototype.hasOwnProperty.call(DIGIT_WORDS, token)) { digits.push(DIGIT_WORDS[/** @type {keyof typeof DIGIT_WORDS} */ (token)]); } } if (!digits.length) return ""; return `${hasLeadingPlus ? "+" : ""}${digits.join("")}`; } /** * @param {string} query * @returns {string} */ function extractPhoneValue(query) { const patterns = [ /\b(?:my\s+)?(?:phone|mobile|whatsapp|number)\s*(?:number)?\s*(?:is|=|:)?\s*([^.!,;\n]+)/i, /\bcall me at\s*([^.!,;\n]+)/i, /\breach me at\s*([^.!,;\n]+)/i, ]; for (const pattern of patterns) { const match = query.match(pattern); const normalized = normalizePhoneText(match?.[1] || ""); if (normalized) return normalized; } return normalizePhoneText(query); } /** * @param {string} query * @param {string} name * @returns {string} */ function inferStringParamValue(query, name) { if (isPhoneLikeParam(name)) return extractPhoneValue(query); return ""; } /** * @param {string} query * @param {string[]} aliases * @returns {string} */ function matchParamValueText(query, aliases) { for (const alias of aliases) { const escaped = alias.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const patterns = [ new RegExp(`\\b${escaped}\\b\\s*(?:=|:|is|to)?\\s*"([^"]+)"`, "i"), new RegExp(`\\b${escaped}\\b\\s*(?:=|:|is|to)?\\s*'([^']+)'`, "i"), new RegExp(`\\b${escaped}\\b\\s*(?:=|:|is|to)?\\s*([^,;\\n]+)`, "i"), ]; for (const pattern of patterns) { const match = query.match(pattern); if (match?.[1]) return match[1].trim(); } } return ""; } /** * @param {string} query * @param {any} schema * @returns {Record} */ function inferToolArgsFromQuery(query, schema) { const props = schemaProperties(schema); /** @type {Record} */ const inferred = {}; for (const [name, raw] of Object.entries(props)) { const spec = raw && typeof raw === "object" ? raw : {}; const valueText = matchParamValueText(query, paramAliases(name)); const fallbackText = !valueText && schemaSupportsType(spec, "string") ? inferStringParamValue(query, name) : ""; const resolvedText = valueText || fallbackText; if (!resolvedText) continue; if (schemaSupportsType(spec, "number")) { const num = Number(resolvedText); if (!Number.isNaN(num)) inferred[name] = num; continue; } if (schemaSupportsType(spec, "boolean")) { const normalized = resolvedText.toLowerCase(); if (["true", "yes", "1", "on"].includes(normalized)) inferred[name] = true; else if (["false", "no", "0", "off"].includes(normalized)) inferred[name] = false; continue; } if (isStringArraySchema(spec)) { inferred[name] = resolvedText .split(",") .map((item) => item.trim()) .filter(Boolean); continue; } if (schemaSupportsType(spec, "string")) { inferred[name] = resolvedText; } } return inferred; } /** * @param {Record} args * @param {any} schema * @returns {Record} */ function applyToolArgDefaults(args, schema) { const props = schemaProperties(schema); /** @type {Record} */ const merged = { ...args }; for (const [name, raw] of Object.entries(props)) { const spec = raw && typeof raw === "object" ? raw : {}; if (!(name in merged) && Object.prototype.hasOwnProperty.call(spec, "default")) { merged[name] = spec.default; } } return merged; } /** * @param {Record} args * @param {any} schema * @returns {string[]} */ function missingRequiredToolArgs(args, schema) { return requiredToolParams(schema).filter((name) => { const value = args[name]; return value === undefined || value === null || value === ""; }); } /** * @param {string} toolName * @returns {{ argsText: string, status: "idle" | "running" | "success" | "error", resultText: string, lastAction: "call" | "send" }} */ function getMcpToolCallState(toolName) { if (!mcpToolCallState[toolName]) { mcpToolCallState[toolName] = { argsText: "{}", status: "idle", resultText: "", lastAction: "call", }; } return mcpToolCallState[toolName]; } /** * @param {string} toolName * @returns {import("./ws/s2s-ws-client.js").ToolDef | undefined} */ function findAgentToolDef(toolName) { return agentToolDefs.find((tool) => tool.name === toolName); } /** * @param {string} toolName * @returns {string} */ function renderMcpToolCallResultHtml(toolName) { const state = getMcpToolCallState(toolName); if (state.status === "idle" && !state.resultText) return ""; const cls = state.status === "error" ? "agent-tool-call-result error" : state.status === "success" ? "agent-tool-call-result success" : "agent-tool-call-result running"; const label = state.status === "running" ? (state.lastAction === "send" ? "Calling tool and sending to conversation..." : "Calling tool...") : state.status === "error" ? "Call failed" : (state.lastAction === "send" ? "Sent to conversation" : "Call result"); return `
${escHtml(label)}
${escHtml(state.resultText || "")}
`; } /** * @param {string} toolName * @param {boolean} [sendToConversation] * @param {string} [sourceQuery] * @returns {Promise} */ async function callAgentToolFromPanel(toolName, sendToConversation = false, sourceQuery = "") { const tool = findAgentToolDef(toolName); if (!tool) return; const state = getMcpToolCallState(toolName); state.lastAction = sendToConversation ? "send" : "call"; let args = /** @type {Record} */ ({}); try { args = state.argsText.trim() ? JSON.parse(state.argsText) : {}; } catch (err) { const msg = err instanceof Error ? err.message : String(err); state.status = "error"; state.resultText = `Invalid JSON arguments: ${msg}`; syncToolsUi(); console.warn(`[Vocero MCP PANEL] invalid args name=${toolName} error=${msg}`); return; } if (!args || typeof args !== "object" || Array.isArray(args)) { state.status = "error"; state.resultText = "Arguments must be a JSON object."; syncToolsUi(); console.warn(`[Vocero MCP PANEL] invalid args shape name=${toolName}`); return; } const inferredArgs = sourceQuery.trim() ? inferToolArgsFromQuery(sourceQuery, tool.parameters) : {}; args = applyToolArgDefaults({ ...inferredArgs, ...args }, tool.parameters); const missingArgs = missingRequiredToolArgs(args, tool.parameters); if (missingArgs.length) { state.status = "error"; state.resultText = `Missing required arguments: ${missingArgs.join(", ")}`; syncToolsUi(); console.warn( `[Vocero MCP PANEL] missing required args name=${toolName} missing=${missingArgs.join(",")} ` + `sourceQuery=${JSON.stringify(sourceQuery)}` ); return; } state.status = "running"; state.resultText = ""; syncToolsUi(); reportDebug("E", "main.js:callAgentToolFromPanel", "manual MCP tool call from tools panel", { name: toolName, args, sendToConversation, sourceQuery, }); console.debug( `[Vocero MCP PANEL] call name=${toolName} sendToConversation=${sendToConversation} ` + `sourceQuery=${JSON.stringify(sourceQuery)} inferred=${JSON.stringify(inferredArgs)} args=${JSON.stringify(args)}` ); try { const execStartedAt = performance.now(); const { rawOutput, speakableOutput } = await execRemoteMcpTool(toolName, args); const output = rawOutput; reportDebug("A", "main.js:callAgentToolFromPanel", "MCP tool execution finished", { name: toolName, sendToConversation, outputLength: output.length, speakableOutputLength: speakableOutput.length, durationMs: Math.round(performance.now() - execStartedAt), }); if (sendToConversation) { if (!client || !LIVE_STATES.has(currentState)) { throw new Error("No live conversation is active."); } const promptBuildStartedAt = performance.now(); const modelOutput = speakableOutput || summarizeMcpOutputForConversation(output); const prompt = ( sourceQuery.trim() ? `User request: ${sourceQuery.trim()}\n\n` : "" ) + `Tool ${toolName} returned the following internal facts:\n\n${modelOutput}\n\n` + "Answer the user naturally in their language. Do not read JSON keys, field names, or raw URLs aloud unless the user explicitly asks for them."; reportDebug("A", "main.js:callAgentToolFromPanel", "built live prompt from MCP output", { name: toolName, outputLength: output.length, modelOutputLength: modelOutput.length, promptLength: prompt.length, buildMs: Math.round(performance.now() - promptBuildStartedAt), }); const chatRenderStartedAt = performance.now(); chat.onToolResult(toolName, JSON.stringify(args), output); reportDebug("C", "main.js:callAgentToolFromPanel", "rendered MCP tool result in chat", { name: toolName, outputLength: output.length, renderMs: Math.round(performance.now() - chatRenderStartedAt), }); chat.onTranscript({ role: "user", text: prompt, partial: false, itemId: `manual-mcp-${toolName}-${Date.now()}`, }); reportDebug("B", "main.js:callAgentToolFromPanel", "sending MCP prompt back to live conversation", { name: toolName, promptLength: prompt.length, outputLength: output.length, }); client.sendUserText(prompt); client.requestResponse(); console.debug(`[Vocero MCP PANEL] sent to conversation name=${toolName} length=${prompt.length}`); } state.status = "success"; state.resultText = output; console.debug(`[Vocero MCP PANEL] result name=${toolName} output=${output.slice(0, 500)}`); } catch (err) { const msg = err instanceof Error ? err.message : String(err); state.status = "error"; state.resultText = msg; console.warn(`[Vocero MCP PANEL] error name=${toolName} error=${msg}`); } syncToolsUi(); } toolsBtn.addEventListener("click", () => { syncToolsUi(); toolsModal.showModal(); }); toolsClose.addEventListener("click", () => toolsModal.close()); toolsModal.addEventListener("click", (e) => { if (e.target === toolsModal) toolsModal.close(); }); agentToolsList.addEventListener("input", (event) => { const target = /** @type {HTMLElement | null} */ (event.target instanceof HTMLElement ? event.target : null); if (!(target instanceof HTMLTextAreaElement)) return; const toolName = target.dataset.mcpToolArgs; if (!toolName) return; const state = getMcpToolCallState(toolName); state.argsText = target.value; }); agentToolsList.addEventListener("click", (event) => { const target = /** @type {HTMLElement | null} */ (event.target instanceof HTMLElement ? event.target : null); const button = target?.closest("[data-mcp-call-btn]"); if (button instanceof HTMLButtonElement) { const toolName = button.dataset.mcpCallBtn; if (!toolName) return; void callAgentToolFromPanel(toolName, false); return; } const sendButton = target?.closest("[data-mcp-send-btn]"); if (!(sendButton instanceof HTMLButtonElement)) return; const toolName = sendButton.dataset.mcpSendBtn; if (!toolName) return; void callAgentToolFromPanel(toolName, true); }); toolWebSwitch.addEventListener("change", () => { if (toolWebSwitch.checked && !searchAvailable()) { toolWebSwitch.checked = false; // guard: can't enable without a key return; } toolsEnabled.web_search = toolWebSwitch.checked; saveTools(); pushToolsToSession(); }); searchKeyInput.addEventListener("input", () => { if (serverSearchKey) return; userSearchKey = searchKeyInput.value.trim(); if (userSearchKey) localStorage.setItem(STORAGE_KEYS.searchKey, userSearchKey); else localStorage.removeItem(STORAGE_KEYS.searchKey); const avail = searchAvailable(); toolWebSwitch.disabled = !avail; toolWebRow.classList.toggle("disabled", !avail); // Losing the key disables a previously-enabled tool. if (!avail && toolsEnabled.web_search) { toolsEnabled.web_search = false; toolWebSwitch.checked = false; saveTools(); pushToolsToSession(); } toolWebHint.textContent = userSearchKey ? "Using your key — stored in this browser only." : "No server key configured. Add your own Serper key to enable web search."; }); // ── Tool executor ───────────────────────────────────────────────────────── // Runs the function the model called, returns the result, and asks for a // response so the model speaks it. Errors come back as the tool output too, so // the model can recover gracefully instead of the turn stalling. /** * Run the function the model called, return its result to the backend, and ask * for a follow-up response. We also hand the result back to the caller so it * can be shown in the conversation once the tool has actually run. * @param {string} name @param {string} argsJson @param {string} callId * @returns {Promise<{ output: string, image?: string }>} */ async function runTool(name, argsJson, callId) { if (!client) return { output: "" }; let args = /** @type {Record} */ ({}); try { args = JSON.parse(argsJson || "{}"); } catch { /* keep {} */ } // #region debug-point D:run-tool-entry reportDebug("D", "main.js:runTool", "tool execution started", { name, callId, args, knownMcpTool: agentToolDefs.some((tool) => tool.name === name), }); // #endregion if (DEBUG) console.debug(`[tool] run name=${name} callId=${JSON.stringify(callId)} args=${argsJson}`); if (!callId) console.warn("[tool] empty call_id — the backend didn't tag the call, can't return a function_call_output"); /** @type {{ output: string, image?: string }} */ let result = { output: "" }; try { if (name === "web_search") { const query = typeof args.query === "string" ? args.query : ""; result.output = await execWebSearch(query); // Return the result and let the bare response.create (below) trigger the // spoken answer. client.sendToolOutput(callId, result.output); } else if (agentToolDefs.some((tool) => tool.name === name)) { // #region debug-point E:mcp-proxy-before-fetch reportDebug("E", "main.js:runTool", "calling MCP proxy", { name, callId, args, }); // #endregion const mcpResult = await execRemoteMcpTool(name, args); result.output = mcpResult.rawOutput; console.debug(`[Vocero MCP OUT] name=${name} callId=${JSON.stringify(callId)} output=${result.output}`); // #region debug-point E:mcp-proxy-after-fetch reportDebug("E", "main.js:runTool", "MCP proxy returned result", { name, callId, outputPreview: result.output.slice(0, 300), speakableOutputLength: mcpResult.speakableOutput.length, }); // #endregion client.sendToolOutput(callId, mcpResult.speakableOutput || summarizeMcpOutputForConversation(result.output)); } else { result.output = `Unknown tool: ${name}`; client.sendToolOutput(callId, result.output); } } catch (err) { const msg = err instanceof Error ? err.message : String(err); result.output = `Tool failed: ${msg}`; // #region debug-point E:mcp-proxy-error reportDebug("E", "main.js:runTool", "tool execution failed", { name, callId, error: msg, }); // #endregion client.sendToolOutput(callId, result.output); } if (DEBUG) console.debug(`[tool] requesting model response after ${name}`); client.requestResponse(result.image ? { image: result.image } : undefined); return result; } /** * Execute an MCP-backed tool through the app's same-origin proxy. * @param {string} name * @param {Record} args * @returns {Promise<{ rawOutput: string, speakableOutput: string }>} */ async function execRemoteMcpTool(name, args) { const startedAt = performance.now(); if (DEBUG) { console.debug(`[Vocero MCP HTTP] request name=${name} url=/api/mcp/tool args=${JSON.stringify(args || {})}`); } const res = await fetch("api/mcp/tool", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ name, arguments: args || {} }), }); if (DEBUG) { console.debug(`[Vocero MCP HTTP] response name=${name} status=${res.status} ok=${res.ok}`); } if (!res.ok) { let detail = String(res.status); try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {} console.warn(`[Vocero MCP HTTP] error name=${name} detail=${detail}`); throw new Error(`mcp error (${detail})`); } const json = await res.json(); reportDebug("A", "main.js:execRemoteMcpTool", "received MCP HTTP response body", { name, durationMs: Math.round(performance.now() - startedAt), resultJsonLength: JSON.stringify(json?.result ?? null).length, }); if (DEBUG) { const preview = JSON.stringify(json?.result ?? null).slice(0, 300); console.debug(`[Vocero MCP HTTP] payload name=${name} preview=${preview}`); } const rawOutput = formatMcpToolResult(json.result); const speakableOutput = formatSpeakableMcpToolResult(json.result, rawOutput); return { rawOutput, speakableOutput }; } /** * Turn an MCP tool result into plain text the model can read back. * @param {any} result * @returns {string} */ function formatMcpToolResult(result) { if (!result) return "Tool returned no result."; if (result.structuredContent) { return JSON.stringify(result.structuredContent, null, 2); } /** @type {{ type?: string, text?: string }[]} */ const parts = Array.isArray(result.content) ? result.content : []; /** @type {string[]} */ const textParts = []; for (const item of parts) { if (item && item.type === "text" && typeof item.text === "string") { const text = item.text.trim(); if (text) textParts.push(text); } } if (textParts.length) return textParts.join("\n\n"); return JSON.stringify(result, null, 2); } /** * Turn an MCP tool result into compact speech-ready facts for the live model/TTS path. * @param {any} result * @param {string} [rawOutput] * @returns {string} */ function formatSpeakableMcpToolResult(result, rawOutput = "") { if (!result) return "Tool returned no result."; if (result.structuredContent != null) { const summary = buildSpeakableMcpFacts(result.structuredContent); if (summary) return summary.length > 1200 ? `${summary.slice(0, 1200).trim()}\n...` : summary; } /** @type {{ type?: string, text?: string }[]} */ const parts = Array.isArray(result.content) ? result.content : []; /** @type {string[]} */ const textParts = []; for (const item of parts) { if (item && item.type === "text" && typeof item.text === "string") { const text = item.text.trim(); if (text) textParts.push(text); } } if (textParts.length) { return summarizeMcpOutputForConversation(textParts.join("\n\n")); } return summarizeMcpOutputForConversation(rawOutput || JSON.stringify(result, null, 2)); } const SPEAKABLE_PRIORITY_KEYS = new Set([ "name", "title", "summary", "description", "message", "status", "role", "date", "reportId", "report_id", "center", "priority", "audience", "briefing", "value", "label", "text", ]); const SKIP_SPEAKABLE_KEYS = new Set([ "id", "userId", "user_id", "openapiUrl", "docsUrl", "mcpPath", "url", "urls", "image", "images", "raw", "json", ]); /** * @param {string} text * @param {number} maxLen * @returns {string} */ function trimSpeakableText(text, maxLen) { const clean = String(text || "").replace(/\s+/g, " ").trim(); if (!clean) return ""; return clean.length > maxLen ? `${clean.slice(0, maxLen).trim()}...` : clean; } /** * @param {string} key * @returns {string} */ function humanizeFactKey(key) { return key .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .replace(/[_-]+/g, " ") .replace(/\s+/g, " ") .trim() .replace(/^./, (c) => c.toUpperCase()); } /** * @param {any} value * @param {number} [maxLen] * @returns {string} */ function pickSpeakableSnippet(value, maxLen = 180) { if (value == null) return ""; if (typeof value === "string") return trimSpeakableText(value, maxLen); if (typeof value === "number" || typeof value === "boolean") return String(value); if (Array.isArray(value)) { const parts = value .slice(0, 3) .map((item) => pickSpeakableSnippet(item, 80)) .filter(Boolean); return trimSpeakableText(parts.join("; "), maxLen); } if (typeof value === "object") { const obj = /** @type {Record} */ (value); for (const key of SPEAKABLE_PRIORITY_KEYS) { if (key in obj) { const text = pickSpeakableSnippet(obj[key], maxLen); if (text) return text; } } for (const [key, item] of Object.entries(obj)) { if (SKIP_SPEAKABLE_KEYS.has(key)) continue; const text = pickSpeakableSnippet(item, maxLen); if (text) return text; } } return ""; } /** * @param {Record} obj * @returns {string[]} */ function collectSpeakableObjectFacts(obj) { /** @type {string[]} */ const facts = []; for (const [key, value] of Object.entries(obj)) { if (SKIP_SPEAKABLE_KEYS.has(key) || value == null) continue; if (typeof value === "string") { const text = trimSpeakableText(value, 220); if (text) facts.push(`${humanizeFactKey(key)}: ${text}`); continue; } if (typeof value === "number" || typeof value === "boolean") { facts.push(`${humanizeFactKey(key)}: ${String(value)}`); continue; } if (Array.isArray(value)) { if (!value.length) continue; const first = pickSpeakableSnippet(value[0], 160); if (first) { facts.push(`${humanizeFactKey(key)}: ${first}`); if (value.length > 1) facts.push(`${humanizeFactKey(key)} count: ${value.length}`); } continue; } if (typeof value === "object") { const nested = pickSpeakableSnippet(value, 180); if (nested) facts.push(`${humanizeFactKey(key)}: ${nested}`); } } return facts; } /** * @param {any} parsed * @returns {string} */ function buildSpeakableMcpFacts(parsed) { if (parsed == null) return "No useful facts returned by the tool."; if (typeof parsed === "string") return trimSpeakableText(parsed, 900); if (typeof parsed === "number" || typeof parsed === "boolean") return String(parsed); if (Array.isArray(parsed)) { const items = parsed .slice(0, 4) .map((item, index) => { const text = pickSpeakableSnippet(item, 180); return text ? `- Item ${index + 1}: ${text}` : ""; }) .filter(Boolean); if (parsed.length > 4) items.push(`- Additional items not shown: ${parsed.length - 4}`); return items.join("\n") || "The tool returned a list with no speakable details."; } const obj = /** @type {Record} */ (parsed); const facts = collectSpeakableObjectFacts(obj).slice(0, 8); if (facts.length) return facts.map((line) => `- ${line}`).join("\n"); const fallback = pickSpeakableSnippet(obj, 900); return fallback || "The tool returned structured data with no speakable summary."; } /** * @param {string} output * @returns {string} */ function summarizeMcpOutputForConversation(output) { const text = (output || "").trim(); if (!text) return "Tool returned no result."; try { const parsed = JSON.parse(text); const summary = buildSpeakableMcpFacts(parsed); return summary.length > 1200 ? `${summary.slice(0, 1200).trim()}\n...` : summary; } catch { return trimSpeakableText(text, 1200); } } /** * Normalize agent profile tools into OpenAI-compatible function tool defs. * @param {any[]} tools * @returns {import("./ws/s2s-ws-client.js").ToolDef[]} */ function normalizeAgentTools(tools) { if (!Array.isArray(tools)) return []; return tools .filter((tool) => tool && tool.enabled !== false && typeof tool.name === "string") .map((tool) => ({ type: "function", name: tool.name, description: typeof tool.description === "string" ? tool.description : "", parameters: tool.parameters && typeof tool.parameters === "object" ? tool.parameters : { type: "object", properties: {}, required: [] }, })); } /** @param {string} query @returns {Promise} */ async function execWebSearch(query) { if (!query) return "No query provided."; /** @type {Record} */ const body = { query }; // Only send a user key when there's no server key (server prefers its own). if (!serverSearchKey && userSearchKey) body.key = userSearchKey; const res = await fetch("api/search", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), }); if (!res.ok) { let detail = String(res.status); try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {} throw new Error(`search error (${detail})`); } const json = await res.json(); // Date-stamp the header so the model treats these as fresh realtime facts // rather than its (older) training knowledge. const today = new Date().toISOString().slice(0, 10); /** @type {string[]} */ const lines = [`Google search result from ${today}:`]; if (json.answer) lines.push(`Answer: ${json.answer}`); for (const r of json.results || []) { lines.push(`- ${r.title}: ${r.snippet} (${r.url})`); } return lines.length > 1 ? lines.join("\n") : `${lines[0]}\nNo results found.`; } async function fetchVoceroUserProfile() { if (!voceroUserPhoneNumber) return; try { const res = await fetch(`api/user/phone?phoneNumber=${encodeURIComponent(voceroUserPhoneNumber)}`); if (!res.ok) { const message = await res.text(); throw new Error(message || `HTTP ${res.status}`); } const json = await res.json(); voceroUserProfile = json && typeof json === "object" && json.result && typeof json.result === "object" ? json.result : null; voceroUserProfileLoaded = true; console.debug( `[Vocero USER AUTH] phoneNumber=${JSON.stringify(voceroUserPhoneNumber)} ` + `loaded=${!!voceroUserProfile} keys=${voceroUserProfile ? Object.keys(voceroUserProfile).join(",") : "none"}` ); syncVoceroHeaderUser(); if (client && LIVE_STATES.has(currentState)) { pushLiveInstructions(effectiveInstructions()); } } catch (err) { voceroUserProfile = null; voceroUserProfileLoaded = false; syncVoceroHeaderUser(); const message = err instanceof Error ? err.message : String(err); console.warn(`[Vocero USER AUTH] phoneNumber=${JSON.stringify(voceroUserPhoneNumber)} error=${message}`); } } /** Learn server config (search key + connection target), then refresh the UI. */ async function fetchConfig() { try { const res = await fetch("api/config"); if (res.ok) { const json = await res.json(); serverSearchKey = !!json.search; lbMode = !!json.lb; reasoningType = ["OPENAI","NODE_RED"].includes((json.reasoningType || "").toUpperCase()) ? (json.reasoningType || "").toUpperCase() : "PROXY"; // Lock to LB mode only when the deploy reports a load balancer. allowDirect = json.allowDirect ?? !lbMode; // Deploy-pinned direct URL (overrides the LB server-side already). pinnedUrl = (json.s2sUrl || "").trim(); pinnedPrompt = (json.primaryPrompt || "").trim(); pinnedPromptSource = (json.agentProfile || "").trim(); agentToolDefs = normalizeAgentTools(json.agentTools || []); searchMcpEnabled = !!json.searchMcp; rankedAgentTools = rankAgentTools(lastUserQuery); // #region debug-point B:config-loaded reportDebug("B", "main.js:fetchConfig", "loaded config from api/config", { searchMcpEnabled, reasoningType, agentToolNames: agentToolDefs.map((tool) => tool.name), agentToolCount: agentToolDefs.length, pinnedPromptLength: pinnedPrompt.length, }); // #endregion // The conversation-time limiter rides on the LB being present. limiterOn = lbMode; } // Non-OK response: leave the fail-open default (allowDirect = true). } catch { // Config endpoint unreachable (e.g. static hosting): keep direct entry. } if (DEBUG) console.debug(`[ui] config: allowDirect=${allowDirect} lbMode=${lbMode}`); // Login chip + remaining-budget (no-op / hidden when the limiter is off). void account.refresh(); syncToolsUi(); syncConnectionUi(); } /** * Resolve where to connect, per the deploy's mode: * • LB mode -> `{ sessionUrl }`, the client POSTs the same-origin /api/session * proxy and the server forwards to the LB (its address stays server-side). * • direct -> `{ directUrl }`, connect straight to the s2s WebSocket. * Throws a user-facing error if direct mode is on but no URL was entered. * @returns {{ sessionUrl: string } | { directUrl: string }} */ function connectionTarget() { if (!allowDirect) { return { sessionUrl: "api/session" }; } const directUrl = buildDirectWsUrl(pinnedUrl || settings.directUrl); if (!directUrl) { throw new Error("Enter a speech-to-speech server URL in Settings."); } return { directUrl }; } /** * Normalise a user-typed server address into a realtime WebSocket URL. * Accepts bare hosts (`localhost:8080`), http(s) URLs, or ws(s) URLs, and adds * the `/v1/realtime` path when none is given. A full connect URL (with path * and/or query) is preserved as-is. * @param {string} raw @returns {string} */ function buildDirectWsUrl(raw) { let s = (raw || "").trim(); if (!s) return ""; if (!/^wss?:\/\//i.test(s)) { if (/^https?:\/\//i.test(s)) { s = s.replace(/^http/i, "ws"); // http→ws, https→wss } else { const isLocal = /^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(s); s = (isLocal ? "ws://" : "wss://") + s; } } try { const u = new URL(s); if (u.pathname === "" || u.pathname === "/") u.pathname = "/v1/realtime"; return u.toString(); } catch { return s; } } /** Create + resume an AudioContext synchronously (must run inside the user * gesture so iOS lets it start). Returns null if construction fails. */ function createResumedAudioContext() { try { const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext; const ctx = new Ctx({ latencyHint: "interactive" }); if (ctx.state === "suspended") void ctx.resume().catch(() => {}); return /** @type {AudioContext} */ (ctx); } catch (err) { console.warn("[main] AudioContext init failed:", err); return null; } } /** Read the editable settings out of the form. The URL field is only honoured * in free direct mode — in LB mode it's hidden, and when the deploy pins a * URL it's read-only, so the user's saved URL survives either way. */ function readSettingsFromForm() { return { directUrl: allowDirect && !pinnedUrl ? inputLbUrl.value.trim() : settings.directUrl, voice: inputVoice.value || DEFAULT_VOICE, instructions: pinnedPrompt ? settings.instructions : inputInstructions.value.trim() || DEFAULT_INSTRUCTIONS, noiseGate: readGateThreshold(), }; } /** Gate threshold (dBFS) currently shown on the slider, clamped to range. */ function readGateThreshold() { const v = Math.round(Number(inputNoiseGate.value)); if (!Number.isFinite(v)) return GATE_OFF_DB; return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, v)); } /** Adapt the connection field to the mode learned from /api/config. */ function syncConnectionUi() { if (pinnedUrl) { // Deploy-pinned URL: show it, but locked — the deployment owns it. connField.hidden = false; inputLbUrl.value = pinnedUrl; inputLbUrl.readOnly = true; connHint.classList.remove("error"); connHint.textContent = "Speech-to-speech server URL pinned by this deployment."; } else if (allowDirect) { // Direct mode: the user sets their own s2s server URL. connField.hidden = false; inputLbUrl.value = settings.directUrl; inputLbUrl.readOnly = false; inputLbUrl.placeholder = "http://localhost:port"; connHint.classList.remove("error"); connHint.textContent = "URL of your speech-to-speech server, e.g. http://localhost:8080 (the app adds /v1/realtime)."; } else { // LB mode: the load balancer URL is deployment-owned — hide it entirely so // its address is never exposed in Settings. connField.hidden = true; } } /** True when the user must supply a server URL before connecting (direct mode * with nothing set). */ function missingServerUrl() { return allowDirect && !pinnedUrl && !buildDirectWsUrl(settings.directUrl); } /** Open Settings and point the user at the empty server-URL field. */ function promptServerUrl() { if (settingsModal.open) syncConnectionUi(); else openSettings(); connHint.textContent = "Set the speech-to-speech server URL to start."; connHint.classList.add("error"); inputLbUrl.focus(); } settingsForm.addEventListener("submit", (event) => { const submitter = /** @type {HTMLButtonElement | null} */ ((/** @type {SubmitEvent} */ (event)).submitter); if (submitter?.value !== "save") return; settings = readSettingsFromForm(); saveSettings(settings); // Voice + instructions can apply to a live session without reconnecting; a // changed connection URL only takes effect on the next restart. if (client && LIVE_STATES.has(currentState)) { client.updateSession({ voice: settings.voice, instructions: effectiveInstructions() }); } }); // The noise gate applies live (worklet param), so tune it without a restart: // update the label/marker, persist, and push straight to the running client. inputNoiseGate.addEventListener("input", () => { setGateThreshold(readGateThreshold()); }); restartBtn.addEventListener("click", async () => { if (currentState === "connecting") return; // a connect is already underway settings = readSettingsFromForm(); saveSettings(settings); if (missingServerUrl()) { promptServerUrl(); return; } // keep settings open settingsModal.close(); // Grab the AudioContext NOW, inside the click gesture — teardown() awaits, and // creating it afterwards would fall outside the gesture (silent on iOS). const audioContext = createResumedAudioContext(); try { if (client) await teardown(); await doStart(audioContext); } catch (err) { await handleStartError(err); } }); circleBtn.addEventListener("click", async () => { try { if (currentState === "idle" || currentState === "error") { if (missingServerUrl()) { promptServerUrl(); return; } await doStart(); } } catch (err) { await handleStartError(err); } }); /** A failed start is either the daily limit (show the modal, return to idle) or * a real fault (surface it). doStart already closed any orphan AudioContext. * @param {any} err */ async function handleStartError(err) { if (err && err.code === "limit") { await teardown(); account.showLimit(err.tier); return; } // The user left the queue (close() aborted the wait): teardown already reset // the UI to idle, so there's nothing to report. if (err && err.code === "aborted") return; // The whole waiting line is full: a warm, reassuring modal rather than an error. if (err && err.code === "queue-full") { await teardown(); account.showBusy(); return; } // Our place lapsed (ticket reaped, or the join window ran out). Recoverable, not // a fault: land on the retry state with a kind, plain-language reason. if (err && (err.code === "queue-expired" || err.code === "join-expired")) { await teardown(); setState("error"); setCaption( err.code === "join-expired" ? "Your spot expired. Tap to rejoin." : "That took a while. Tap to rejoin.", "error", ); return; } onFatalError(err); } micBtn.addEventListener("click", () => { if (!micStream || !client) return; micMuted = !micMuted; for (const track of micStream.getAudioTracks()) { track.enabled = !micMuted; } client.setMuted(micMuted); micBtn.classList.toggle("muted", micMuted); micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute"); micBtn.title = micMuted ? "Unmute" : "Mute"; }); stopBtn.addEventListener("click", async () => { await teardown(); }); // "Leave queue": tear down the pending connect (aborts the poll wait) and drop // our place in line. Same teardown path as stopping a live call. leaveQueueBtn.addEventListener("click", async () => { await teardown(); }); // "Join now": accept the held slot. The click is a user gesture, so the client // re-resumes the AudioContext here (iOS) before dialing. joinQueueBtn.addEventListener("click", () => { stopJoinCountdown(); if (client) client.join(); }); const MIC_CONSTRAINTS = { audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }; /** Prompt for mic permission up front, then immediately release the tracks so no * recording indicator lingers during a queue wait. Throws a friendly error if the * user denies. */ async function primeMicPermission() { try { const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); for (const track of s.getTracks()) track.stop(); } catch (err) { throw new Error( `Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`, ); } } /** Acquire the live capture stream once a slot is granted. Permission was primed * in the tap gesture, so this is silent. Stored module-side for mute + teardown. */ async function acquireMicStream() { micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); return micStream; } /** @param {number} position Update the queued caption ("You're #N in line"). */ function onQueuePosition(position) { const n = Number(position) || 0; setCaption(n > 0 ? `You're #${n} in line` : "Finding you a spot…", "muted"); } // ── "Your turn" join countdown ────────────────────────────────────────────── // While a slot is held for us, show how long is left to accept it. The client's // join gate expires just before the load balancer reclaims the slot. let joinCountdownTimer = 0; /** @param {number} sec */ function startJoinCountdown(sec) { stopJoinCountdown(); let left = Math.max(0, Math.floor(sec)); const paint = () => { joinQueueBtn.textContent = left > 0 ? `Join now (${left}s)` : "Join now"; }; paint(); joinCountdownTimer = window.setInterval(() => { left -= 1; if (left <= 0) { stopJoinCountdown(); joinQueueBtn.textContent = "Join now"; return; } paint(); }, 1000); } function stopJoinCountdown() { if (joinCountdownTimer) { clearInterval(joinCountdownTimer); joinCountdownTimer = 0; } } /** * Start a conversation. Pass a pre-created AudioContext when the caller already * made one inside the tap/click gesture (required on iOS); otherwise one is * created here, which is still inside the gesture for a direct orb tap. * @param {AudioContext | null} [audioContext] */ async function doStart(audioContext = null) { // Resolve the target before touching mic/audio so a misconfiguration (e.g. // direct mode with no URL) fails fast with a clear message. const target = connectionTarget(); chat.clear(); chat.reset(); setState("connecting"); setCaption("Asking for mic…", "muted"); // Create + resume the AudioContext SYNCHRONOUSLY, still inside the gesture. // iOS Safari only starts an AudioContext from a user gesture; if we waited // until after the getUserMedia / session-creation awaits below, it would stay // suspended and the whole pipeline would be silent. if (!audioContext) audioContext = createResumedAudioContext(); // Prime the mic permission now (get the prompt out of the way up front), then // release it. The real capture stream is acquired only once a slot is granted // (see acquireMicStream), so the mic 'in use' indicator never lights while we // sit in the queue. Permission persists, so the later acquire is silent. try { await primeMicPermission(); } catch (err) { if (audioContext) void audioContext.close().catch(() => {}); throw err; } const startInstructions = effectiveInstructions(); const startTools = activeToolDefs(); // #region debug-point B:client-construction reportDebug("B", "main.js:doStart", "constructing realtime client", { toolNames: startTools.map((tool) => tool.name), toolCount: startTools.length, instructionLength: startInstructions.length, searchMcpEnabled, }); // #endregion const c = new S2sWsRealtimeClient({ ...target, voice: settings.voice, instructions: startInstructions, acquireMic: acquireMicStream, tools: startTools, noiseGate: gateParams(settings.noiseGate), ...(audioContext ? { audioContext } : {}), }); client = c; liveInstructionsSnapshot = startInstructions; c.addEventListener("queue", (e) => { const { position, queueId } = /** @type {CustomEvent<{ position: number; queueId: string }>} */ (e).detail; if (queueId) queuedTicketId = queueId; onQueuePosition(position); }); c.addEventListener("ready-to-join", (e) => { const { info, expiresSec } = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo; expiresSec: number }>} */ (e).detail; // A slot is held for us. We're out of the queue now, so drop the ticket ref. // Track the granted session id already so that leaving (or letting the timer // lapse) refunds the budget the server reserved at claim, even before we dial. queuedTicketId = ""; if (info?.sessionId) { trackedSessionId = info.sessionId; trackedTier = info.tier || "anon"; } startJoinCountdown(expiresSec); }); c.addEventListener("status", (e) => { const detail = /** @type {CustomEvent<{ status: string }>} */ (e).detail; _kickIdleTimer(`client-status:${detail.status}`); onClientStatus(detail.status); }); c.addEventListener("transcript", (e) => { const d = /** @type {CustomEvent<{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }>} */ (e).detail; _kickIdleTimer(`transcript-${d.role}${d.partial ? ":partial" : ""}`); chat.onTranscript(d); if (d.role === "user") { updateMcpToolSuggestions(d.text || "", { partial: d.partial, itemId: d.itemId }); if (!d.partial) void maybeRunOpenAiReasoning(d.text || "", d.itemId || ""); } if (d.role === "assistant" && !d.partial) maybeRunDeclaredAssistantTool(d.text || "", d.responseId || ""); }); c.addEventListener("response-finished", (e) => { const detail = /** @type {CustomEvent<{ responseId: string; status: string; audible?: boolean; transcript?: string }>} */ (e).detail; _kickIdleTimer(`response-finished:${detail.status || "done"}`); chat.onResponseFinished(detail); maybeRunDeclaredAssistantTool(detail.transcript || "", detail.responseId || ""); }); c.addEventListener("toolcall", (e) => { const { name, arguments: args, callId } = /** @type {CustomEvent<{ name: string; arguments: string; callId: string }>} */ (e).detail; _kickIdleTimer(`toolcall:${name}`); clearPendingAutoMcpFallback(); // #region debug-point D:toolcall-event reportDebug("D", "main.js:client.toolcall", "toolcall event received by main app", { name, callId, args, knownMcpTool: agentToolDefs.some((tool) => tool.name === name), }); // #endregion if (agentToolDefs.some((tool) => tool.name === name)) { console.debug(`[Vocero MCP CALL] name=${name} callId=${JSON.stringify(callId)} args=${args}`); } chat.onToolCall(name); // Execute the tool, then push it to the conversation once the result is in, // so the toggle shows both the call input and its output together. void runTool(name, args, callId).then(({ output, image }) => { _kickIdleTimer(`tool-result:${name}`); chat.onToolResult(name, args, output, image); }); }); c.addEventListener("error", (e) => { const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail; onFatalError(detail.error); }); c.addEventListener("server-error", (e) => { // Non-fatal: the backend reported an error mid-session. Log it, keep the // socket and the conversation alive (the model can recover on its own). const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail; const msg = detail.error instanceof Error ? detail.error.message : String(detail.error); console.warn("[main] server error (non-fatal):", msg); }); c.addEventListener("session", (e) => { const info = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo }>} */ (e).detail.info; console.log("[ws] session created:", info.sessionId); // A slot was granted — we're out of the queue; drop the ticket reference so // teardown doesn't try to leave a line we already left. queuedTicketId = ""; // A metered tier (anon / free): heartbeat so the server can extend the // reservation and tell us when the daily budget runs out. PRO isn't limited. if (info.limited && info.sessionId) { trackedSessionId = info.sessionId; trackedTier = info.tier || "anon"; startHeartbeat(info.heartbeatSec || 5); } }); c.addEventListener("input-level", (e) => { const { rms } = /** @type {CustomEvent<{ rms: number }>} */ (e).detail; paintInputLevel(rms); // Proxy for "user is speaking" before VAD/STT emits the first transcript // delta. RMS > ~-52 dBFS (≈ 0.0025 linear) consistently means actual // speech or close enough to a real utterance that the 10s idle window // should restart. Threshold is generous (≈ GATE_DEFAULT_DB - 2dB) so even // quiet speakers keep the session alive; linear fast path avoids log10 // on the hot 50 Hz level path. if (rms > 0.0025) _kickIdleTimer("input-level"); }); try { await c.connect(); } catch (err) { // The grant can be refused (402 → limit) or the dial can fail. In LB mode // the AudioContext hasn't been adopted by the client yet (the session POST // runs first), so close the one we created here to avoid leaking it. if (audioContext) void audioContext.close().catch(() => {}); throw err; } } // ── Conversation-time heartbeat ───────────────────────────────────────────── /** Ping the server every `sec` seconds so it can meter the live session; when * it reports the daily budget is spent, cut the call and show the limit modal. * @param {number} sec */ function startHeartbeat(sec) { stopHeartbeat(); heartbeatTimer = window.setInterval(async () => { if (!trackedSessionId) return; try { const res = await fetch("api/session/heartbeat", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: trackedSessionId }), keepalive: true, }); const json = await res.json().catch(() => ({})); if (json.expired) await onLimitReached(); } catch (err) { // A transient network blip shouldn't kill the call; the next tick retries. if (DEBUG) console.debug("[ui] heartbeat failed:", err); } }, Math.max(1, sec) * 1000); } function stopHeartbeat() { if (heartbeatTimer) { clearInterval(heartbeatTimer); heartbeatTimer = 0; } } /** The server cut the live session: tear down and explain why. */ async function onLimitReached() { const tier = trackedTier; stopHeartbeat(); await teardown(); account.showLimit(tier); } /** Tell the server a session ended so it reconciles + refunds the unused chunk. * Uses sendBeacon so it still fires when the tab is closing. */ function endTrackedSession() { if (!trackedSessionId) return; const body = JSON.stringify({ sessionId: trackedSessionId }); try { const blob = new Blob([body], { type: "application/json" }); if (!navigator.sendBeacon("api/session/end", blob)) { void fetch("api/session/end", { method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, }).catch(() => {}); } } catch { // Best-effort; the server sweep reaps the session anyway. } trackedSessionId = ""; trackedTier = ""; } /** Leave the waiting queue so the LB frees our place. sendBeacon so it still * fires on tab close; the LB also reaps the ticket on TTL as a backstop. */ function endQueueTicket() { if (!queuedTicketId) return; const body = JSON.stringify({ queueId: queuedTicketId }); try { const blob = new Blob([body], { type: "application/json" }); if (!navigator.sendBeacon("api/queue/end", blob)) { void fetch("api/queue/end", { method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, }).catch(() => {}); } } catch { // Best-effort; the LB reaps the ticket on TTL anyway. } queuedTicketId = ""; } /** @param {string} status */ function onClientStatus(status) { switch (status) { case "creating-session": case "connecting": setState("connecting"); break; case "queued": setState("queued"); break; case "your-turn": setState("your-turn"); break; case "connected": setState("listening"); // Wake word confirmation: if this session was triggered by the "Hola // Vocero" hot-word, inject the phrase into the conversation exactly as // though the user had spoken it, then ask the model to reply so the // assistant greets back instead of silently waiting for the next turn. if (pendingWakeWordConfirmationText && client) { const phrase = pendingWakeWordConfirmationText.trim() || "hola vocero"; pendingWakeWordConfirmationText = ""; console.info(`[WAKE WORD] 📤 Injecting confirmation phrase to ai-server: ${JSON.stringify(phrase)}`); try { client.sendUserText(phrase); client.requestResponse(); } catch (err) { console.warn("[WAKE WORD] failed to inject confirmation phrase:", err && err instanceof Error ? err.message : err); } } break; case "user-speaking": setState("user-speaking"); break; case "processing": setState("processing"); break; case "ai-speaking": setState("ai-speaking"); break; case "closed": // teardown() will move us to idle break; case "error": setState("error"); break; } } async function teardown() { clearPendingAutoMcpFallback(); stopHeartbeat(); stopJoinCountdown(); endTrackedSession(); endQueueTicket(); chat.reset({ dismiss: true }); updateMcpToolSuggestions(""); liveInstructionsSnapshot = ""; liveToolPolicySnapshot = ""; // If teardown fires *before* the first "connected" status (e.g. failed dial, // user left queue, limit exceeded) the pending wake-word confirmation would // be re-injected on the next session. Clear it so we never double-fire. pendingWakeWordConfirmationText = ""; if (client) { try { await client.close(); } catch (err) { console.warn("[main] error closing client:", err); } client = null; } if (micStream) { for (const track of micStream.getTracks()) track.stop(); micStream = null; } micMuted = false; micBtn.classList.remove("muted"); setState("idle"); // Refresh the chip's remaining-today after the budget moved. if (limiterOn) void account.refresh(); } /** @param {unknown} err */ function onFatalError(err) { console.error("[main] fatal:", err); setState("error"); const message = err instanceof Error ? err.message : String(err); setCaption(truncateError(message), "error"); void teardown().catch(() => { setState("error"); setCaption(truncateError(message), "error"); }); } setState("idle"); chat.renderEmptyState(); initGateArc(); void fetchConfig(); void fetchVoceroUserProfile(); (() => { const toolCamRow = document.getElementById("tool-cam-row"); if (toolCamRow) toolCamRow.setAttribute("hidden", ""); if (toolCamHint) toolCamHint.setAttribute("hidden", ""); if (camPip) camPip.setAttribute("hidden", ""); if (camVideo) camVideo.setAttribute("hidden", ""); })(); // Reconcile a live session if the tab is closed/hidden mid-call (no teardown). window.addEventListener("pagehide", () => { endTrackedSession(); endQueueTicket(); }); requestAnimationFrame(() => { document.body.classList.remove("booting"); });