client sdk
This commit is contained in:
+976
@@ -0,0 +1,976 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>audio-agent — SDK demo</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0b0f1a;
|
||||
--panel: #141b2d;
|
||||
--accent: #4f8cff;
|
||||
--accent-dim: #2a4a8a;
|
||||
--ok: #3ddc97;
|
||||
--warn: #ffb454;
|
||||
--err: #ff5470;
|
||||
--text: #e6edf7;
|
||||
--muted: #8a94a6;
|
||||
--border: #243049;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html, body { height: 100%; }
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 18px;
|
||||
}
|
||||
.app {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
max-width: 900px;
|
||||
height: calc(100vh - 36px);
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
overflow: hidden;
|
||||
}
|
||||
header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
padding: 12px 16px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
header h1 { font-size: 15px; margin: 0; white-space: nowrap; }
|
||||
header h1 span { color: var(--muted); font-weight: 400; }
|
||||
.status { display: inline-flex; align-items: center; gap: 7px; font-size: 12px; color: var(--muted); }
|
||||
.dot { width: 9px; height: 9px; border-radius: 50%; background: var(--muted); flex: none; }
|
||||
.dot.on { background: var(--ok); box-shadow: 0 0 8px var(--ok); }
|
||||
.dot.err { background: var(--err); box-shadow: 0 0 8px var(--err); }
|
||||
.dot.warn { background: var(--warn); box-shadow: 0 0 8px var(--warn); }
|
||||
/* Authenticated user name, shown in the header after phone-login. */
|
||||
#userStatus {
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
background: var(--accent-dim);
|
||||
border: 1px solid var(--accent);
|
||||
border-radius: 999px;
|
||||
padding: 3px 11px;
|
||||
max-width: 220px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
#userStatus.show { display: inline-flex; }
|
||||
#userStatus::before { content: "\1F464"; font-size: 12px; line-height: 1; }
|
||||
#userStatus.bad { background: transparent; border-color: var(--err); color: var(--err); }
|
||||
#userStatus.bad::before { content: "\26A0"; }
|
||||
input[type="text"], input[type="url"], input[type="number"], textarea {
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
color: var(--text);
|
||||
padding: 8px 10px;
|
||||
font-size: 13px;
|
||||
font-family: inherit;
|
||||
outline: none;
|
||||
min-width: 0;
|
||||
}
|
||||
input:focus, textarea:focus { border-color: var(--accent); }
|
||||
textarea { width: 100%; resize: vertical; min-height: 68px; line-height: 1.4; }
|
||||
#promptPanel { padding: 0 16px 12px; display: flex; flex-direction: column; gap: 8px; }
|
||||
#promptPanel .meta { font-size: 11.5px; color: var(--muted); }
|
||||
#promptPanel .meta b { color: var(--text); font-weight: 600; }
|
||||
button {
|
||||
background: var(--accent-dim);
|
||||
color: var(--text);
|
||||
border: 1px solid var(--accent);
|
||||
border-radius: 8px;
|
||||
padding: 8px 13px;
|
||||
font-size: 13px;
|
||||
font-family: inherit;
|
||||
cursor: pointer;
|
||||
transition: all .15s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
button:hover:not(:disabled) { background: var(--accent); }
|
||||
button:disabled { opacity: .4; cursor: not-allowed; }
|
||||
button.ghost { background: transparent; border-color: var(--border); }
|
||||
button.ghost:hover:not(:disabled) { background: var(--border); }
|
||||
button.danger { border-color: var(--err); background: rgba(255,84,112,.15); }
|
||||
button.danger:hover:not(:disabled) { background: var(--err); }
|
||||
button.active { background: var(--ok); border-color: var(--ok); color: #06170f; }
|
||||
|
||||
.banner {
|
||||
display: none;
|
||||
margin: 10px 16px 0;
|
||||
padding: 9px 12px;
|
||||
font-size: 12.5px;
|
||||
line-height: 1.4;
|
||||
border: 1px solid var(--warn);
|
||||
border-radius: 8px;
|
||||
background: rgba(255,180,84,.08);
|
||||
color: var(--warn);
|
||||
}
|
||||
.banner.show { display: block; }
|
||||
|
||||
#chat {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
.hint { color: var(--muted); font-size: 13px; text-align: center; margin: auto; }
|
||||
.msg { display: flex; flex-direction: column; gap: 4px; max-width: 82%; }
|
||||
.msg .who { font-size: 10.5px; text-transform: uppercase; letter-spacing: .5px; color: var(--muted); }
|
||||
.msg .bubble {
|
||||
padding: 9px 12px;
|
||||
border-radius: 12px;
|
||||
font-size: 14px;
|
||||
line-height: 1.45;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
.msg.user { align-self: flex-end; align-items: flex-end; }
|
||||
.msg.user .bubble { background: var(--accent-dim); border: 1px solid var(--accent); }
|
||||
.msg.agent { align-self: flex-start; }
|
||||
.msg.agent .bubble { background: var(--bg); border: 1px solid var(--border); }
|
||||
.msg.error .bubble { border-color: var(--err); color: var(--err); }
|
||||
.msg .kw { color: var(--muted); transition: color .18s ease; }
|
||||
.msg .kw.done { color: var(--text); }
|
||||
.msg .kw.now { color: var(--accent); text-shadow: 0 0 8px rgba(79,140,255,.55); }
|
||||
.msg .bubble.markdown { white-space: normal; overflow-x: auto; }
|
||||
.msg .bubble.markdown > :first-child { margin-top: 0; }
|
||||
.msg .bubble.markdown > :last-child { margin-bottom: 0; }
|
||||
.msg .bubble.markdown h1,
|
||||
.msg .bubble.markdown h2,
|
||||
.msg .bubble.markdown h3,
|
||||
.msg .bubble.markdown h4 { margin: 0 0 10px; line-height: 1.25; }
|
||||
.msg .bubble.markdown p,
|
||||
.msg .bubble.markdown ul,
|
||||
.msg .bubble.markdown ol,
|
||||
.msg .bubble.markdown blockquote,
|
||||
.msg .bubble.markdown pre,
|
||||
.msg .bubble.markdown table { margin: 0 0 10px; }
|
||||
.msg .bubble.markdown ul,
|
||||
.msg .bubble.markdown ol { padding-left: 20px; }
|
||||
.msg .bubble.markdown li + li { margin-top: 4px; }
|
||||
.msg .bubble.markdown code {
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
font-size: 12.5px;
|
||||
background: rgba(79,140,255,.10);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
padding: 1px 5px;
|
||||
}
|
||||
.msg .bubble.markdown pre {
|
||||
background: #0a1222;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 10px 12px;
|
||||
overflow-x: auto;
|
||||
}
|
||||
.msg .bubble.markdown pre code {
|
||||
background: transparent;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
font-size: 12px;
|
||||
}
|
||||
.msg .bubble.markdown blockquote {
|
||||
border-left: 3px solid var(--accent);
|
||||
padding-left: 10px;
|
||||
color: var(--muted);
|
||||
}
|
||||
.msg .bubble.markdown table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 13px;
|
||||
}
|
||||
.msg .bubble.markdown th,
|
||||
.msg .bubble.markdown td {
|
||||
border: 1px solid var(--border);
|
||||
padding: 6px 8px;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
.msg .bubble.markdown th { color: var(--text); background: rgba(79,140,255,.10); }
|
||||
.msg .bubble.markdown a { color: #8db6ff; }
|
||||
|
||||
/* Inline chips for cue / tool_call / audio_source events. */
|
||||
.chips { display: flex; flex-wrap: wrap; gap: 6px; align-self: flex-start; }
|
||||
.chip {
|
||||
font-size: 11.5px;
|
||||
color: var(--muted);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
padding: 3px 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.chip b { color: var(--text); font-weight: 600; }
|
||||
.chip.cue { border-color: var(--accent); color: var(--accent); }
|
||||
.chip.tool { border-color: var(--ok); color: var(--ok); }
|
||||
.chip.flash { animation: flash .5s ease; }
|
||||
@keyframes flash { from { background: rgba(255,84,112,.35); } to { background: transparent; } }
|
||||
|
||||
footer { border-top: 1px solid var(--border); padding: 10px 16px; display: flex; flex-direction: column; gap: 9px; }
|
||||
.row { display: flex; align-items: center; gap: 8px; }
|
||||
.row.grow input[type="text"] { flex: 1; }
|
||||
.meters { display: flex; align-items: center; gap: 12px; font-size: 12px; color: var(--muted); flex-wrap: wrap; }
|
||||
.bargein { display: inline-flex; align-items: center; gap: 6px; border: 1px solid var(--border); border-radius: 999px; padding: 4px 10px; transition: border-color .2s, color .2s, box-shadow .2s; }
|
||||
.bargein.active { border-color: var(--err); color: var(--err); box-shadow: 0 0 12px -2px var(--err); }
|
||||
.bargein b { color: var(--text); font-variant-numeric: tabular-nums; }
|
||||
.mic { display: inline-flex; align-items: center; gap: 6px; border: 1px solid var(--border); border-radius: 999px; padding: 4px 10px; }
|
||||
.mic .bar { display: inline-flex; gap: 2px; align-items: flex-end; height: 12px; }
|
||||
.mic .bar i { width: 3px; background: var(--muted); border-radius: 1px; height: 3px; transition: height .08s; }
|
||||
.mic.on .bar i { background: var(--ok); }
|
||||
|
||||
details { border-top: 1px solid var(--border); }
|
||||
details summary { cursor: pointer; padding: 8px 16px; font-size: 12px; color: var(--muted); user-select: none; }
|
||||
#log { max-height: 190px; overflow-y: auto; padding: 0 16px 12px; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 11.5px; color: var(--muted); }
|
||||
#log div { padding: 1px 0; white-space: pre-wrap; word-break: break-all; }
|
||||
#log .l-user { color: var(--text); }
|
||||
#log .l-cue { color: var(--accent); }
|
||||
#log .l-tool { color: var(--ok); }
|
||||
#log .l-err { color: var(--err); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app">
|
||||
<header>
|
||||
<h1>audio-agent <span>· SDK demo</span></h1>
|
||||
<span class="status"><i class="dot" id="dot"></i><span id="state">disconnected</span></span>
|
||||
<span class="status" id="userStatus"></span>
|
||||
<span style="flex:1"></span>
|
||||
<input type="text" id="baseUrl" style="width: 230px" spellcheck="false" />
|
||||
<button id="connectBtn">Connect</button>
|
||||
</header>
|
||||
|
||||
<div class="banner" id="insecure">
|
||||
The microphone requires a <b>secure context</b>: open this page over <code>https://</code> or
|
||||
<code>localhost</code>, not from <code>file://</code>.
|
||||
</div>
|
||||
|
||||
<div id="chat"><div class="hint" id="hint">Connect to start talking to the agent.</div></div>
|
||||
|
||||
<footer>
|
||||
<div class="row grow">
|
||||
<input type="text" id="textInput" placeholder="Type a message (text turn)…" autocomplete="off" />
|
||||
<button id="sendBtn" disabled>Send</button>
|
||||
</div>
|
||||
|
||||
<div class="row grow">
|
||||
<input type="url" id="sourceUrl" placeholder="External audio source URL (radio / stream)…" style="flex:1" spellcheck="false" />
|
||||
<button id="sourcePlay" class="ghost" disabled>Play</button>
|
||||
<button id="sourceStop" class="ghost" disabled>Stop</button>
|
||||
<input type="range" id="sourceVol" min="0" max="100" value="70" title="External source volume" disabled style="width:90px" />
|
||||
</div>
|
||||
|
||||
<div class="meters">
|
||||
<button id="micBtn" disabled>Microphone: OFF</button>
|
||||
<span class="mic" id="micMeter"><span>mic</span><span class="bar"><i></i><i></i><i></i><i></i><i></i></span></span>
|
||||
<button id="interruptBtn" class="danger" disabled>Interrupt</button>
|
||||
<span class="bargein" id="bargein"><span>barge-in</span><b id="bargeCount">0</b></span>
|
||||
<span style="flex:1"></span>
|
||||
<label title="Local playback buffer" style="display:inline-flex;align-items:center;gap:6px">
|
||||
buffer <input type="number" id="bufferMs" value="150" min="20" max="1000" step="10" style="width:70px" /> ms
|
||||
</label>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
<details id="promptDetails">
|
||||
<summary>System prompt (per session)</summary>
|
||||
<div id="promptPanel">
|
||||
<div class="meta">
|
||||
session <b id="sessionId">—</b> · source <b id="promptSource">—</b> ·
|
||||
override <b id="promptOverride">—</b>
|
||||
</div>
|
||||
<textarea id="promptText" spellcheck="false"
|
||||
placeholder="Connect to load the session's system prompt…" disabled></textarea>
|
||||
<div class="row">
|
||||
<button id="promptLoad" class="ghost" disabled>Reload</button>
|
||||
<button id="promptApply" disabled>Apply override</button>
|
||||
<button id="promptClear" class="ghost" disabled>Clear override</button>
|
||||
<span style="flex:1"></span>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Event log (raw SDK events)</summary>
|
||||
<div id="log"></div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<script type="module">
|
||||
import { AudioSession, AudioAgentRest, MicCapture, AUDIO } from './audio-agent-client.js';
|
||||
|
||||
/* System prompt applied automatically to every new session (per-session
|
||||
override via POST /api/prompt). Edit it freely, or clear the override from the
|
||||
"System prompt" panel below. */
|
||||
const SESSION_PROMPT = `You are Vocero, a friendly, proactive AI voice assistant for AquaChile, helping users understand operational data, KPIs, reports and alerts.
|
||||
Always speak exclusively in Spanish, regardless of the user's language. Use natural, professional Chilean Spanish.
|
||||
Keep spoken answers short, usually one to three sentences, without markdown, lists or unnecessary details.
|
||||
Prioritize relevant information, interpret verified data and suggest actions when appropriate. Never invent facts or figures.
|
||||
Maintain conversation context, especially when initiated from an alert or report. Provide deeper explanations only when requested.
|
||||
Use the available tools immediately on the first relevant user request instead of replying generically.
|
||||
If the user asks for today's podcast, the daily podcast, the MP3, the audio file, or asks to play or listen to the podcast, call get_today_podcast_mp3 right away.
|
||||
If the user asks for today's report, the day report, the daily briefing, alerts, or operational highlights for a date, call get_public_day_report right away.
|
||||
When a tool can provide the requested data or media, call it before answering. Do not ask the user to repeat the request just to trigger a tool.
|
||||
If the user explicitly asks for a Markdown document, a report in Markdown, or a long structured written document, first give a very short spoken introduction outside the block, then put the full Markdown document inside <chat_markdown>...</chat_markdown>.
|
||||
Only the Markdown document itself may appear inside <chat_markdown>...</chat_markdown>. Do not repeat the full document outside that block.
|
||||
Answer directly without revealing reasoning or internal processes.`;
|
||||
|
||||
/* Runtime tools installed by the demo through `POST /api/tools`. The backend
|
||||
executes them server-side; the browser only installs the definitions and
|
||||
observes `tool_call` events over WebSocket. */
|
||||
const DEMO_TOOLS = [
|
||||
{
|
||||
name: 'get_public_day_report',
|
||||
description: 'Use this immediately when the user asks for a day report, daily briefing, alerts, operational highlights, or a report for a specific date. Returns the AquaChile public day briefing ordered by severity and recency.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
limit: {
|
||||
type: 'integer',
|
||||
description: 'Maximum number of insights to return after severity and recency ordering.',
|
||||
minimum: 1,
|
||||
maximum: 10,
|
||||
default: 3,
|
||||
},
|
||||
date: {
|
||||
type: 'string',
|
||||
description: 'Exact report date in YYYY-MM-DD format. Omit it to use today or the latest earlier processed report date.',
|
||||
},
|
||||
},
|
||||
},
|
||||
url: 'https://aquachile.azurianlab.com/api/v1/day-report',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
accept: '*/*',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'get_today_podcast_mp3',
|
||||
description: 'Use this immediately when the user asks to listen to, play, open, or get today\'s AquaChile podcast or its MP3/audio file. Returns the MP3 file of the latest published daily AquaChile podcast episode.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
download: {
|
||||
type: 'boolean',
|
||||
description: 'When true (default) the MP3 is returned as a download attachment. When false it is served inline for playback.',
|
||||
default: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
url: 'https://aquachile.azurianlab.com/api/v1/podcast/today/mp3',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
accept: '*/*',
|
||||
},
|
||||
// `audio: true` marks a tool whose result is a media file. The backend
|
||||
// ignores this extra field; the demo uses it to route the call's URL to the
|
||||
// engine's external audio source and start playback.
|
||||
audio: true,
|
||||
},
|
||||
];
|
||||
|
||||
/* Verbs whose arguments travel as query parameters (same rule the backend
|
||||
applies), so the demo can rebuild the exact URL a tool call used. */
|
||||
const QUERY_VERBS = new Set(['GET', 'HEAD', 'OPTIONS', 'TRACE', 'DELETE']);
|
||||
|
||||
/** Effective URL of a tool call, mirroring the backend's argument placement. */
|
||||
function toolUrl(def, args = {}) {
|
||||
const url = new URL(def.url);
|
||||
if (QUERY_VERBS.has((def.method || 'POST').toUpperCase())) {
|
||||
for (const [k, v] of Object.entries(args || {})) {
|
||||
if (v !== undefined && v !== null) url.searchParams.set(k, String(v));
|
||||
}
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
/* Phone-login: the page reads `phoneNumber` from its URL at session start and
|
||||
authenticates through the engine's server-side proxy (`POST
|
||||
/api/auth/phone-login`), which forwards to the AquaChile auth API. The proxy
|
||||
exists because that API blocks browser (CORS) calls from this origin. */
|
||||
|
||||
/* ------------------------------------------------------------------ helpers */
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const chat = $('chat');
|
||||
const hint = $('hint');
|
||||
const logEl = $('log');
|
||||
|
||||
function log(kind, text) {
|
||||
const line = document.createElement('div');
|
||||
line.className = 'l-' + kind;
|
||||
line.textContent = `${new Date().toLocaleTimeString()} ${text}`;
|
||||
logEl.appendChild(line);
|
||||
logEl.scrollTop = logEl.scrollHeight;
|
||||
}
|
||||
|
||||
function setState(text, cls) {
|
||||
$('state').textContent = text;
|
||||
$('dot').className = 'dot' + (cls ? ' ' + cls : '');
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
return s.replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c]));
|
||||
}
|
||||
|
||||
/** Append a bubble to the chat (scrolls to the bottom). */
|
||||
function addBubble(who, cls) {
|
||||
hint.remove();
|
||||
const wrap = document.createElement('div');
|
||||
wrap.className = 'msg ' + cls;
|
||||
const label = document.createElement('div');
|
||||
label.className = 'who';
|
||||
label.textContent = who;
|
||||
const bubble = document.createElement('div');
|
||||
bubble.className = 'bubble';
|
||||
wrap.append(label, bubble);
|
||||
chat.appendChild(wrap);
|
||||
chat.scrollTop = chat.scrollHeight;
|
||||
return bubble;
|
||||
}
|
||||
|
||||
function addChip(cls, html) {
|
||||
hint.remove();
|
||||
let chips = chat.lastElementChild;
|
||||
if (!chips || !chips.classList.contains('chips')) {
|
||||
chips = document.createElement('div');
|
||||
chips.className = 'chips';
|
||||
chat.appendChild(chips);
|
||||
}
|
||||
const chip = document.createElement('span');
|
||||
chip.className = 'chip ' + cls;
|
||||
chip.innerHTML = html;
|
||||
chips.appendChild(chip);
|
||||
chat.scrollTop = chat.scrollHeight;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ karaoke render */
|
||||
// The engine streams the reply in `assistant_delta` chunks; the words of the
|
||||
// newest chunk are highlighted until the next one arrives.
|
||||
function renderKaraoke(bubble, fullText, latest) {
|
||||
const words = fullText.split(/(\s+)/);
|
||||
const latestWords = new Set(latest.trim().split(/\s+/).filter(Boolean));
|
||||
bubble.innerHTML = words
|
||||
.map((w) => {
|
||||
if (!w.trim()) return escapeHtml(w);
|
||||
const cls = latestWords.has(w) ? 'now' : 'done';
|
||||
return `<span class="kw ${cls}">${escapeHtml(w)}</span>`;
|
||||
})
|
||||
.join('');
|
||||
}
|
||||
|
||||
function renderInlineMarkdown(text) {
|
||||
let html = escapeHtml(text);
|
||||
html = html.replace(/`([^`]+)`/g, '<code>$1</code>');
|
||||
html = html.replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>');
|
||||
html = html.replace(/\*([^*]+)\*/g, '<em>$1</em>');
|
||||
html = html.replace(/\[([^\]]+)\]\((https?:\/\/[^\s)]+)\)/g, '<a href="$2" target="_blank" rel="noreferrer noopener">$1</a>');
|
||||
return html;
|
||||
}
|
||||
|
||||
function markdownTableCells(line) {
|
||||
return line.trim().replace(/^\||\|$/g, '').split('|').map((cell) => cell.trim());
|
||||
}
|
||||
|
||||
function renderMarkdown(text) {
|
||||
const lines = String(text || '').replace(/\r\n?/g, '\n').split('\n');
|
||||
const html = [];
|
||||
let i = 0;
|
||||
|
||||
while (i < lines.length) {
|
||||
const line = lines[i];
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
const fence = line.match(/^```(\w+)?\s*$/);
|
||||
if (fence) {
|
||||
const lang = fence[1] ? ` class="language-${escapeHtml(fence[1])}"` : '';
|
||||
const body = [];
|
||||
i++;
|
||||
while (i < lines.length && !/^```/.test(lines[i])) {
|
||||
body.push(lines[i]);
|
||||
i++;
|
||||
}
|
||||
if (i < lines.length) i++;
|
||||
html.push(`<pre><code${lang}>${escapeHtml(body.join('\n'))}</code></pre>`);
|
||||
continue;
|
||||
}
|
||||
const heading = line.match(/^(#{1,4})\s+(.*)$/);
|
||||
if (heading) {
|
||||
const level = heading[1].length;
|
||||
html.push(`<h${level}>${renderInlineMarkdown(heading[2].trim())}</h${level}>`);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (trimmed.includes('|') && i + 1 < lines.length && /^[\s|:-]+$/.test(lines[i + 1].trim())) {
|
||||
const head = markdownTableCells(line);
|
||||
const rows = [];
|
||||
i += 2;
|
||||
while (i < lines.length && lines[i].includes('|') && lines[i].trim()) {
|
||||
rows.push(markdownTableCells(lines[i]));
|
||||
i++;
|
||||
}
|
||||
const thead = `<thead><tr>${head.map((cell) => `<th>${renderInlineMarkdown(cell)}</th>`).join('')}</tr></thead>`;
|
||||
const tbody = rows.length
|
||||
? `<tbody>${rows.map((row) => `<tr>${row.map((cell) => `<td>${renderInlineMarkdown(cell)}</td>`).join('')}</tr>`).join('')}</tbody>`
|
||||
: '';
|
||||
html.push(`<table>${thead}${tbody}</table>`);
|
||||
continue;
|
||||
}
|
||||
if (/^\s*>\s?/.test(line)) {
|
||||
const quote = [];
|
||||
while (i < lines.length && /^\s*>\s?/.test(lines[i])) {
|
||||
quote.push(lines[i].replace(/^\s*>\s?/, ''));
|
||||
i++;
|
||||
}
|
||||
html.push(`<blockquote>${quote.map((part) => renderInlineMarkdown(part)).join('<br>')}</blockquote>`);
|
||||
continue;
|
||||
}
|
||||
if (/^\s*[-*+]\s+/.test(line)) {
|
||||
const items = [];
|
||||
while (i < lines.length && /^\s*[-*+]\s+/.test(lines[i])) {
|
||||
items.push(lines[i].replace(/^\s*[-*+]\s+/, ''));
|
||||
i++;
|
||||
}
|
||||
html.push(`<ul>${items.map((item) => `<li>${renderInlineMarkdown(item)}</li>`).join('')}</ul>`);
|
||||
continue;
|
||||
}
|
||||
if (/^\s*\d+\.\s+/.test(line)) {
|
||||
const items = [];
|
||||
while (i < lines.length && /^\s*\d+\.\s+/.test(lines[i])) {
|
||||
items.push(lines[i].replace(/^\s*\d+\.\s+/, ''));
|
||||
i++;
|
||||
}
|
||||
html.push(`<ol>${items.map((item) => `<li>${renderInlineMarkdown(item)}</li>`).join('')}</ol>`);
|
||||
continue;
|
||||
}
|
||||
const para = [];
|
||||
while (
|
||||
i < lines.length &&
|
||||
lines[i].trim() &&
|
||||
!/^```/.test(lines[i]) &&
|
||||
!/^(#{1,4})\s+/.test(lines[i]) &&
|
||||
!/^\s*>\s?/.test(lines[i]) &&
|
||||
!/^\s*[-*+]\s+/.test(lines[i]) &&
|
||||
!/^\s*\d+\.\s+/.test(lines[i]) &&
|
||||
!(lines[i].includes('|') && i + 1 < lines.length && /^[\s|:-]+$/.test(lines[i + 1].trim()))
|
||||
) {
|
||||
para.push(lines[i].trim());
|
||||
i++;
|
||||
}
|
||||
html.push(`<p>${renderInlineMarkdown(para.join(' '))}</p>`);
|
||||
}
|
||||
|
||||
return html.join('') || `<p>${renderInlineMarkdown(text)}</p>`;
|
||||
}
|
||||
|
||||
function addMarkdownBubble(text) {
|
||||
const bubble = addBubble('agent', 'agent');
|
||||
bubble.classList.add('markdown');
|
||||
bubble.innerHTML = renderMarkdown(text);
|
||||
return bubble;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------------- state */
|
||||
let session = null;
|
||||
let rest = null;
|
||||
let sessionId = null;
|
||||
let userData = null;
|
||||
let mic = null;
|
||||
let micAnalyser = null;
|
||||
let micRaf = 0;
|
||||
let agentBubble = null;
|
||||
let agentText = '';
|
||||
let bargeCount = 0;
|
||||
let pendingExternalSourceUrl = '';
|
||||
let agentTextEnded = false;
|
||||
let pendingExternalSourceTimer = 0;
|
||||
|
||||
const AGENT_AUDIO_AUTOPLAY_DELAY_MS = 8000;
|
||||
|
||||
const secure = window.isSecureContext ||
|
||||
['localhost', '127.0.0.1', '[::1]'].includes(location.hostname);
|
||||
if (!secure) $('insecure').classList.add('show');
|
||||
if (location.protocol.startsWith('http')) $('baseUrl').value = location.origin;
|
||||
else $('baseUrl').value = 'http://localhost:8787';
|
||||
|
||||
/* --------------------------------------------------------------- mic meter */
|
||||
async function micMeterOn(stream) {
|
||||
micAnalyser = null;
|
||||
const ctx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
const src = ctx.createMediaStreamSource(stream);
|
||||
const analyser = ctx.createAnalyser();
|
||||
analyser.fftSize = 256;
|
||||
src.connect(analyser);
|
||||
const data = new Uint8Array(analyser.frequencyBinCount);
|
||||
const bars = [...$('micMeter').querySelectorAll('.bar i')];
|
||||
const tick = () => {
|
||||
analyser.getByteFrequencyData(data);
|
||||
for (let i = 0; i < bars.length; i++) {
|
||||
const v = data[i * 3] / 255;
|
||||
bars[i].style.height = Math.max(3, v * 12) + 'px';
|
||||
}
|
||||
micRaf = requestAnimationFrame(tick);
|
||||
};
|
||||
tick();
|
||||
micAnalyser = ctx;
|
||||
}
|
||||
|
||||
function micMeterOff() {
|
||||
cancelAnimationFrame(micRaf);
|
||||
if (micAnalyser) micAnalyser.close().catch(() => {});
|
||||
micAnalyser = null;
|
||||
for (const b of $('micMeter').querySelectorAll('.bar i')) b.style.height = '3px';
|
||||
$('micMeter').classList.remove('on');
|
||||
}
|
||||
|
||||
async function micStart() {
|
||||
if (mic) return;
|
||||
mic = new MicCapture({ sampleRate: AUDIO.sampleRate });
|
||||
mic.on('frame', (samples) => session.client.sendAudio(samples));
|
||||
await mic.start();
|
||||
$('micMeter').classList.add('on');
|
||||
// The SDK does not expose the MediaStream, so re-open one just for the meter.
|
||||
navigator.mediaDevices
|
||||
.getUserMedia({ audio: true })
|
||||
.then((stream) => micMeterOn(stream))
|
||||
.catch(() => {});
|
||||
$('micBtn').textContent = 'Microphone: ON';
|
||||
$('micBtn').classList.add('active');
|
||||
log('user', 'mic → ON');
|
||||
}
|
||||
|
||||
function micStop() {
|
||||
if (!mic) return;
|
||||
mic.stop();
|
||||
mic = null;
|
||||
micMeterOff();
|
||||
$('micBtn').textContent = 'Microphone: OFF';
|
||||
$('micBtn').classList.remove('active');
|
||||
log('user', 'mic → OFF');
|
||||
}
|
||||
|
||||
function clearPendingExternalSourceAutoplay() {
|
||||
if (pendingExternalSourceTimer) clearTimeout(pendingExternalSourceTimer);
|
||||
pendingExternalSourceTimer = 0;
|
||||
}
|
||||
|
||||
function schedulePendingExternalSourceAutoplay() {
|
||||
if (!session || !pendingExternalSourceUrl || !agentTextEnded) return;
|
||||
clearPendingExternalSourceAutoplay();
|
||||
const url = pendingExternalSourceUrl;
|
||||
pendingExternalSourceTimer = setTimeout(() => {
|
||||
pendingExternalSourceTimer = 0;
|
||||
if (!session) return;
|
||||
if (!pendingExternalSourceUrl || pendingExternalSourceUrl !== url) return;
|
||||
if (!agentTextEnded) return;
|
||||
pendingExternalSourceUrl = '';
|
||||
agentTextEnded = false;
|
||||
playExternalSource(url);
|
||||
}, AGENT_AUDIO_AUTOPLAY_DELAY_MS);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ connect / wire */
|
||||
function wire(s) {
|
||||
s.on('state', (st) => {
|
||||
if (st === 'connecting') setState('connecting…', 'warn');
|
||||
else if (st === 'open') setState('connected', 'on');
|
||||
else if (st === 'closed') setState('disconnected', '');
|
||||
});
|
||||
s.on('reconnecting', (e) => setState(`reconnecting… (${e.attempt + 1})`, 'warn'));
|
||||
|
||||
s.on('session', async (e) => {
|
||||
sessionId = e.id;
|
||||
log('user', `session id = ${e.id}`);
|
||||
// Apply the demo's system prompt for the whole session, right at startup.
|
||||
try {
|
||||
showPrompt(await rest.setPrompt(SESSION_PROMPT, sessionId));
|
||||
log('tool', `session prompt applied (session ${sessionId})`);
|
||||
} catch (err) {
|
||||
log('err', `prompt apply failed: ${err.message}`);
|
||||
refreshPrompt();
|
||||
}
|
||||
// Authenticate the user identified by ?phoneNumber=… in the page URL.
|
||||
await authenticateUser();
|
||||
});
|
||||
|
||||
s.on('cues', (e) => {
|
||||
log('cue', `cues lang=${e.lang} ${JSON.stringify(e.config ?? {})}`);
|
||||
});
|
||||
|
||||
s.on('user', (e) => {
|
||||
clearPendingExternalSourceAutoplay();
|
||||
agentTextEnded = false;
|
||||
pendingExternalSourceUrl = '';
|
||||
log('user', `user: ${e.text}`);
|
||||
});
|
||||
|
||||
s.on('assistant_delta', (e) => {
|
||||
agentTextEnded = false;
|
||||
if (!agentBubble) agentBubble = addBubble('agent', 'agent');
|
||||
agentText += e.text;
|
||||
renderKaraoke(agentBubble, agentText, e.text);
|
||||
chat.scrollTop = chat.scrollHeight;
|
||||
});
|
||||
|
||||
s.on('assistant_markdown', (e) => {
|
||||
addMarkdownBubble(e.text ?? '');
|
||||
log('tool', `assistant_markdown (${String(e.text ?? '').length} char)`);
|
||||
});
|
||||
|
||||
s.on('assistant_end', () => {
|
||||
if (agentBubble) renderKaraoke(agentBubble, agentText, '');
|
||||
log('user', `assistant_end (${agentText.length} char)`);
|
||||
agentBubble = null;
|
||||
agentText = '';
|
||||
agentTextEnded = true;
|
||||
schedulePendingExternalSourceAutoplay();
|
||||
});
|
||||
|
||||
s.on('cue', (e) => {
|
||||
log('cue', `cue ${e.kind} ${e.name ?? ''} ${e.category ?? ''}`);
|
||||
addChip('cue flash', `cue <b>${escapeHtml(e.kind)}</b>${e.name ? ' · ' + escapeHtml(e.name) : ''}`);
|
||||
});
|
||||
|
||||
s.on('barge_in', (e) => {
|
||||
bargeCount++;
|
||||
$('bargeCount').textContent = bargeCount;
|
||||
$('bargein').classList.add('active');
|
||||
setTimeout(() => $('bargein').classList.remove('active'), 700);
|
||||
clearPendingExternalSourceAutoplay();
|
||||
agentTextEnded = false;
|
||||
pendingExternalSourceUrl = '';
|
||||
log('cue', `barge_in source=${e.source ?? '?'} prob=${e.prob ?? '-'}`);
|
||||
});
|
||||
|
||||
s.on('tool_call', (e) => {
|
||||
log('tool', `tool_call ${e.name} ${JSON.stringify(e.arguments ?? {})}`);
|
||||
addChip('tool', `tool <b>${escapeHtml(e.name)}</b>`);
|
||||
// A tool flagged `audio` returns a media file: queue its URL and start
|
||||
// playback only after the agent finishes speaking for this turn.
|
||||
const def = DEMO_TOOLS.find((t) => t.name === e.name);
|
||||
if (def?.audio) {
|
||||
try {
|
||||
pendingExternalSourceUrl = toolUrl(def, e.arguments);
|
||||
$('sourceUrl').value = pendingExternalSourceUrl;
|
||||
addChip('tool', `audio <b>queued</b>`);
|
||||
if (agentTextEnded) schedulePendingExternalSourceAutoplay();
|
||||
} catch (err) {
|
||||
log('err', `audio tool playback failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
s.on('audio_source', (e) => {
|
||||
log('cue', `audio_source ${e.state} ${e.url ?? ''}`);
|
||||
if (e.state === 'error') addChip('chip flash', `source <b>error</b> ${escapeHtml(e.text ?? '')}`);
|
||||
});
|
||||
|
||||
s.on('error', (e) => {
|
||||
if (e.text) {
|
||||
log('err', `server error: ${e.text}`);
|
||||
addBubble('error', 'error').textContent = e.text;
|
||||
} else {
|
||||
log('err', `socket error${e.fatal ? ' (fatal)' : ''}: ${e.error?.message ?? ''}`);
|
||||
}
|
||||
});
|
||||
|
||||
s.on('audio', () => {}); // frames are played by the session; logged nowhere (too noisy)
|
||||
}
|
||||
|
||||
async function connect() {
|
||||
const baseUrl = $('baseUrl').value.trim();
|
||||
rest = new AudioAgentRest(baseUrl);
|
||||
session = new AudioSession({
|
||||
baseUrl,
|
||||
autoReconnect: true,
|
||||
forwardMic: false, // the mic is toggled separately below
|
||||
bufferMs: Number($('bufferMs').value) || 150,
|
||||
});
|
||||
wire(session);
|
||||
try {
|
||||
await session.start({ microphone: false });
|
||||
} catch (err) {
|
||||
log('err', `connect failed: ${err.message}`);
|
||||
setState('error', 'err');
|
||||
return;
|
||||
}
|
||||
$('connectBtn').textContent = 'Disconnect';
|
||||
for (const id of ['sendBtn', 'micBtn', 'interruptBtn', 'sourcePlay', 'sourceStop', 'sourceVol',
|
||||
'promptLoad', 'promptApply', 'promptClear', 'promptText']) {
|
||||
$(id).disabled = false;
|
||||
}
|
||||
try {
|
||||
const tools = await rest.setTools(DEMO_TOOLS);
|
||||
log('tool', `demo tools installed (${tools.count})`);
|
||||
} catch (err) {
|
||||
log('err', `tool install failed: ${err.message}`);
|
||||
}
|
||||
log('user', `connected → ${session.client.url}`);
|
||||
}
|
||||
|
||||
function disconnect() {
|
||||
micStop();
|
||||
if (session) session.stop();
|
||||
session = null;
|
||||
rest = null;
|
||||
sessionId = null;
|
||||
userData = null;
|
||||
clearPendingExternalSourceAutoplay();
|
||||
pendingExternalSourceUrl = '';
|
||||
agentTextEnded = false;
|
||||
setUser('');
|
||||
$('sessionId').textContent = '—';
|
||||
$('promptSource').textContent = '—';
|
||||
$('promptOverride').textContent = '—';
|
||||
$('promptText').value = '';
|
||||
$('connectBtn').textContent = 'Connect';
|
||||
for (const id of ['sendBtn', 'micBtn', 'interruptBtn', 'sourcePlay', 'sourceStop', 'sourceVol',
|
||||
'promptLoad', 'promptApply', 'promptClear', 'promptText']) {
|
||||
$(id).disabled = true;
|
||||
}
|
||||
setState('disconnected', '');
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------- system prompt */
|
||||
function showPrompt(res) {
|
||||
$('sessionId').textContent = res.session_id ?? '—';
|
||||
$('promptSource').textContent = res.source;
|
||||
$('promptOverride').textContent = res.has_override ? 'yes' : 'no';
|
||||
$('promptText').value = res.prompt;
|
||||
}
|
||||
|
||||
async function refreshPrompt() {
|
||||
if (!rest) return;
|
||||
try {
|
||||
showPrompt(await rest.getPrompt(sessionId));
|
||||
} catch (err) {
|
||||
log('err', `prompt load failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------- phone auth */
|
||||
/** Read `phoneNumber` from the page URL. Kept raw (not via URLSearchParams) so
|
||||
* a leading `+` in `?phoneNumber=+44…` survives instead of becoming a space. */
|
||||
function phoneFromUrl() {
|
||||
const m = location.search.match(/[?&](?:phoneNumber|phone)=([^&]*)/i);
|
||||
return m ? decodeURIComponent(m[1]) : '';
|
||||
}
|
||||
|
||||
/** Best-effort human label out of the login payload (unknown schema). */
|
||||
function userLabel(u) {
|
||||
if (!u || typeof u !== 'object') return '';
|
||||
const cand = [
|
||||
u.name, u.fullName, u.displayName, u.email, u.phoneNumber,
|
||||
u.user?.name, u.user?.fullName, u.user?.email,
|
||||
];
|
||||
return cand.find((v) => typeof v === 'string' && v) || '';
|
||||
}
|
||||
|
||||
/** Show (or clear) the authenticated user's name in the header. */
|
||||
function setUser(label, ok = true) {
|
||||
const el = $('userStatus');
|
||||
el.textContent = label || '';
|
||||
el.classList.toggle('show', !!label);
|
||||
el.classList.toggle('bad', !ok);
|
||||
el.title = label ? `${label} · session ${sessionId ?? '—'}` : '';
|
||||
}
|
||||
|
||||
/** Authenticate the user identified in the URL, at session start. */
|
||||
async function authenticateUser() {
|
||||
const phoneNumber = phoneFromUrl();
|
||||
if (!phoneNumber) {
|
||||
log('user', 'no phoneNumber in URL — skipping phone-login');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
userData = await rest.phoneLogin(phoneNumber, sessionId);
|
||||
const label = userLabel(userData) || phoneNumber;
|
||||
setUser(label);
|
||||
addChip('tool', `user <b>${escapeHtml(label)}</b>`);
|
||||
log('tool', `phone-login ok (${phoneNumber}) → ${JSON.stringify(userData).slice(0, 300)}`);
|
||||
} catch (err) {
|
||||
setUser('auth failed', false);
|
||||
log('err', `phone-login failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------------- events */
|
||||
$('connectBtn').onclick = () => (session ? disconnect() : connect());
|
||||
|
||||
$('sendBtn').onclick = () => {
|
||||
const text = $('textInput').value.trim();
|
||||
if (!text || !session) return;
|
||||
session.sendUserText(text);
|
||||
$('textInput').value = '';
|
||||
};
|
||||
$('textInput').onkeydown = (e) => {
|
||||
if (e.key === 'Enter') $('sendBtn').click();
|
||||
};
|
||||
|
||||
$('micBtn').onclick = async () => {
|
||||
if (!session) return;
|
||||
try {
|
||||
mic ? micStop() : await micStart();
|
||||
} catch (err) {
|
||||
log('err', `mic error: ${err.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
$('interruptBtn').onclick = () => session && session.interrupt();
|
||||
|
||||
/** Route a URL to the engine's external audio source channel and start it. */
|
||||
function playExternalSource(url) {
|
||||
if (!session || !url) return;
|
||||
$('sourceUrl').value = url;
|
||||
session.client.playAudioSource(url);
|
||||
log('cue', `audio_source play ${url}`);
|
||||
}
|
||||
|
||||
$('sourcePlay').onclick = () => playExternalSource($('sourceUrl').value.trim());
|
||||
$('sourceStop').onclick = () => session && session.client.stopAudioSource();
|
||||
$('sourceVol').oninput = (e) => session && session.client.setAudioSourceVolume(e.target.value / 100);
|
||||
|
||||
$('promptLoad').onclick = () => refreshPrompt();
|
||||
$('promptApply').onclick = async () => {
|
||||
if (!rest) return;
|
||||
const prompt = $('promptText').value;
|
||||
if (!prompt.trim()) return;
|
||||
try {
|
||||
showPrompt(await rest.setPrompt(prompt, sessionId));
|
||||
log('tool', `prompt override set (session ${sessionId})`);
|
||||
} catch (err) {
|
||||
log('err', `prompt apply failed: ${err.message}`);
|
||||
}
|
||||
};
|
||||
$('promptClear').onclick = async () => {
|
||||
if (!rest) return;
|
||||
try {
|
||||
showPrompt(await rest.setPrompt(null, sessionId));
|
||||
log('tool', `prompt override cleared (session ${sessionId})`);
|
||||
} catch (err) {
|
||||
log('err', `prompt clear failed: ${err.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
setState('disconnected', '');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user