Files
piperust/src/web/mivocero.html
T
2026-10-17 22:01:31 +01:00

1171 lines
40 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Mivocero — Speech-to-Speech Agent</title>
<style>
:root {
--bg: #0b0f1a;
--panel: #141b2d;
--accent: #4f8cff;
--accent-dim: #2a4a8a;
--ok: #3ddc97;
--warn: #ffb454;
--err: #ff5470;
--text: #e6edf7;
--muted: #8a94a6;
--border: #243049;
}
* { box-sizing: border-box; }
html, body { height: 100%; }
body {
margin: 0;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
background: var(--bg);
color: var(--text);
display: flex;
justify-content: center;
padding: 18px;
}
.app {
display: flex;
flex-direction: column;
width: 100%;
max-width: 820px;
height: calc(100vh - 36px);
background: var(--panel);
border: 1px solid var(--border);
border-radius: 14px;
overflow: hidden;
}
header {
display: flex;
align-items: center;
gap: 12px;
padding: 14px 18px;
border-bottom: 1px solid var(--border);
}
header h1 { font-size: 16px; margin: 0; flex: 1; }
.status { display: inline-flex; align-items: center; gap: 8px; font-size: 13px; color: var(--muted); }
.dot { width: 10px; height: 10px; border-radius: 50%; background: var(--muted); }
.dot.on { background: var(--ok); box-shadow: 0 0 8px var(--ok); }
.dot.err { background: var(--err); box-shadow: 0 0 8px var(--err); }
button {
background: var(--accent-dim);
color: var(--text);
border: 1px solid var(--accent);
border-radius: 8px;
padding: 8px 14px;
font-size: 13px;
cursor: pointer;
transition: all .15s;
}
button:hover:not(:disabled) { background: var(--accent); }
button:disabled { opacity: .4; cursor: not-allowed; }
button.danger { border-color: var(--err); background: rgba(255,84,112,.15); }
button.danger:hover:not(:disabled) { background: var(--err); }
#chat {
flex: 1;
overflow-y: auto;
padding: 18px;
display: flex;
flex-direction: column;
gap: 12px;
}
.msg { display: flex; flex-direction: column; gap: 4px; max-width: 80%; }
.msg .who { font-size: 11px; text-transform: uppercase; letter-spacing: .5px; color: var(--muted); }
.msg .bubble {
padding: 10px 13px;
border-radius: 12px;
font-size: 14px;
line-height: 1.45;
white-space: pre-wrap;
word-break: break-word;
}
.msg.user { align-self: flex-end; align-items: flex-end; }
.msg.user .bubble { background: var(--accent-dim); border: 1px solid var(--accent); }
.msg.agent { align-self: flex-start; }
.msg.agent .bubble { background: var(--bg); border: 1px solid var(--border); }
.msg.error .bubble { border-color: var(--err); color: var(--err); }
/* Karaoke: words light up as they are spoken by the agent. */
.bubble.karaoke .kw { color: var(--muted); transition: color .18s ease; }
.bubble.karaoke .kw.done { color: var(--text); }
.bubble.karaoke .kw.now { color: var(--accent); text-shadow: 0 0 8px rgba(79,140,255,.55); }
/* Same karaoke style on the user's own bubbles, tuned for the blue bubble. */
.msg.user .bubble.karaoke .kw { color: rgba(230,237,247,.5); }
.msg.user .bubble.karaoke .kw.done { color: #ffffff; }
.msg.user .bubble.karaoke .kw.now { color: #ffffff; text-shadow: 0 0 9px rgba(255,255,255,.85); }
.hint { color: var(--muted); font-size: 13px; text-align: center; margin: auto; }
/* Barge-in indicator: flashes red whenever the agent is interrupted. */
.bargein {
display: inline-flex; align-items: center; gap: 7px;
font-size: 12px; color: var(--muted);
border: 1px solid var(--border); border-radius: 999px;
padding: 5px 11px; white-space: nowrap;
transition: border-color .2s, box-shadow .2s, color .2s;
}
.bargein .bi-dot {
width: 9px; height: 9px; border-radius: 50%; flex: none;
background: var(--err); opacity: .25;
transition: opacity .15s, box-shadow .15s, transform .15s;
}
.bargein b { color: var(--text); font-variant-numeric: tabular-nums; }
.bargein.active { border-color: var(--err); color: var(--err); box-shadow: 0 0 12px -2px var(--err); }
.bargein.active .bi-dot { opacity: 1; box-shadow: 0 0 9px var(--err); transform: scale(1.35); }
footer { border-top: 1px solid var(--border); padding: 12px 18px; }
.meters { display: flex; align-items: center; gap: 14px; margin-bottom: 10px; }
.meter { height: 8px; background: var(--bg); border-radius: 4px; overflow: hidden; flex: 1; }
.meter > div { height: 100%; width: 0%; background: linear-gradient(90deg, var(--ok), var(--warn), var(--err)); transition: width .05s; }
.meters span { font-size: 11px; color: var(--muted); white-space: nowrap; }
/* Cue indicators: flash + counter for filler / start filler / backchannel. */
.cues { display: flex; gap: 8px; margin-bottom: 10px; }
.cue {
--cue: var(--accent);
flex: 1;
min-width: 0;
display: flex;
align-items: center;
gap: 7px;
padding: 6px 9px;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 8px;
font-size: 11px;
color: var(--muted);
transition: border-color .25s, box-shadow .25s;
}
.cue[data-kind="filler"] { --cue: var(--ok); }
.cue[data-kind="start_filler"] { --cue: var(--warn); }
.cue[data-kind="backchannel"] { --cue: var(--accent); }
.cue .cue-dot {
width: 9px; height: 9px; border-radius: 50%; flex: none;
background: var(--cue); opacity: .25;
transition: opacity .15s, box-shadow .15s, transform .15s;
}
.cue.active { border-color: var(--cue); box-shadow: 0 0 10px -2px var(--cue); }
.cue.active .cue-dot { opacity: 1; box-shadow: 0 0 8px var(--cue); transform: scale(1.3); }
.cue .cue-label { text-transform: uppercase; letter-spacing: .4px; white-space: nowrap; }
.cue .cue-last {
flex: 1 1 auto; min-width: 0; text-align: right;
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
}
.cue .cue-count { color: var(--text); font-weight: 600; font-variant-numeric: tabular-nums; }
/* Cue methods: collapsible panel with the live policy of each cue. */
.cuemethods {
margin-bottom: 10px;
border: 1px solid var(--border);
border-radius: 8px;
background: var(--bg);
font-size: 12px;
}
.cuemethods > summary {
display: flex; align-items: center; gap: 8px;
padding: 7px 10px; cursor: pointer; list-style: none; color: var(--muted);
}
.cuemethods > summary::-webkit-details-marker { display: none; }
.cuemethods > summary::before {
content: '\25B8'; font-size: 10px; transition: transform .15s;
}
.cuemethods[open] > summary::before { transform: rotate(90deg); }
.cuemethods .cm-title { text-transform: uppercase; letter-spacing: .4px; color: var(--text); font-weight: 600; }
.cuemethods .cm-hint { margin-left: auto; color: var(--muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.cm-cards {
display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
gap: 8px; padding: 0 10px 10px; max-height: 246px; overflow-y: auto;
}
.cm-empty { color: var(--muted); padding: 2px 0 4px; }
.cm-card { --cue: var(--accent); border: 1px solid var(--border); border-left: 3px solid var(--cue); border-radius: 8px; padding: 8px 10px; }
.cm-card[data-kind="filler"] { --cue: var(--ok); }
.cm-card[data-kind="start_filler"] { --cue: var(--warn); }
.cm-card[data-kind="backchannel"] { --cue: var(--accent); }
.cm-head { display: flex; align-items: center; gap: 7px; margin-bottom: 5px; }
.cm-head .cm-name { text-transform: uppercase; letter-spacing: .4px; font-size: 11px; color: var(--cue); font-weight: 600; }
.cm-head .cm-badge {
margin-left: auto; font-size: 10px; padding: 1px 7px; border-radius: 999px;
border: 1px solid var(--border); color: var(--muted);
}
.cm-head .cm-badge.on { border-color: var(--cue); color: var(--cue); }
.cm-head .cm-badge.off { border-color: var(--err); color: var(--err); }
.cm-desc { color: var(--muted); line-height: 1.45; margin-bottom: 6px; }
.cm-params { display: flex; flex-wrap: wrap; gap: 4px 6px; }
.cm-param {
font-size: 11px; color: var(--text);
background: var(--panel); border: 1px solid var(--border);
border-radius: 6px; padding: 1px 6px;
}
.cm-param b { color: var(--muted); font-weight: 600; }
.composer { display: flex; gap: 8px; }
.composer input {
flex: 1;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 8px;
padding: 10px 12px;
color: var(--text);
font-size: 14px;
}
.mediabar {
display: flex;
align-items: center;
gap: 8px;
margin-bottom: 10px;
}
.mediabar input[type="text"] {
flex: 1;
min-width: 0;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 8px;
padding: 8px 10px;
color: var(--text);
font-size: 13px;
}
.mediabar input[type="range"] { width: 88px; accent-color: var(--accent); }
.mediabar .state {
font-size: 11px;
color: var(--muted);
white-space: nowrap;
min-width: 72px;
text-align: right;
}
#log {
display: none;
max-height: 110px;
overflow-y: auto;
margin-top: 10px;
padding: 8px;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 8px;
font-family: ui-monospace, Menlo, monospace;
font-size: 11px;
color: var(--muted);
white-space: pre-wrap;
}
/* --- Shared side menu --- */
.menu-btn {
background: transparent;
border: 1px solid var(--border);
color: var(--text);
border-radius: 8px;
padding: 6px 11px;
font-size: 16px;
line-height: 1;
cursor: pointer;
}
.menu-btn:hover { background: var(--bg); }
.sidemenu {
position: fixed;
top: 0; left: 0;
height: 100%;
width: 240px;
background: var(--panel);
border-right: 1px solid var(--border);
transform: translateX(-100%);
transition: transform .2s ease;
z-index: 40;
display: flex;
flex-direction: column;
padding: 18px 14px;
gap: 6px;
}
.sidemenu.open { transform: translateX(0); }
.sidemenu h2 {
font-size: 11px;
text-transform: uppercase;
letter-spacing: .7px;
color: var(--muted);
margin: 0 0 10px;
padding: 0 12px;
}
.sidemenu a {
display: block;
padding: 10px 12px;
border-radius: 8px;
color: var(--text);
text-decoration: none;
font-size: 14px;
border: 1px solid transparent;
}
.sidemenu a:hover { background: var(--bg); }
.sidemenu a.active { background: var(--accent-dim); border-color: var(--accent); }
.backdrop {
position: fixed;
inset: 0;
background: rgba(0,0,0,.45);
opacity: 0;
pointer-events: none;
transition: opacity .2s;
z-index: 30;
}
.backdrop.open { opacity: 1; pointer-events: auto; }
</style>
</head>
<body>
<div class="backdrop" id="menuBackdrop"></div>
<nav class="sidemenu" id="sidemenu">
<h2>Menu</h2>
<a href="/mivocero" class="active">🎙️ Mivocero</a>
<a href="/customvoice">🎨 Custom Voice</a>
</nav>
<div class="app">
<header>
<button class="menu-btn" id="menuBtn" title="Menu">☰</button>
<h1>🎙️ Mivocero</h1>
<span class="status"><span class="dot" id="connDot"></span><span id="connText">disconnected</span></span>
<span class="bargein" id="bargeinInd" title="Agent interrupted by the user (VAD barge-in or the Stop button)">
<span class="bi-dot"></span>Barge-in <b id="bargeinCount">0</b>
</span>
<button id="btnConnect" class="ok">Connect</button>
<button id="btnInterrupt" class="danger" disabled>Stop</button>
</header>
<div id="chat">
<div class="hint" id="emptyHint">
Press <b>Connect</b>, allow microphone access and start talking.<br />
The agent replies with synthesized speech and the transcript appears here.
</div>
</div>
<footer>
<div class="meters">
<span>🎤</span><div class="meter"><div id="micMeter"></div></div>
<span>🔊</span><div class="meter"><div id="outMeter"></div></div>
</div>
<div class="cues">
<div class="cue" data-kind="filler" id="cueFiller" title="End-of-turn filler (latency mask)">
<span class="cue-dot"></span>
<span class="cue-label">Filler</span>
<span class="cue-last">—</span>
<span class="cue-count">0</span>
</div>
<div class="cue" data-kind="start_filler" id="cueStartFiller" title="Turn-start acknowledgment (latency mask)">
<span class="cue-dot"></span>
<span class="cue-label">Start filler</span>
<span class="cue-last">—</span>
<span class="cue-count">0</span>
</div>
<div class="cue" data-kind="backchannel" id="cueBackchannel" title="Backchannel cue while the user speaks">
<span class="cue-dot"></span>
<span class="cue-label">Backchannel</span>
<span class="cue-last">—</span>
<span class="cue-count">0</span>
</div>
</div>
<details class="cuemethods" id="cueMethods">
<summary>
<span class="cm-title">Cue methods</span>
<span class="cm-hint" id="cueMethodsHint">connect to load the live policy</span>
</summary>
<div class="cm-cards" id="cueCards">
<div class="cm-empty">Connect to load the live cue policy from the server.</div>
</div>
</details>
<div class="mediabar">
<input type="text" id="mediaUrl" placeholder="🎵 Audio URL / radio / YouTube…" disabled autocomplete="off" />
<button type="button" id="btnMediaPlay" disabled>Play</button>
<button type="button" id="btnMediaStop" class="danger" disabled>Stop</button>
<input type="range" id="mediaVol" min="0" max="150" value="60" title="External audio volume" disabled />
<span class="state" id="mediaState">idle</span>
</div>
<form class="composer" id="composer">
<input type="text" id="textInput" placeholder="Or type a message…" disabled autocomplete="off" />
<button type="submit" id="btnSend" disabled>Send</button>
</form>
<div id="log"></div>
</footer>
</div>
<script>
// ---------------------------------------------------------------------------
// Mivocero — speech-to-speech agent client
//
// mic -> AudioWorklet (44.1 kHz, 16-bit LE mono, 20 ms frames) -> WebSocket
// WebSocket binary frames -> playback queue -> AudioWorklet -> speakers
//
// Text frames received from the server are JSON chat events:
// {"type":"user","text":"…"} recognized user speech
// {"type":"assistant_delta","text":"…"} incremental assistant reply
// {"type":"assistant_end"} assistant finished
// {"type":"error","text":"…"} pipeline error
// {"type":"audio_source","state":"…", …} external audio source status
// {"type":"cue","kind":"…","name":"…"} filler / start filler / backchannel emitted
// {"type":"cues","lang":"…","config":{…}} live cue policy (sent at start)
// {"type":"barge_in","source":"…"} agent interrupted (VAD onset)
//
// The session closes itself after IDLE_TIMEOUT_MS without any traffic in
// either direction (see the idle guard below).
// ---------------------------------------------------------------------------
// --- Shared side menu (chat / custom voice navigation) ---
(() => {
const menu = document.getElementById('sidemenu');
const backdrop = document.getElementById('menuBackdrop');
const setOpen = (open) => {
menu.classList.toggle('open', open);
backdrop.classList.toggle('open', open);
};
document.getElementById('menuBtn').addEventListener('click', () => setOpen(true));
backdrop.addEventListener('click', () => setOpen(false));
})();
const SAMPLE_RATE_OUT = 44100; // server <-> client (matches the TTS rate)
const SAMPLE_RATE_IN = 44100;
const FRAME_MS = 20;
const FRAME_SAMPLES = SAMPLE_RATE_IN * FRAME_MS / 1000; // 882
// Disconnect the session (releasing the microphone) after this much time
// without activity.
const IDLE_TIMEOUT_MS = 2 * 60 * 1000;
const $ = (id) => document.getElementById(id);
const wsDot = $('connDot');
const wsText = $('connText');
const btnConnect = $('btnConnect');
const btnInterrupt = $('btnInterrupt');
const micMeter = $('micMeter');
const outMeter = $('outMeter');
const chatEl = $('chat');
const emptyHint = $('emptyHint');
const composer = $('composer');
const textInput = $('textInput');
const btnSend = $('btnSend');
const mediaUrl = $('mediaUrl');
const btnMediaPlay = $('btnMediaPlay');
const btnMediaStop = $('btnMediaStop');
const mediaVol = $('mediaVol');
const mediaState = $('mediaState');
const logEl = $('log');
// Cue indicators: one tile per filler type, flashed on each server "cue" event.
const cueEls = {
filler: $('cueFiller'),
start_filler: $('cueStartFiller'),
backchannel: $('cueBackchannel'),
};
const cueCounts = { filler: 0, start_filler: 0, backchannel: 0 };
const cueTimers = { filler: 0, start_filler: 0, backchannel: 0 };
// Barge-in indicator (header pill): flashes red on each interruption.
const bargeinInd = $('bargeinInd');
const bargeinCountEl = $('bargeinCount');
let bargeinTotal = 0;
let bargeinTimer = 0;
let ws = null;
let audioCtx = null;
let micStream = null;
let captureNode = null;
let playbackNode = null;
let outputGain = null;
let agentBubble = null; // bubble being streamed by the assistant
let agentText = ''; // accumulated assistant text for that bubble
let agentWords = []; // [{el, start}] word spans of the current reply
let agentPlayedAt = 0; // playback sample counter when the reply started
let agentEnded = false; // assistant_end received for the current reply
let playedSamples = 0; // cumulative samples emitted by the playback worklet
let karaokeRAF = 0; // requestAnimationFrame id while the reply streams
let karaokePrevPlayed = 0;
let karaokePrevTs = 0;
let karaokeIdleMs = 0;
// Approximate speaking rate used to map the audio playhead onto characters.
// Only drives the visual highlight — the spoken audio is unaffected.
const KARAOKE_CPS = 15;
function log(msg) {
const ts = new Date().toLocaleTimeString();
logEl.textContent = `[${ts}] ${msg}\n` + logEl.textContent;
if (logEl.textContent.length > 4000) logEl.textContent = logEl.textContent.slice(0, 4000);
}
function setConn(state, text) {
wsDot.className = 'dot ' + (state === 'on' ? 'on' : state === 'err' ? 'err' : '');
wsText.textContent = text;
}
function atBottom() {
return chatEl.scrollHeight - chatEl.scrollTop - chatEl.clientHeight < 60;
}
function scrollDown(force) {
if (force || atBottom()) chatEl.scrollTop = chatEl.scrollHeight;
}
// --- Chat rendering --------------------------------------------------------
function addMessage(role, text, cls = '') {
emptyHint?.remove();
const wrap = document.createElement('div');
wrap.className = 'msg ' + role + (cls ? ' ' + cls : '');
const who = document.createElement('div');
who.className = 'who';
who.textContent = role === 'user' ? 'You' : role === 'error' ? 'Error' : 'Agent';
const bubble = document.createElement('div');
bubble.className = 'bubble';
bubble.textContent = text;
wrap.append(who, bubble);
chatEl.appendChild(wrap);
scrollDown(true);
return bubble;
}
function startAgentMessage() {
agentText = '';
agentWords = [];
agentBubble = addMessage('agent', '');
agentBubble.classList.add('karaoke');
agentPlayedAt = playedSamples; // count only the audio spoken after this point
startKaraoke();
}
function appendAgentDelta(text) {
if (!agentBubble) startAgentMessage();
agentText += text;
renderKaraoke();
applyKaraoke(karaokeChars());
scrollDown(false);
}
// Rebuild the bubble as one <span> per word, keeping the whitespace between
// them so the layout is identical to plain text.
function renderKaraoke() {
if (!agentBubble) return;
agentWords = [];
agentBubble.textContent = '';
let idx = 0;
for (const tok of agentText.split(/(\s+)/)) {
if (tok === '') continue;
if (/^\s+$/.test(tok)) {
agentBubble.appendChild(document.createTextNode(tok));
} else {
const span = document.createElement('span');
span.className = 'kw';
span.textContent = tok;
agentBubble.appendChild(span);
agentWords.push({ el: span, start: idx });
}
idx += tok.length;
}
}
// Characters the playhead has "reached", from the audio actually emitted.
function karaokeChars() {
const played = Math.max(0, playedSamples - agentPlayedAt);
return Math.floor((played / SAMPLE_RATE_OUT) * KARAOKE_CPS);
}
function applyKaraoke(chars) {
for (const w of agentWords) {
const end = w.start + w.el.textContent.length;
w.el.classList.toggle('done', end <= chars);
w.el.classList.toggle('now', w.start < chars && end > chars);
}
}
function startKaraoke() {
stopKaraoke();
agentEnded = false;
karaokePrevPlayed = playedSamples;
karaokeIdleMs = 0;
karaokePrevTs = performance.now();
karaokeRAF = requestAnimationFrame(karaokeTick);
}
function stopKaraoke() {
if (karaokeRAF) cancelAnimationFrame(karaokeRAF);
karaokeRAF = 0;
}
function karaokeTick(ts) {
karaokeRAF = 0;
if (!agentBubble) return;
const dt = ts - karaokePrevTs;
karaokePrevTs = ts;
const played = Math.max(0, playedSamples - agentPlayedAt);
if (played > karaokePrevPlayed) { karaokePrevPlayed = played; karaokeIdleMs = 0; }
else karaokeIdleMs += dt;
// Once the text has ended and the audio has drained (playback idle for a
// beat after it had actually started), light up anything still pending so
// the reply always ends fully highlighted.
if (agentEnded && karaokePrevPlayed > 0 && karaokeIdleMs > 600) {
for (const w of agentWords) { w.el.classList.add('done'); w.el.classList.remove('now'); }
return;
}
applyKaraoke(karaokeChars());
karaokeRAF = requestAnimationFrame(karaokeTick);
}
function endAgentMessage() {
stopKaraoke();
agentBubble = null;
agentWords = [];
agentText = '';
}
// --- User-message karaoke --------------------------------------------------
// The user's transcript reaches the client only once the utterance has ended
// (the STT runs server-side on the whole segment), so there is no per-word
// audio timing to follow. The words are instead lit up by a short left-to-right
// sweep, giving the same karaoke feel as the agent's reply.
const USER_REVEAL_MS_PER_WORD = 90;
let userRevealRAF = 0;
let userRevealWords = null;
function addUserMessage(text) {
const bubble = addMessage('user', '');
bubble.classList.add('karaoke');
const words = [];
for (const tok of text.split(/(\s+)/)) {
if (tok === '') continue;
if (/^\s+$/.test(tok)) {
bubble.appendChild(document.createTextNode(tok));
} else {
const span = document.createElement('span');
span.className = 'kw';
span.textContent = tok;
bubble.appendChild(span);
words.push(span);
}
}
revealWords(words);
return bubble;
}
function revealWords(words) {
if (userRevealRAF) { cancelAnimationFrame(userRevealRAF); userRevealRAF = 0; }
if (userRevealWords) {
for (const w of userRevealWords) { w.classList.add('done'); w.classList.remove('now'); }
userRevealWords = null;
}
if (!words.length) return;
const start = performance.now();
userRevealWords = words;
const tick = (now) => {
const p = Math.floor((now - start) / USER_REVEAL_MS_PER_WORD) + 1;
for (let i = 0; i < words.length; i++) {
words[i].classList.toggle('done', i < p - 1);
words[i].classList.toggle('now', i === p - 1);
}
if (p <= words.length) {
userRevealRAF = requestAnimationFrame(tick);
} else {
for (const w of words) { w.classList.add('done'); w.classList.remove('now'); }
userRevealWords = null;
userRevealRAF = 0;
}
};
userRevealRAF = requestAnimationFrame(tick);
}
// --- Audio worklets --------------------------------------------------------
const CAPTURE_PROCESSOR = `
class CaptureProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.buf = [];
this.frameSize = ${FRAME_SAMPLES};
this.targetRate = ${SAMPLE_RATE_IN};
this.ratio = this.targetRate / sampleRate;
this.phase = 0;
this.prev = 0;
}
process(inputs) {
const inp = inputs[0][0];
if (!inp) return true;
let acc = 0;
for (let i = 0; i < inp.length; i++) {
const cur = inp[i];
acc += cur * cur;
while (this.phase < i + 1) {
const frac = this.phase - Math.floor(this.phase);
const s = this.prev + (cur - this.prev) * frac;
this.buf.push(s);
if (this.buf.length >= this.frameSize) this.flush();
this.phase += this.ratio;
}
this.prev = cur;
}
this.phase -= inp.length;
const rms = Math.sqrt(acc / inp.length);
this.port.postMessage({ rms });
return true;
}
flush() {
const n = this.frameSize;
const buf = new ArrayBuffer(n * 2);
const dv = new DataView(buf);
for (let i = 0; i < n; i++) {
let v = Math.max(-1, Math.min(1, this.buf[i]));
dv.setInt16(i * 2, v < 0 ? v * 0x8000 : v * 0x7FFF, true);
}
this.buf.splice(0, n);
this.port.postMessage({ pcm: buf }, [buf]);
}
}
registerProcessor('capture-processor', CaptureProcessor);
`;
const PLAYBACK_PROCESSOR = `
class PlaybackProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.buf = new Float32Array(48000 * 2); // ~2 s ring @ 44.1 kHz
this.head = 0;
this.tail = 0;
this.preBuffer = Math.floor(sampleRate * 0.30); // 300 ms jitter buffer
this.buffering = true;
this.underrunSamples = 0;
this.underrunThreshold = Math.floor(sampleRate * 0.10);
this.played = 0; // samples actually emitted (drives the karaoke highlight)
this.port.onmessage = (e) => {
const pcm = e.data;
for (let i = 0; i < pcm.length; i++) {
this.buf[this.head] = pcm[i];
this.head = (this.head + 1) % this.buf.length;
if (this.head === this.tail) this.tail = (this.tail + 1) % this.buf.length;
}
};
}
available() { return (this.head - this.tail + this.buf.length) % this.buf.length; }
process(_inputs, outputs) {
const out = outputs[0][0];
let level = 0;
for (let i = 0; i < out.length; i++) {
let s = 0;
if (this.buffering) {
if (this.available() >= this.preBuffer) { this.buffering = false; this.underrunSamples = 0; }
}
if (!this.buffering) {
if (this.tail !== this.head) {
s = this.buf[this.tail];
this.tail = (this.tail + 1) % this.buf.length;
this.underrunSamples = 0;
this.played++;
} else {
this.underrunSamples++;
if (this.underrunSamples >= this.underrunThreshold) { this.buffering = true; this.underrunSamples = 0; }
}
}
out[i] = s;
level += s * s;
}
this.port.postMessage({ rms: Math.sqrt(level / out.length), played: this.played });
return true;
}
}
registerProcessor('playback-processor', PlaybackProcessor);
`;
async function startAudio() {
audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE_OUT });
if (audioCtx.state !== 'running') {
try { await audioCtx.resume(); } catch (e) {}
}
await audioCtx.audioWorklet.addModule(URL.createObjectURL(new Blob([CAPTURE_PROCESSOR], { type: 'application/javascript' })));
await audioCtx.audioWorklet.addModule(URL.createObjectURL(new Blob([PLAYBACK_PROCESSOR], { type: 'application/javascript' })));
micStream = await navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }
});
const source = audioCtx.createMediaStreamSource(micStream);
captureNode = new AudioWorkletNode(audioCtx, 'capture-processor');
captureNode.port.onmessage = (e) => {
if (e.data.rms !== undefined) {
micMeter.style.width = Math.min(100, e.data.rms * 200) + '%';
} else if (e.data.pcm && ws && ws.readyState === WebSocket.OPEN) {
ws.send(e.data.pcm);
}
};
source.connect(captureNode);
const silent = audioCtx.createGain();
silent.gain.value = 0;
captureNode.connect(silent).connect(audioCtx.destination);
playbackNode = new AudioWorkletNode(audioCtx, 'playback-processor');
playbackNode.port.onmessage = (e) => {
if (e.data.played !== undefined) playedSamples = e.data.played;
if (e.data.rms !== undefined) outMeter.style.width = Math.min(100, e.data.rms * 200) + '%';
};
outputGain = audioCtx.createGain();
outputGain.gain.value = 3.0; // Fish Audio output level is fairly low
playbackNode.connect(outputGain).connect(audioCtx.destination);
}
function stopAudio() {
try { captureNode?.disconnect(); } catch (e) {}
try { playbackNode?.disconnect(); } catch (e) {}
try { outputGain?.disconnect(); } catch (e) {}
micStream?.getTracks().forEach((t) => t.stop());
audioCtx?.close();
audioCtx = null;
micStream = null;
captureNode = playbackNode = outputGain = null;
micMeter.style.width = '0%';
outMeter.style.width = '0%';
}
function feedPlayback(arrayBuffer) {
const view = new DataView(arrayBuffer);
const n = Math.floor(view.byteLength / 2);
const f32 = new Float32Array(n);
for (let i = 0; i < n; i++) f32[i] = view.getInt16(i * 2, true) / 32768;
playbackNode?.port.postMessage(f32);
}
// --- WebSocket -------------------------------------------------------------
// Cue indicators: flash the matching tile and bump its counter. `kind` is one
// of "filler" | "start_filler" | "backchannel"; the detail line shows the
// backchannel category or the clip name.
function updateCue(ev) {
const el = cueEls[ev.kind];
if (!el) return;
cueCounts[ev.kind] += 1;
el.querySelector('.cue-count').textContent = cueCounts[ev.kind];
el.querySelector('.cue-last').textContent = ev.category || ev.name || '';
el.classList.add('active');
clearTimeout(cueTimers[ev.kind]);
cueTimers[ev.kind] = setTimeout(() => el.classList.remove('active'), 400);
}
function resetCues() {
for (const kind of Object.keys(cueCounts)) {
cueCounts[kind] = 0;
clearTimeout(cueTimers[kind]);
cueEls[kind].classList.remove('active');
cueEls[kind].querySelector('.cue-count').textContent = '0';
cueEls[kind].querySelector('.cue-last').textContent = '—';
}
}
// Barge-in indicator: bump the counter and flash the pill. Fired by the
// server "barge_in" event (VAD onset) and by the local Stop button.
function flashBargeIn(source) {
bargeinTotal += 1;
bargeinCountEl.textContent = bargeinTotal;
if (source) bargeinInd.title = 'Last barge-in: ' + source;
bargeinInd.classList.add('active');
clearTimeout(bargeinTimer);
bargeinTimer = setTimeout(() => bargeinInd.classList.remove('active'), 700);
}
function resetBargeIn() {
bargeinTotal = 0;
clearTimeout(bargeinTimer);
bargeinInd.classList.remove('active');
bargeinInd.title = 'Agent interrupted by the user (VAD barge-in or the Stop button)';
bargeinCountEl.textContent = '0';
}
// --- Cue methods panel -----------------------------------------------------
// The method (how each cue decides to fire) plus the live parameters reported
// by the server in the "cues" event at session start. `params` turns the
// server payload into [label, value] chips.
const fmt2 = (v) => (typeof v === 'number' ? v.toFixed(2) : '—');
const CUE_METHODS = {
filler: {
name: 'Filler',
desc: 'End-of-turn latency mask. At most one clip per turn, and only when the '
+ 'reply audio has not started within the delay — otherwise it is dropped, so '
+ 'it never talks over the answer. Picked round-robin from the active bank.',
params: (c) => [
['delay', c.delay_ms + ' ms'],
['gain idle', fmt2(c.gain_idle)],
['gain duck', fmt2(c.gain_duck)],
['clips', c.clips],
['bank', c.bank],
],
},
start_filler: {
name: 'Start filler',
desc: 'Turn-start acknowledgment. Fires only if the first audio chunk of the '
+ 'reply is still missing after its (shorter) delay; when audio arrives in '
+ 'time the cue is dropped. A clip is picked at random from the bank.',
params: (c) => [
['go delay', c.go_delay_ms + ' ms'],
['clips', c.clips],
['bank', c.bank],
],
},
backchannel: {
name: 'Backchannel',
desc: 'While the user keeps speaking (and the agent is silent): a short ack '
+ 'mixed at low gain. Fires only inside a natural micro-pause, at a random '
+ 'depth in the pause window, through a probabilistic gate, with randomized '
+ 'spacing and volume and a per-turn cap. Categories progress ack → agree.',
params: (c) => [
['gain', fmt2(c.gain)],
['min speech', c.min_speech_ms + ' ms'],
['interval', c.interval_ms + ' ms'],
['jitter', c.jitter_ms + ' ms'],
['pause', c.pause_min_ms + '–' + c.pause_max_ms + ' ms'],
['prob', fmt2(c.prob)],
['vol jitter', fmt2(c.volume_jitter)],
['max/turn', c.max_per_turn],
['categories', (c.categories || []).join(', ')],
['clips', c.clips],
],
},
};
// Render the three cards from the server "cues" payload (live params + method).
function renderCues(lang, cfg) {
const host = $('cueCards');
if (!host || !cfg) return;
host.textContent = '';
for (const kind of ['filler', 'start_filler', 'backchannel']) {
const m = CUE_METHODS[kind];
const c = cfg[kind];
if (!m || !c) continue;
const card = document.createElement('div');
card.className = 'cm-card';
card.dataset.kind = kind;
const head = document.createElement('div');
head.className = 'cm-head';
const name = document.createElement('span');
name.className = 'cm-name';
name.textContent = m.name;
const badge = document.createElement('span');
badge.className = 'cm-badge ' + (c.enabled ? 'on' : 'off');
badge.textContent = c.enabled ? 'on' : 'off';
head.append(name, badge);
const desc = document.createElement('div');
desc.className = 'cm-desc';
desc.textContent = m.desc;
const params = document.createElement('div');
params.className = 'cm-params';
for (const [k, v] of m.params(c)) {
const chip = document.createElement('span');
chip.className = 'cm-param';
const b = document.createElement('b');
b.textContent = k + ' ';
chip.append(b, document.createTextNode(String(v)));
params.append(chip);
}
card.append(head, desc, params);
host.append(card);
}
const hint = $('cueMethodsHint');
if (hint) hint.textContent = (lang ? 'lang ' + lang + ' · ' : '') + 'filler · start filler · backchannel';
}
function handleServerText(text) {
let event = null;
try { event = JSON.parse(text); } catch (e) { log('server: ' + text); return; }
switch (event.type) {
case 'user':
endAgentMessage(); // a new turn starts
addUserMessage(event.text);
break;
case 'assistant_delta':
appendAgentDelta(event.text);
break;
case 'assistant_end':
agentEnded = true; // keep the bubble: karaoke finishes when audio drains
break;
case 'barge_in':
// The server cut the reply (VAD onset): flash the indicator and stop the
// karaoke sweep — the audio is gone, so the reply is left as spoken.
flashBargeIn(event.source || 'vad');
endAgentMessage();
break;
case 'error':
endAgentMessage();
addMessage('error', event.text, 'error');
log('error: ' + event.text);
break;
case 'audio_source':
updateMediaState(event);
break;
case 'cue':
updateCue(event);
break;
case 'cues':
renderCues(event.lang, event.config);
break;
default:
log('server: ' + text);
}
}
// External audio source status → status label + button states.
function updateMediaState(event) {
const st = event.state || '';
mediaState.textContent =
st === 'playing' ? 'playing' : st === 'loading' ? 'loading…' : st === 'error' ? 'error' : 'idle';
btnMediaStop.disabled = !(st === 'playing' || st === 'loading');
if (st === 'error') {
const msg = event.text || 'unknown error';
log('audio source error: ' + msg);
addMessage('error', 'Audio source error: ' + msg, 'error');
}
}
function setMediaControlsEnabled(on) {
mediaUrl.disabled = !on;
mediaVol.disabled = !on;
btnMediaPlay.disabled = !on;
if (!on) btnMediaStop.disabled = true;
}
// --- Idle session guard ----------------------------------------------------
// Reset on any traffic (server audio / events) and on every user action. When
// it expires the session is closed, which stops the microphone and re-enables
// the Connect button. A playing external source keeps sending audio frames, so
// it counts as activity too.
let idleTimer = 0;
function touchActivity() {
if (!ws) return;
clearTimeout(idleTimer);
idleTimer = setTimeout(onIdleTimeout, IDLE_TIMEOUT_MS);
}
function stopIdleWatch() {
clearTimeout(idleTimer);
idleTimer = 0;
}
function onIdleTimeout() {
if (!ws) return;
log(`idle timeout — no activity for ${IDLE_TIMEOUT_MS / 60000} min, disconnecting`);
addMessage('error', 'Session closed after 2 minutes of inactivity. Press Connect to resume.', 'error');
disconnect();
}
async function connect() {
if (!navigator.mediaDevices?.getUserMedia) {
const why = window.isSecureContext
? 'navigator.mediaDevices is unavailable in this browser'
: `insecure origin (${location.origin}) — the microphone requires HTTPS or localhost`;
addMessage('error', 'Microphone unavailable: ' + why, 'error');
setConn('err', 'mic error');
return;
}
try {
await startAudio();
} catch (e) {
addMessage('error', 'Microphone access failed: ' + (e.name ? e.name + ': ' : '') + (e.message || e), 'error');
setConn('err', 'mic error');
return;
}
const proto = location.protocol === 'https:' ? 'wss:' : 'ws:';
const url = `${proto}//${location.host}/ws`;
log('connecting to ' + url);
ws = new WebSocket(url);
ws.binaryType = 'arraybuffer';
ws.onopen = () => {
setConn('on', 'connected');
log('connected');
resetCues();
resetBargeIn();
btnConnect.disabled = true;
btnInterrupt.disabled = false;
textInput.disabled = false;
btnSend.disabled = false;
setMediaControlsEnabled(true);
mediaState.textContent = 'idle';
touchActivity();
// Sync the server's external-audio gain with the slider.
sendJson({ type: 'audio_source', action: 'volume', gain: mediaVol.value / 100 });
};
ws.onmessage = (e) => {
touchActivity();
if (e.data instanceof ArrayBuffer) {
feedPlayback(e.data);
} else {
handleServerText(e.data);
}
};
ws.onerror = () => { log('websocket error'); setConn('err', 'error'); };
ws.onclose = () => {
stopIdleWatch();
ws = null; // allow reconnecting from the Connect button
log('disconnected');
setConn('', 'disconnected');
btnConnect.disabled = false;
btnInterrupt.disabled = true;
textInput.disabled = true;
btnSend.disabled = true;
setMediaControlsEnabled(false);
mediaState.textContent = 'idle';
endAgentMessage();
stopAudio();
};
}
function disconnect() {
try { ws?.close(); } catch (e) {}
stopAudio();
}
function sendJson(obj) {
if (ws && ws.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify(obj));
touchActivity();
return true;
}
return false;
}
btnConnect.onclick = () => (ws ? disconnect() : connect());
btnInterrupt.onclick = () => {
if (sendJson({ type: 'interrupt' })) {
log('→ interrupt');
flashBargeIn('client');
endAgentMessage();
}
};
// External audio source (URL / radio / YouTube) controls.
btnMediaPlay.onclick = () => {
const url = mediaUrl.value.trim();
if (!url) return;
if (sendJson({ type: 'audio_source', action: 'play', url })) {
mediaState.textContent = 'loading…';
btnMediaStop.disabled = false;
log('→ audio_source play ' + url);
}
};
btnMediaStop.onclick = () => {
if (sendJson({ type: 'audio_source', action: 'stop' })) log('→ audio_source stop');
};
mediaUrl.addEventListener('keydown', (e) => {
if (e.key === 'Enter') { e.preventDefault(); btnMediaPlay.click(); }
});
mediaVol.oninput = () => {
sendJson({ type: 'audio_source', action: 'volume', gain: mediaVol.value / 100 });
};
composer.onsubmit = (e) => {
e.preventDefault();
const text = textInput.value.trim();
if (!text) return;
if (sendJson({ type: 'user_text', text })) {
endAgentMessage();
addUserMessage(text);
textInput.value = '';
log('→ user_text');
}
};
// Optional verbose log: click the header status to toggle.
wsText.style.cursor = 'pointer';
wsText.onclick = () => { logEl.style.display = logEl.style.display === 'block' ? 'none' : 'block'; };
</script>
</body>
</html>