/** * audio-agent-client — JavaScript SDK for the `audio-agent-engine`. * * Zero dependencies, single-file ESM. The core (WebSocket session + REST * client + PCM helpers) runs in the browser **and** in Node; the audio helpers * (`MicCapture`, `PcmPlayer`, `AudioSession`) are browser-only and throw a clear * error when the Web Audio API is missing. * * The point of the SDK is decoupling: the engine stays where it is, the client * can live on another origin/server. For browser clients on a different origin * enable CORS on the engine (`CORS_ALLOW_ORIGIN=https://your.app`); the * WebSocket needs no CORS. * * ```js * import { AudioSession } from './audio-agent-client.js'; * * const session = new AudioSession({ baseUrl: 'https://engine.example.com:8787' }); * session.on('assistant_delta', (e) => console.log(e.text)); * await session.start(); // connect + mic + playback * session.sendUserText('ciao'); * ``` * * @module audio-agent-client */ /** * Audio wire format shared with the engine (see `src/config.rs`). * The engine speaks **16-bit little-endian mono PCM at 44.1 kHz** in both * directions; uplink frames are 20 ms (882 samples / 1764 bytes) and the server * resamples to 16 kHz for the STT engine. */ export const AUDIO = Object.freeze({ /** Sample rate of the PCM on the wire, both directions. */ sampleRate: 44100, /** Samples per 20 ms frame at {@link AUDIO.sampleRate}. */ frameSamples: 882, /** Bytes per 20 ms frame (16-bit mono). */ frameBytes: 1764, /** Sample rate the engine's STT engine runs at (server-side detail). */ sttSampleRate: 16000, }); /** Server → client text event types emitted by the engine. */ export const CLIENT_EVENTS = Object.freeze([ 'session', 'cues', 'user', 'assistant_delta', 'assistant_markdown', 'assistant_end', 'cue', 'barge_in', 'tool_call', 'audio_source', 'error', ]); /* -------------------------------------------------------------------------- */ /* PCM helpers */ /* -------------------------------------------------------------------------- */ /** * Decode little-endian 16-bit PCM bytes into an `Int16Array`. * An odd trailing byte is dropped. * @param {ArrayBuffer|ArrayBufferView} buffer * @returns {Int16Array} */ export function int16ArrayFromBytes(buffer) { const view = buffer instanceof ArrayBuffer ? new DataView(buffer) : new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); const count = Math.floor(view.byteLength / 2); const out = new Int16Array(count); for (let i = 0; i < count; i++) out[i] = view.getInt16(i * 2, true); return out; } /** * Convert normalized floats (-1..1) to 16-bit PCM, clamping out-of-range values. * @param {Float32Array|number[]} floats * @returns {Int16Array} */ export function float32ToInt16(floats) { const out = new Int16Array(floats.length); for (let i = 0; i < floats.length; i++) { let s = floats[i]; s = s < -1 ? -1 : s > 1 ? 1 : s; out[i] = s < 0 ? s * 0x8000 : s * 0x7fff; } return out; } /** * Convert 16-bit PCM to normalized floats (-1..1). * @param {Int16Array} ints * @returns {Float32Array} */ export function int16ToFloat32(ints) { const out = new Float32Array(ints.length); for (let i = 0; i < ints.length; i++) out[i] = ints[i] / 32768; return out; } /** * Normalize any accepted audio chunk to `Int16Array`. * @param {Int16Array|Float32Array|ArrayBuffer|ArrayBufferView|number[]} chunk * @returns {Int16Array} */ export function toInt16(chunk) { if (chunk instanceof Int16Array) return chunk; if (chunk instanceof Float32Array) return float32ToInt16(chunk); if (Array.isArray(chunk)) return Int16Array.from(chunk); if (chunk instanceof ArrayBuffer) return int16ArrayFromBytes(chunk); if (ArrayBuffer.isView(chunk)) { return int16ArrayFromBytes( chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength), ); } throw new TypeError('unsupported audio chunk: expected Int16Array, Float32Array, ArrayBuffer or number[]'); } /* -------------------------------------------------------------------------- */ /* Tiny event emitter (browser + Node, no CustomEvent dependency) */ /* -------------------------------------------------------------------------- */ class Emitter { constructor() { /** @type {Map>} */ this._listeners = new Map(); } /** * Subscribe to an event. Returns an unsubscribe function. * @param {string} type * @param {Function} handler * @returns {() => void} */ on(type, handler) { let set = this._listeners.get(type); if (!set) this._listeners.set(type, (set = new Set())); set.add(handler); return () => this.off(type, handler); } /** * Subscribe once. * @param {string} type * @param {Function} handler * @returns {() => void} */ once(type, handler) { const off = this.on(type, (payload) => { off(); handler(payload); }); return off; } /** * Unsubscribe. * @param {string} type * @param {Function} handler */ off(type, handler) { const set = this._listeners.get(type); if (set) set.delete(handler); } /** * Emit an event to every subscriber. A throwing subscriber does not stop the * others (its error is reported on `console.error`). * @param {string} type * @param {*} [payload] */ emit(type, payload) { const set = this._listeners.get(type); if (!set) return; for (const handler of [...set]) { try { handler(payload); } catch (err) { if (typeof console !== 'undefined') console.error(`listener for "${type}" threw`, err); } } } } /* -------------------------------------------------------------------------- */ /* WebSocket client */ /* -------------------------------------------------------------------------- */ /** Resolve the WebSocket constructor: explicit option, then the global one. */ function resolveWebSocket(options) { if (options.WebSocket) return options.WebSocket; if (typeof globalThis.WebSocket !== 'undefined') return globalThis.WebSocket; throw new Error( 'no WebSocket implementation available — on Node < 22 pass one, e.g. ' + "new AudioAgentClient({ baseUrl, WebSocket: require('ws') })", ); } /** Build the `ws(s)://…/ws` URL from either a full `url` or an http `baseUrl`. */ function toWsUrl(url, baseUrl, path) { if (url) return url; if (!baseUrl) throw new Error('AudioAgentClient needs either `url` or `baseUrl`'); const base = String(baseUrl).replace(/\/+$/, ''); const wsBase = base.replace(/^http:/i, 'ws:').replace(/^https:/i, 'wss:'); return wsBase + path; } /** * Real-time session over the engine's `/ws` endpoint. * * Wire protocol: * - **uplink** `sendAudio()` — binary 16-bit LE mono PCM (44.1 kHz); * - **downlink** `audio` event — binary 16-bit LE mono PCM (44.1 kHz); * - **text** — JSON control messages, surfaced as one event per `type`. */ export class AudioAgentClient extends Emitter { /** * @param {object} [options] * @param {string} [options.url] full `ws://`/`wss://` URL (wins over `baseUrl`) * @param {string} [options.baseUrl] `http(s)://host:port`, the `/ws` path is appended * @param {string} [options.path='/ws'] path appended to `baseUrl` * @param {boolean} [options.autoReconnect=false] reconnect after an unexpected close * @param {number} [options.reconnectMinDelayMs=500] * @param {number} [options.reconnectMaxDelayMs=8000] * @param {typeof WebSocket} [options.WebSocket] inject a WebSocket (Node < 22) */ constructor(options = {}) { super(); this._options = options; this.url = toWsUrl(options.url, options.baseUrl, options.path || '/ws'); this.autoReconnect = options.autoReconnect === true; this.reconnectMinDelayMs = options.reconnectMinDelayMs ?? 500; this.reconnectMaxDelayMs = options.reconnectMaxDelayMs ?? 8000; /** @type {WebSocket|null} */ this._ws = null; /** @type {'idle'|'connecting'|'open'|'closed'} */ this._state = 'idle'; this._reconnectAttempts = 0; /** @type {ReturnType|null} */ this._reconnectTimer = null; this._closing = false; } /** @returns {'idle'|'connecting'|'open'|'closed'} */ get state() { return this._state; } /** Whether the socket is open and ready to send. */ get connected() { return this._state === 'open'; } /** * Open the session. Resolves once the socket is open; rejects if the handshake * fails or the socket closes before opening. * @returns {Promise} */ connect() { return new Promise((resolve, reject) => { if (this._ws && (this._ws.readyState === 0 || this._ws.readyState === 1)) { resolve(this); return; } this._closing = false; clearTimeout(this._reconnectTimer ?? undefined); let ws; try { const WS = resolveWebSocket(this._options); ws = new WS(this.url); } catch (err) { reject(err); return; } ws.binaryType = 'arraybuffer'; this._ws = ws; this._setState('connecting'); let settled = false; ws.onopen = () => { this._reconnectAttempts = 0; this._setState('open'); this.emit('open', { url: this.url }); if (!settled) { settled = true; resolve(this); } }; ws.onmessage = (ev) => this._onMessage(ev.data); ws.onerror = (ev) => { const error = new Error('WebSocket error'); this.emit('error', { error, event: ev, fatal: !settled }); if (!settled) { settled = true; reject(error); } }; ws.onclose = (ev) => { this._ws = null; this._setState('closed'); this.emit('close', { code: ev.code, reason: ev.reason, wasClean: ev.wasClean }); if (!settled) { settled = true; reject(new Error(`WebSocket closed before opening (code ${ev.code})`)); } if (!this._closing && this.autoReconnect) this._scheduleReconnect(); }; }); } /** * Close the session (stops auto-reconnect). * @param {number} [code=1000] * @param {string} [reason=''] */ close(code = 1000, reason = '') { this._closing = true; clearTimeout(this._reconnectTimer ?? undefined); this._reconnectTimer = null; const ws = this._ws; this._ws = null; if (ws && (ws.readyState === 0 || ws.readyState === 1)) ws.close(code, reason); } /** * Send one PCM chunk (any size; 882 samples keeps the engine's 20 ms cadence). * @param {Int16Array|Float32Array|ArrayBuffer|ArrayBufferView|number[]} chunk * @returns {boolean} whether the chunk was sent */ sendAudio(chunk) { const ws = this._ws; if (!ws || ws.readyState !== 1) return false; ws.send(toInt16(chunk)); return true; } /** * Send a raw text frame to the engine. * @param {string} text * @returns {boolean} */ sendText(text) { const ws = this._ws; if (!ws || ws.readyState !== 1) return false; ws.send(text); return true; } /** * Send a JSON control object. * @param {object} message * @returns {boolean} */ sendControl(message) { return this.sendText(JSON.stringify(message)); } /** * Run a text turn through the LLM (same path as a spoken utterance). * @param {string} text * @returns {boolean} */ sendUserText(text) { return this.sendControl({ type: 'user_text', text }); } /** Flush the queued audio and cancel the in-flight reply (barge-in). */ interrupt() { return this.sendControl({ type: 'interrupt' }); } /** * Play an external audio source (URL / radio / YouTube) on the engine's * dedicated mixer channel (not interrupted by barge-in). * @param {string} url */ playAudioSource(url) { return this.sendControl({ type: 'audio_source', action: 'play', url }); } /** Stop the external audio source. */ stopAudioSource() { return this.sendControl({ type: 'audio_source', action: 'stop' }); } /** * Set the external audio source gain. * @param {number} gain 0..1 */ setAudioSourceVolume(gain) { return this.sendControl({ type: 'audio_source', action: 'volume', gain }); } /** * Install a voice-cloning reference (legacy text control). * @param {string} path audio path readable by the TTS backend * @param {string} [text] matching transcript */ setReference(path, text) { return this.sendText(`SET_REFERENCE:${path}${text ? `|${text}` : ''}`); } /** Drop the voice-cloning reference. */ clearReference() { return this.sendText('CLEAR_REFERENCE'); } /** * Synthesize `text` as speech (TTS pipeline test). * @param {string} text */ speak(text) { return this.sendText(text); } _setState(state) { if (state === this._state) return; this._state = state; this.emit('state', state); } _scheduleReconnect() { const attempt = this._reconnectAttempts++; const delay = Math.min( this.reconnectMinDelayMs * 2 ** attempt, this.reconnectMaxDelayMs, ); this.emit('reconnecting', { attempt, delay }); this._reconnectTimer = setTimeout(() => { this.connect().catch((err) => this.emit('error', { error: err, fatal: false })); }, delay); } async _onMessage(data) { if (typeof data === 'string') { this._emitText(data); return; } let buffer; if (data instanceof ArrayBuffer) buffer = data; else if (ArrayBuffer.isView(data)) { buffer = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength); } else if (typeof Blob !== 'undefined' && data instanceof Blob) { buffer = await data.arrayBuffer(); } else { return; } const samples = int16ArrayFromBytes(buffer); this.emit('audio', { samples, buffer, sampleRate: AUDIO.sampleRate }); } _emitText(text) { let message; try { message = JSON.parse(text); } catch { this.emit('event', { type: 'unknown', raw: text }); this.emit('unknown', { raw: text }); return; } this.emit('event', message); this.emit(typeof message.type === 'string' ? message.type : 'unknown', message); } } /* -------------------------------------------------------------------------- */ /* REST client */ /* -------------------------------------------------------------------------- */ /** * Typed wrapper over the engine's REST surface (health, tools, custom voice, * transcribe, OpenAPI). Works in the browser and in Node 18+ (global `fetch`). */ export class AudioAgentRest { /** * @param {string} baseUrl engine origin, e.g. `http://localhost:8787` * @param {object} [options] * @param {typeof fetch} [options.fetch] inject a fetch implementation * @param {Record} [options.headers] extra headers on every call */ constructor(baseUrl, options = {}) { this.baseUrl = String(baseUrl).replace(/\/+$/, ''); this._fetch = options.fetch ?? ((...args) => globalThis.fetch(...args)); this._headers = options.headers ?? {}; } /** * `GET /health` — liveness probe. * @returns {Promise} the response body, e.g. `"ok\n"` */ async health() { const res = await this._fetch(`${this.baseUrl}/health`); if (!res.ok) throw this._error('GET', '/health', res.status, null); return res.text(); } /** * `GET /api/tools` — active tool definitions and their source. * @returns {Promise<{ok:boolean,count:number,source:'file'|'request'|'none',tools:object[]}>} */ getTools() { return this._json('GET', '/api/tools'); } /** * `POST /api/tools` — install tool definitions at runtime (in-memory until the * next call; a restart restores the `LLM_TOOLS_FILE` ones). * @param {object[]} tools JSON array of tool definitions * @returns {Promise<{ok:boolean,count:number,source:string,tools:object[]}>} */ setTools(tools) { return this._json('POST', '/api/tools', { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(tools), }); } /** * `GET /api/prompt` — the system prompt for a session (or the process-wide * default when `sessionId` is omitted). * @param {string} [sessionId] session id from the `session` event * @returns {Promise<{ok:boolean,session_id:string|null,source:string,prompt:string,has_override:boolean,direct_answers_applied:boolean}>} */ getPrompt(sessionId) { const query = sessionId ? `?${new URLSearchParams({ session_id: sessionId })}` : ''; return this._json('GET', `/api/prompt${query}`); } /** * `POST /api/prompt` — set (or clear) the system prompt for a session, or the * process-wide fallback when `sessionId` is omitted. Pass `null` as `prompt` * to clear the override and fall back to the configured default. * @param {string|null} prompt * @param {string} [sessionId] * @returns {Promise<{ok:boolean,session_id:string|null,source:string,prompt:string,has_override:boolean,direct_answers_applied:boolean}>} */ setPrompt(prompt, sessionId) { return this._json('POST', '/api/prompt', { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ session_id: sessionId ?? null, prompt: prompt ?? null }), }); } /** * `POST /api/auth/phone-login` — authenticate a phone number through the * engine's proxy to the AquaChile auth API. Resolves with the upstream * payload (`sessionToken` + `user`); rejects on a non-2xx response. When * `sessionId` is provided, the engine also stores the user's sanitized * profile as hidden context for that live LLM session. * @param {string} phoneNumber E.164 phone number, e.g. `+447432477433`. * @param {string} [sessionId] * @returns {Promise<{sessionToken?:string,user?:object,[k:string]:any}>} */ phoneLogin(phoneNumber, sessionId) { return this._json('POST', '/api/auth/phone-login', { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ phoneNumber, session_id: sessionId ?? null }), }); } /** * `GET /api/customvoice/phrases` — reading phrases + default language. * @returns {Promise<{default:string,languages:{code:string,label:string,phrase:string}[]}>} */ getCustomVoicePhrases() { return this._json('GET', '/api/customvoice/phrases'); } /** * `POST /api/customvoice` — install a voice-cloning reference from a recording. * @param {string} lang ISO 639-1 code (`es`, `en`, `it`, `fr`, `de`, `pt`) * @param {Blob|ArrayBuffer|ArrayBufferView} audio raw audio bytes * @param {string} [text] transcript override (defaults to the reading phrase) * @returns {Promise<{ok:boolean,lang:string,audio:string,phrase:string}>} */ setCustomVoice(lang, audio, text) { const query = new URLSearchParams({ lang }); if (text) query.set('text', text); return this._json('POST', `/api/customvoice?${query}`, { headers: { 'Content-Type': 'application/octet-stream' }, body: audio, }); } /** * `POST /api/customvoice/clear` — restore the configured reference. * @returns {Promise<{ok:boolean}>} */ clearCustomVoice() { return this._json('POST', '/api/customvoice/clear'); } /** * `POST /api/transcribe` — transcribe an audio file (any ffmpeg-readable container). * @param {Blob|ArrayBuffer|ArrayBufferView} audio * @returns {Promise<{ok:boolean,text:string,duration_ms:number,tokens:number}>} */ transcribe(audio) { return this._json('POST', '/api/transcribe', { headers: { 'Content-Type': 'application/octet-stream' }, body: audio, }); } /** * `GET /api-docs/openapi.json` — the OpenAPI 3 document. * @returns {Promise} */ openapi() { return this._json('GET', '/api-docs/openapi.json'); } async _json(method, path, init = {}) { const res = await this._fetch(`${this.baseUrl}${path}`, { method, ...init, headers: { ...this._headers, ...(init.headers ?? {}) }, }); const text = await res.text(); let body = null; if (text) { try { body = JSON.parse(text); } catch { body = text; } } if (!res.ok) throw this._error(method, path, res.status, body); return body; } _error(method, path, status, body) { const detail = (body && body.error) || (typeof body === 'string' ? body : ''); const error = new Error(`${method} ${path} failed (${status})${detail ? `: ${detail}` : ''}`); error.status = status; error.body = body; return error; } } /* -------------------------------------------------------------------------- */ /* Browser audio helpers */ /* -------------------------------------------------------------------------- */ /** AudioWorklet that converts mic audio to 16-bit LE 20 ms frames. */ const CAPTURE_WORKLET = ` class AaeCaptureProcessor extends AudioWorkletProcessor { constructor(options) { super(); const opts = (options && options.processorOptions) || {}; this.frameSamples = opts.frameSamples || 882; this.buf = new Float32Array(this.frameSamples); this.fill = 0; } process(inputs) { const channel = inputs[0] && inputs[0][0]; if (!channel) return true; for (let i = 0; i < channel.length; i++) { this.buf[this.fill++] = channel[i]; if (this.fill === this.frameSamples) { const out = new Int16Array(this.frameSamples); for (let j = 0; j < this.frameSamples; j++) { let s = this.buf[j]; s = s < -1 ? -1 : s > 1 ? 1 : s; out[j] = s < 0 ? s * 0x8000 : s * 0x7fff; } this.port.postMessage(out, [out.buffer]); this.fill = 0; } } return true; } } registerProcessor('aae-capture', AaeCaptureProcessor); `; let _captureWorkletUrl = null; /** Lazily create the Blob URL for the capture worklet (keeps the SDK single-file). */ function captureWorkletUrl() { if (!_captureWorkletUrl) { _captureWorkletUrl = URL.createObjectURL( new Blob([CAPTURE_WORKLET], { type: 'application/javascript' }), ); } return _captureWorkletUrl; } /** Resolve the browser's AudioContext constructor. */ function audioContextCtor() { const Ctor = globalThis.AudioContext || globalThis.webkitAudioContext; if (!Ctor) { throw new Error('the Web Audio API is unavailable — MicCapture/PcmPlayer are browser-only helpers'); } return Ctor; } /** * Microphone capture: emits `frame` events carrying `Int16Array` chunks of * exactly `frameSamples` samples (882 = 20 ms at 44.1 kHz), ready for * {@link AudioAgentClient#sendAudio}. */ export class MicCapture extends Emitter { /** * @param {object} [options] * @param {number} [options.sampleRate=44100] * @param {number} [options.frameSamples=882] * @param {MediaStreamConstraints} [options.constraints] */ constructor(options = {}) { super(); this.sampleRate = options.sampleRate ?? AUDIO.sampleRate; this.frameSamples = options.frameSamples ?? AUDIO.frameSamples; this.constraints = options.constraints ?? { audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true, }, }; /** @type {AudioContext|null} */ this._ctx = null; /** @type {MediaStream|null} */ this._stream = null; /** @type {AudioWorkletNode|null} */ this._node = null; } /** Actual sample rate of the running AudioContext, or `null` when stopped. */ get contextSampleRate() { return this._ctx ? this._ctx.sampleRate : null; } /** Request the microphone and start emitting `frame` events. */ async start() { if (this._ctx) return; const Ctor = audioContextCtor(); this._stream = await navigator.mediaDevices.getUserMedia(this.constraints); this._ctx = new Ctor({ sampleRate: this.sampleRate, latencyHint: 'interactive' }); if (this._ctx.state === 'suspended') await this._ctx.resume(); await this._ctx.audioWorklet.addModule(captureWorkletUrl()); const node = new AudioWorkletNode(this._ctx, 'aae-capture', { numberOfInputs: 1, numberOfOutputs: 0, processorOptions: { frameSamples: this.frameSamples }, }); node.port.onmessage = (ev) => { const samples = ev.data instanceof Int16Array ? ev.data : new Int16Array(ev.data); this.emit('frame', samples); }; this._ctx.createMediaStreamSource(this._stream).connect(node); this._node = node; this.emit('start', { sampleRate: this._ctx.sampleRate }); } /** Stop capture and release the microphone. */ stop() { if (this._node) { this._node.port.onmessage = null; this._node.disconnect(); this._node = null; } if (this._stream) { for (const track of this._stream.getTracks()) track.stop(); this._stream = null; } if (this._ctx) { this._ctx.close().catch(() => {}); this._ctx = null; } this.emit('stop'); } } /** * Jitter-buffered PCM player: feed it `Int16Array` chunks (the `audio` event of * {@link AudioAgentClient}) and it schedules them back-to-back on a Web Audio * timeline, waiting for `bufferMs` of audio before the first chunk to absorb * network jitter. */ export class PcmPlayer extends Emitter { /** * @param {object} [options] * @param {number} [options.sampleRate=44100] * @param {number} [options.bufferMs=150] pre-roll before playback starts * @param {number} [options.gain=1] */ constructor(options = {}) { super(); this.sampleRate = options.sampleRate ?? AUDIO.sampleRate; this.bufferMs = options.bufferMs ?? 150; this.gain = options.gain ?? 1; /** @type {AudioContext|null} */ this._ctx = null; /** @type {GainNode|null} */ this._gainNode = null; /** @type {Int16Array[]} */ this._queue = []; this._queuedSamples = 0; this._nextTime = 0; this._started = false; /** @type {Set} */ this._sources = new Set(); /** @type {ReturnType|null} */ this._forceTimer = null; } /** Create the AudioContext (call from a user gesture to satisfy autoplay policy). */ async resume() { await this._ensureContext(); if (this._ctx.state === 'suspended') await this._ctx.resume(); } /** * Queue a PCM chunk. * @param {Int16Array|Float32Array|ArrayBuffer|ArrayBufferView|number[]} chunk */ push(chunk) { const samples = toInt16(chunk); if (!samples.length) return; this._ensureContext().then(() => { this._queue.push(samples); this._queuedSamples += samples.length; if (this._started) { this._drain(); } else if (this._millisQueued() >= this.bufferMs) { this._started = true; this._drain(); } else if (!this._forceTimer) { this._forceTimer = setTimeout(() => { this._forceTimer = null; if (!this._started) { this._started = true; this._drain(); } }, this.bufferMs); } }); } /** * Set the playback gain. * @param {number} value */ setGain(value) { this.gain = value; if (this._gainNode) this._gainNode.gain.value = value; } /** Drop everything queued and stop the scheduled audio (barge-in). */ flush() { clearTimeout(this._forceTimer ?? undefined); this._forceTimer = null; for (const source of this._sources) { try { source.stop(); } catch { /* already stopped */ } } this._sources.clear(); this._queue.length = 0; this._queuedSamples = 0; this._nextTime = 0; this._started = false; this.emit('flush'); this.emit('idle'); } /** Stop playback and close the AudioContext. */ close() { this.flush(); if (this._ctx) { this._ctx.close().catch(() => {}); this._ctx = null; this._gainNode = null; } } _millisQueued() { return (this._queuedSamples / this.sampleRate) * 1000; } /** Whether no PCM is queued or scheduled for playback. */ isIdle() { return this._queue.length === 0 && this._sources.size === 0; } async _ensureContext() { if (this._ctx) return; const Ctor = audioContextCtor(); this._ctx = new Ctor({ sampleRate: this.sampleRate, latencyHint: 'interactive' }); this._gainNode = this._ctx.createGain(); this._gainNode.gain.value = this.gain; this._gainNode.connect(this._ctx.destination); } _drain() { if (!this._ctx || !this._gainNode) return; while (this._queue.length) { const samples = this._queue.shift(); this._queuedSamples -= samples.length; const buffer = this._ctx.createBuffer(1, samples.length, this.sampleRate); const channel = buffer.getChannelData(0); for (let i = 0; i < samples.length; i++) channel[i] = samples[i] / 32768; const source = this._ctx.createBufferSource(); source.buffer = buffer; source.connect(this._gainNode); const startAt = Math.max(this._ctx.currentTime, this._nextTime); source.start(startAt); source.onended = () => { this._sources.delete(source); if (this.isIdle()) this.emit('idle'); }; this._sources.add(source); this._nextTime = startAt + buffer.duration; } } } /** * One-object convenience wrapper: connects an {@link AudioAgentClient}, pumps the * microphone into it and plays the returned audio, re-emitting every client * event on the session. * * ```js * const session = new AudioSession({ baseUrl: 'http://localhost:8787' }); * session.on('assistant_delta', (e) => render(e.text)); * await session.start(); * ``` */ export class AudioSession extends Emitter { /** * @param {object} [options] forwarded to {@link AudioAgentClient}; extra keys: * @param {AudioAgentClient} [options.client] reuse an existing client * @param {PcmPlayer} [options.player] reuse an existing player * @param {number} [options.bufferMs=150] playback jitter buffer * @param {boolean} [options.forwardMic=true] stream the microphone automatically */ constructor(options = {}) { super(); this.client = options.client ?? new AudioAgentClient(options); this.player = options.player ?? new PcmPlayer({ bufferMs: options.bufferMs ?? 150 }); this.forwardMic = options.forwardMic !== false; /** @type {MicCapture|null} */ this.mic = null; this._wire(); } _wire() { this.client.on('audio', (e) => this.player.push(e.samples)); this.client.on('barge_in', () => this.player.flush()); for (const type of [...CLIENT_EVENTS, 'open', 'close', 'state', 'reconnecting']) { this.client.on(type, (payload) => this.emit(type, payload)); } } /** * Connect, resume playback and (unless disabled) start the microphone. * @param {object} [options] * @param {boolean} [options.microphone=true] */ async start(options = {}) { await this.player.resume(); await this.client.connect(); if (options.microphone !== false && this.forwardMic) { this.mic = new MicCapture({ sampleRate: AUDIO.sampleRate }); this.mic.on('frame', (samples) => this.client.sendAudio(samples)); await this.mic.start(); } } /** Run a text turn through the LLM. */ sendUserText(text) { return this.client.sendUserText(text); } /** Barge-in: cancel the in-flight reply and flush what is queued locally. */ interrupt() { this.player.flush(); return this.client.interrupt(); } /** Stop the microphone, close the socket and release the audio devices. */ async stop() { if (this.mic) { this.mic.stop(); this.mic = null; } this.client.close(); this.player.close(); } } export default { AUDIO, CLIENT_EVENTS, int16ArrayFromBytes, float32ToInt16, int16ToFloat32, toInt16, AudioAgentClient, AudioAgentRest, MicCapture, PcmPlayer, AudioSession, };