1014 lines
32 KiB
JavaScript
1014 lines
32 KiB
JavaScript
/**
|
|
* audio-agent-client — JavaScript SDK for the `audio-agent-engine`.
|
|
*
|
|
* Zero dependencies, single-file ESM. The core (WebSocket session + REST
|
|
* client + PCM helpers) runs in the browser **and** in Node; the audio helpers
|
|
* (`MicCapture`, `PcmPlayer`, `AudioSession`) are browser-only and throw a clear
|
|
* error when the Web Audio API is missing.
|
|
*
|
|
* The point of the SDK is decoupling: the engine stays where it is, the client
|
|
* can live on another origin/server. For browser clients on a different origin
|
|
* enable CORS on the engine (`CORS_ALLOW_ORIGIN=https://your.app`); the
|
|
* WebSocket needs no CORS.
|
|
*
|
|
* ```js
|
|
* import { AudioSession } from './audio-agent-client.js';
|
|
*
|
|
* const session = new AudioSession({ baseUrl: 'https://engine.example.com:8787' });
|
|
* session.on('assistant_delta', (e) => console.log(e.text));
|
|
* await session.start(); // connect + mic + playback
|
|
* session.sendUserText('ciao');
|
|
* ```
|
|
*
|
|
* @module audio-agent-client
|
|
*/
|
|
|
|
/**
|
|
* Audio wire format shared with the engine (see `src/config.rs`).
|
|
* The engine speaks **16-bit little-endian mono PCM at 44.1 kHz** in both
|
|
* directions; uplink frames are 20 ms (882 samples / 1764 bytes) and the server
|
|
* resamples to 16 kHz for the STT engine.
|
|
*/
|
|
export const AUDIO = Object.freeze({
|
|
/** Sample rate of the PCM on the wire, both directions. */
|
|
sampleRate: 44100,
|
|
/** Samples per 20 ms frame at {@link AUDIO.sampleRate}. */
|
|
frameSamples: 882,
|
|
/** Bytes per 20 ms frame (16-bit mono). */
|
|
frameBytes: 1764,
|
|
/** Sample rate the engine's STT engine runs at (server-side detail). */
|
|
sttSampleRate: 16000,
|
|
});
|
|
|
|
/** Server → client text event types emitted by the engine. */
|
|
export const CLIENT_EVENTS = Object.freeze([
|
|
'session',
|
|
'cues',
|
|
'user',
|
|
'assistant_delta',
|
|
'assistant_markdown',
|
|
'assistant_end',
|
|
'cue',
|
|
'barge_in',
|
|
'tool_call',
|
|
'audio_source',
|
|
'error',
|
|
]);
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
/* PCM helpers */
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
/**
|
|
* Decode little-endian 16-bit PCM bytes into an `Int16Array`.
|
|
* An odd trailing byte is dropped.
|
|
* @param {ArrayBuffer|ArrayBufferView} buffer
|
|
* @returns {Int16Array}
|
|
*/
|
|
export function int16ArrayFromBytes(buffer) {
|
|
const view =
|
|
buffer instanceof ArrayBuffer
|
|
? new DataView(buffer)
|
|
: new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
|
|
const count = Math.floor(view.byteLength / 2);
|
|
const out = new Int16Array(count);
|
|
for (let i = 0; i < count; i++) out[i] = view.getInt16(i * 2, true);
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Convert normalized floats (-1..1) to 16-bit PCM, clamping out-of-range values.
|
|
* @param {Float32Array|number[]} floats
|
|
* @returns {Int16Array}
|
|
*/
|
|
export function float32ToInt16(floats) {
|
|
const out = new Int16Array(floats.length);
|
|
for (let i = 0; i < floats.length; i++) {
|
|
let s = floats[i];
|
|
s = s < -1 ? -1 : s > 1 ? 1 : s;
|
|
out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Convert 16-bit PCM to normalized floats (-1..1).
|
|
* @param {Int16Array} ints
|
|
* @returns {Float32Array}
|
|
*/
|
|
export function int16ToFloat32(ints) {
|
|
const out = new Float32Array(ints.length);
|
|
for (let i = 0; i < ints.length; i++) out[i] = ints[i] / 32768;
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Normalize any accepted audio chunk to `Int16Array`.
|
|
* @param {Int16Array|Float32Array|ArrayBuffer|ArrayBufferView|number[]} chunk
|
|
* @returns {Int16Array}
|
|
*/
|
|
export function toInt16(chunk) {
|
|
if (chunk instanceof Int16Array) return chunk;
|
|
if (chunk instanceof Float32Array) return float32ToInt16(chunk);
|
|
if (Array.isArray(chunk)) return Int16Array.from(chunk);
|
|
if (chunk instanceof ArrayBuffer) return int16ArrayFromBytes(chunk);
|
|
if (ArrayBuffer.isView(chunk)) {
|
|
return int16ArrayFromBytes(
|
|
chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength),
|
|
);
|
|
}
|
|
throw new TypeError('unsupported audio chunk: expected Int16Array, Float32Array, ArrayBuffer or number[]');
|
|
}
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
/* Tiny event emitter (browser + Node, no CustomEvent dependency) */
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
class Emitter {
|
|
constructor() {
|
|
/** @type {Map<string, Set<Function>>} */
|
|
this._listeners = new Map();
|
|
}
|
|
|
|
/**
|
|
* Subscribe to an event. Returns an unsubscribe function.
|
|
* @param {string} type
|
|
* @param {Function} handler
|
|
* @returns {() => void}
|
|
*/
|
|
on(type, handler) {
|
|
let set = this._listeners.get(type);
|
|
if (!set) this._listeners.set(type, (set = new Set()));
|
|
set.add(handler);
|
|
return () => this.off(type, handler);
|
|
}
|
|
|
|
/**
|
|
* Subscribe once.
|
|
* @param {string} type
|
|
* @param {Function} handler
|
|
* @returns {() => void}
|
|
*/
|
|
once(type, handler) {
|
|
const off = this.on(type, (payload) => {
|
|
off();
|
|
handler(payload);
|
|
});
|
|
return off;
|
|
}
|
|
|
|
/**
|
|
* Unsubscribe.
|
|
* @param {string} type
|
|
* @param {Function} handler
|
|
*/
|
|
off(type, handler) {
|
|
const set = this._listeners.get(type);
|
|
if (set) set.delete(handler);
|
|
}
|
|
|
|
/**
|
|
* Emit an event to every subscriber. A throwing subscriber does not stop the
|
|
* others (its error is reported on `console.error`).
|
|
* @param {string} type
|
|
* @param {*} [payload]
|
|
*/
|
|
emit(type, payload) {
|
|
const set = this._listeners.get(type);
|
|
if (!set) return;
|
|
for (const handler of [...set]) {
|
|
try {
|
|
handler(payload);
|
|
} catch (err) {
|
|
if (typeof console !== 'undefined') console.error(`listener for "${type}" threw`, err);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
/* WebSocket client */
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
/** Resolve the WebSocket constructor: explicit option, then the global one. */
|
|
function resolveWebSocket(options) {
|
|
if (options.WebSocket) return options.WebSocket;
|
|
if (typeof globalThis.WebSocket !== 'undefined') return globalThis.WebSocket;
|
|
throw new Error(
|
|
'no WebSocket implementation available — on Node < 22 pass one, e.g. ' +
|
|
"new AudioAgentClient({ baseUrl, WebSocket: require('ws') })",
|
|
);
|
|
}
|
|
|
|
/** Build the `ws(s)://…/ws` URL from either a full `url` or an http `baseUrl`. */
|
|
function toWsUrl(url, baseUrl, path) {
|
|
if (url) return url;
|
|
if (!baseUrl) throw new Error('AudioAgentClient needs either `url` or `baseUrl`');
|
|
const base = String(baseUrl).replace(/\/+$/, '');
|
|
const wsBase = base.replace(/^http:/i, 'ws:').replace(/^https:/i, 'wss:');
|
|
return wsBase + path;
|
|
}
|
|
|
|
/**
|
|
* Real-time session over the engine's `/ws` endpoint.
|
|
*
|
|
* Wire protocol:
|
|
* - **uplink** `sendAudio()` — binary 16-bit LE mono PCM (44.1 kHz);
|
|
* - **downlink** `audio` event — binary 16-bit LE mono PCM (44.1 kHz);
|
|
* - **text** — JSON control messages, surfaced as one event per `type`.
|
|
*/
|
|
export class AudioAgentClient extends Emitter {
|
|
/**
|
|
* @param {object} [options]
|
|
* @param {string} [options.url] full `ws://`/`wss://` URL (wins over `baseUrl`)
|
|
* @param {string} [options.baseUrl] `http(s)://host:port`, the `/ws` path is appended
|
|
* @param {string} [options.path='/ws'] path appended to `baseUrl`
|
|
* @param {boolean} [options.autoReconnect=false] reconnect after an unexpected close
|
|
* @param {number} [options.reconnectMinDelayMs=500]
|
|
* @param {number} [options.reconnectMaxDelayMs=8000]
|
|
* @param {typeof WebSocket} [options.WebSocket] inject a WebSocket (Node < 22)
|
|
*/
|
|
constructor(options = {}) {
|
|
super();
|
|
this._options = options;
|
|
this.url = toWsUrl(options.url, options.baseUrl, options.path || '/ws');
|
|
this.autoReconnect = options.autoReconnect === true;
|
|
this.reconnectMinDelayMs = options.reconnectMinDelayMs ?? 500;
|
|
this.reconnectMaxDelayMs = options.reconnectMaxDelayMs ?? 8000;
|
|
/** @type {WebSocket|null} */
|
|
this._ws = null;
|
|
/** @type {'idle'|'connecting'|'open'|'closed'} */
|
|
this._state = 'idle';
|
|
this._reconnectAttempts = 0;
|
|
/** @type {ReturnType<typeof setTimeout>|null} */
|
|
this._reconnectTimer = null;
|
|
this._closing = false;
|
|
}
|
|
|
|
/** @returns {'idle'|'connecting'|'open'|'closed'} */
|
|
get state() {
|
|
return this._state;
|
|
}
|
|
|
|
/** Whether the socket is open and ready to send. */
|
|
get connected() {
|
|
return this._state === 'open';
|
|
}
|
|
|
|
/**
|
|
* Open the session. Resolves once the socket is open; rejects if the handshake
|
|
* fails or the socket closes before opening.
|
|
* @returns {Promise<this>}
|
|
*/
|
|
connect() {
|
|
return new Promise((resolve, reject) => {
|
|
if (this._ws && (this._ws.readyState === 0 || this._ws.readyState === 1)) {
|
|
resolve(this);
|
|
return;
|
|
}
|
|
this._closing = false;
|
|
clearTimeout(this._reconnectTimer ?? undefined);
|
|
|
|
let ws;
|
|
try {
|
|
const WS = resolveWebSocket(this._options);
|
|
ws = new WS(this.url);
|
|
} catch (err) {
|
|
reject(err);
|
|
return;
|
|
}
|
|
ws.binaryType = 'arraybuffer';
|
|
this._ws = ws;
|
|
this._setState('connecting');
|
|
|
|
let settled = false;
|
|
ws.onopen = () => {
|
|
this._reconnectAttempts = 0;
|
|
this._setState('open');
|
|
this.emit('open', { url: this.url });
|
|
if (!settled) {
|
|
settled = true;
|
|
resolve(this);
|
|
}
|
|
};
|
|
ws.onmessage = (ev) => this._onMessage(ev.data);
|
|
ws.onerror = (ev) => {
|
|
const error = new Error('WebSocket error');
|
|
this.emit('error', { error, event: ev, fatal: !settled });
|
|
if (!settled) {
|
|
settled = true;
|
|
reject(error);
|
|
}
|
|
};
|
|
ws.onclose = (ev) => {
|
|
this._ws = null;
|
|
this._setState('closed');
|
|
this.emit('close', { code: ev.code, reason: ev.reason, wasClean: ev.wasClean });
|
|
if (!settled) {
|
|
settled = true;
|
|
reject(new Error(`WebSocket closed before opening (code ${ev.code})`));
|
|
}
|
|
if (!this._closing && this.autoReconnect) this._scheduleReconnect();
|
|
};
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Close the session (stops auto-reconnect).
|
|
* @param {number} [code=1000]
|
|
* @param {string} [reason='']
|
|
*/
|
|
close(code = 1000, reason = '') {
|
|
this._closing = true;
|
|
clearTimeout(this._reconnectTimer ?? undefined);
|
|
this._reconnectTimer = null;
|
|
const ws = this._ws;
|
|
this._ws = null;
|
|
if (ws && (ws.readyState === 0 || ws.readyState === 1)) ws.close(code, reason);
|
|
}
|
|
|
|
/**
|
|
* Send one PCM chunk (any size; 882 samples keeps the engine's 20 ms cadence).
|
|
* @param {Int16Array|Float32Array|ArrayBuffer|ArrayBufferView|number[]} chunk
|
|
* @returns {boolean} whether the chunk was sent
|
|
*/
|
|
sendAudio(chunk) {
|
|
const ws = this._ws;
|
|
if (!ws || ws.readyState !== 1) return false;
|
|
ws.send(toInt16(chunk));
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Send a raw text frame to the engine.
|
|
* @param {string} text
|
|
* @returns {boolean}
|
|
*/
|
|
sendText(text) {
|
|
const ws = this._ws;
|
|
if (!ws || ws.readyState !== 1) return false;
|
|
ws.send(text);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Send a JSON control object.
|
|
* @param {object} message
|
|
* @returns {boolean}
|
|
*/
|
|
sendControl(message) {
|
|
return this.sendText(JSON.stringify(message));
|
|
}
|
|
|
|
/**
|
|
* Run a text turn through the LLM (same path as a spoken utterance).
|
|
* @param {string} text
|
|
* @returns {boolean}
|
|
*/
|
|
sendUserText(text) {
|
|
return this.sendControl({ type: 'user_text', text });
|
|
}
|
|
|
|
/** Flush the queued audio and cancel the in-flight reply (barge-in). */
|
|
interrupt() {
|
|
return this.sendControl({ type: 'interrupt' });
|
|
}
|
|
|
|
/**
|
|
* Play an external audio source (URL / radio / YouTube) on the engine's
|
|
* dedicated mixer channel (not interrupted by barge-in).
|
|
* @param {string} url
|
|
*/
|
|
playAudioSource(url) {
|
|
return this.sendControl({ type: 'audio_source', action: 'play', url });
|
|
}
|
|
|
|
/** Stop the external audio source. */
|
|
stopAudioSource() {
|
|
return this.sendControl({ type: 'audio_source', action: 'stop' });
|
|
}
|
|
|
|
/**
|
|
* Set the external audio source gain.
|
|
* @param {number} gain 0..1
|
|
*/
|
|
setAudioSourceVolume(gain) {
|
|
return this.sendControl({ type: 'audio_source', action: 'volume', gain });
|
|
}
|
|
|
|
/**
|
|
* Install a voice-cloning reference (legacy text control).
|
|
* @param {string} path audio path readable by the TTS backend
|
|
* @param {string} [text] matching transcript
|
|
*/
|
|
setReference(path, text) {
|
|
return this.sendText(`SET_REFERENCE:${path}${text ? `|${text}` : ''}`);
|
|
}
|
|
|
|
/** Drop the voice-cloning reference. */
|
|
clearReference() {
|
|
return this.sendText('CLEAR_REFERENCE');
|
|
}
|
|
|
|
/**
|
|
* Synthesize `text` as speech (TTS pipeline test).
|
|
* @param {string} text
|
|
*/
|
|
speak(text) {
|
|
return this.sendText(text);
|
|
}
|
|
|
|
_setState(state) {
|
|
if (state === this._state) return;
|
|
this._state = state;
|
|
this.emit('state', state);
|
|
}
|
|
|
|
_scheduleReconnect() {
|
|
const attempt = this._reconnectAttempts++;
|
|
const delay = Math.min(
|
|
this.reconnectMinDelayMs * 2 ** attempt,
|
|
this.reconnectMaxDelayMs,
|
|
);
|
|
this.emit('reconnecting', { attempt, delay });
|
|
this._reconnectTimer = setTimeout(() => {
|
|
this.connect().catch((err) => this.emit('error', { error: err, fatal: false }));
|
|
}, delay);
|
|
}
|
|
|
|
async _onMessage(data) {
|
|
if (typeof data === 'string') {
|
|
this._emitText(data);
|
|
return;
|
|
}
|
|
let buffer;
|
|
if (data instanceof ArrayBuffer) buffer = data;
|
|
else if (ArrayBuffer.isView(data)) {
|
|
buffer = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
} else if (typeof Blob !== 'undefined' && data instanceof Blob) {
|
|
buffer = await data.arrayBuffer();
|
|
} else {
|
|
return;
|
|
}
|
|
const samples = int16ArrayFromBytes(buffer);
|
|
this.emit('audio', { samples, buffer, sampleRate: AUDIO.sampleRate });
|
|
}
|
|
|
|
_emitText(text) {
|
|
let message;
|
|
try {
|
|
message = JSON.parse(text);
|
|
} catch {
|
|
this.emit('event', { type: 'unknown', raw: text });
|
|
this.emit('unknown', { raw: text });
|
|
return;
|
|
}
|
|
this.emit('event', message);
|
|
this.emit(typeof message.type === 'string' ? message.type : 'unknown', message);
|
|
}
|
|
}
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
/* REST client */
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
/**
|
|
* Typed wrapper over the engine's REST surface (health, tools, custom voice,
|
|
* transcribe, OpenAPI). Works in the browser and in Node 18+ (global `fetch`).
|
|
*/
|
|
export class AudioAgentRest {
|
|
/**
|
|
* @param {string} baseUrl engine origin, e.g. `http://localhost:8787`
|
|
* @param {object} [options]
|
|
* @param {typeof fetch} [options.fetch] inject a fetch implementation
|
|
* @param {Record<string,string>} [options.headers] extra headers on every call
|
|
*/
|
|
constructor(baseUrl, options = {}) {
|
|
this.baseUrl = String(baseUrl).replace(/\/+$/, '');
|
|
this._fetch = options.fetch ?? ((...args) => globalThis.fetch(...args));
|
|
this._headers = options.headers ?? {};
|
|
}
|
|
|
|
/**
|
|
* `GET /health` — liveness probe.
|
|
* @returns {Promise<string>} the response body, e.g. `"ok\n"`
|
|
*/
|
|
async health() {
|
|
const res = await this._fetch(`${this.baseUrl}/health`);
|
|
if (!res.ok) throw this._error('GET', '/health', res.status, null);
|
|
return res.text();
|
|
}
|
|
|
|
/**
|
|
* `GET /api/tools` — active tool definitions and their source.
|
|
* @returns {Promise<{ok:boolean,count:number,source:'file'|'request'|'none',tools:object[]}>}
|
|
*/
|
|
getTools() {
|
|
return this._json('GET', '/api/tools');
|
|
}
|
|
|
|
/**
|
|
* `POST /api/tools` — install tool definitions at runtime (in-memory until the
|
|
* next call; a restart restores the `LLM_TOOLS_FILE` ones).
|
|
* @param {object[]} tools JSON array of tool definitions
|
|
* @returns {Promise<{ok:boolean,count:number,source:string,tools:object[]}>}
|
|
*/
|
|
setTools(tools) {
|
|
return this._json('POST', '/api/tools', {
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(tools),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* `GET /api/prompt` — the system prompt for a session (or the process-wide
|
|
* default when `sessionId` is omitted).
|
|
* @param {string} [sessionId] session id from the `session` event
|
|
* @returns {Promise<{ok:boolean,session_id:string|null,source:string,prompt:string,has_override:boolean,direct_answers_applied:boolean}>}
|
|
*/
|
|
getPrompt(sessionId) {
|
|
const query = sessionId ? `?${new URLSearchParams({ session_id: sessionId })}` : '';
|
|
return this._json('GET', `/api/prompt${query}`);
|
|
}
|
|
|
|
/**
|
|
* `POST /api/prompt` — set (or clear) the system prompt for a session, or the
|
|
* process-wide fallback when `sessionId` is omitted. Pass `null` as `prompt`
|
|
* to clear the override and fall back to the configured default.
|
|
* @param {string|null} prompt
|
|
* @param {string} [sessionId]
|
|
* @returns {Promise<{ok:boolean,session_id:string|null,source:string,prompt:string,has_override:boolean,direct_answers_applied:boolean}>}
|
|
*/
|
|
setPrompt(prompt, sessionId) {
|
|
return this._json('POST', '/api/prompt', {
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ session_id: sessionId ?? null, prompt: prompt ?? null }),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* `POST /api/auth/phone-login` — authenticate a phone number through the
|
|
* engine's proxy to the AquaChile auth API. Resolves with the upstream
|
|
* payload (`sessionToken` + `user`); rejects on a non-2xx response. When
|
|
* `sessionId` is provided, the engine also stores the user's sanitized
|
|
* profile as hidden context for that live LLM session.
|
|
* @param {string} phoneNumber E.164 phone number, e.g. `+447432477433`.
|
|
* @param {string} [sessionId]
|
|
* @returns {Promise<{sessionToken?:string,user?:object,[k:string]:any}>}
|
|
*/
|
|
phoneLogin(phoneNumber, sessionId) {
|
|
return this._json('POST', '/api/auth/phone-login', {
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ phoneNumber, session_id: sessionId ?? null }),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* `GET /api/customvoice/phrases` — reading phrases + default language.
|
|
* @returns {Promise<{default:string,languages:{code:string,label:string,phrase:string}[]}>}
|
|
*/
|
|
getCustomVoicePhrases() {
|
|
return this._json('GET', '/api/customvoice/phrases');
|
|
}
|
|
|
|
/**
|
|
* `POST /api/customvoice` — install a voice-cloning reference from a recording.
|
|
* @param {string} lang ISO 639-1 code (`es`, `en`, `it`, `fr`, `de`, `pt`)
|
|
* @param {Blob|ArrayBuffer|ArrayBufferView} audio raw audio bytes
|
|
* @param {string} [text] transcript override (defaults to the reading phrase)
|
|
* @returns {Promise<{ok:boolean,lang:string,audio:string,phrase:string}>}
|
|
*/
|
|
setCustomVoice(lang, audio, text) {
|
|
const query = new URLSearchParams({ lang });
|
|
if (text) query.set('text', text);
|
|
return this._json('POST', `/api/customvoice?${query}`, {
|
|
headers: { 'Content-Type': 'application/octet-stream' },
|
|
body: audio,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* `POST /api/customvoice/clear` — restore the configured reference.
|
|
* @returns {Promise<{ok:boolean}>}
|
|
*/
|
|
clearCustomVoice() {
|
|
return this._json('POST', '/api/customvoice/clear');
|
|
}
|
|
|
|
/**
|
|
* `POST /api/transcribe` — transcribe an audio file (any ffmpeg-readable container).
|
|
* @param {Blob|ArrayBuffer|ArrayBufferView} audio
|
|
* @returns {Promise<{ok:boolean,text:string,duration_ms:number,tokens:number}>}
|
|
*/
|
|
transcribe(audio) {
|
|
return this._json('POST', '/api/transcribe', {
|
|
headers: { 'Content-Type': 'application/octet-stream' },
|
|
body: audio,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* `GET /api-docs/openapi.json` — the OpenAPI 3 document.
|
|
* @returns {Promise<object>}
|
|
*/
|
|
openapi() {
|
|
return this._json('GET', '/api-docs/openapi.json');
|
|
}
|
|
|
|
async _json(method, path, init = {}) {
|
|
const res = await this._fetch(`${this.baseUrl}${path}`, {
|
|
method,
|
|
...init,
|
|
headers: { ...this._headers, ...(init.headers ?? {}) },
|
|
});
|
|
const text = await res.text();
|
|
let body = null;
|
|
if (text) {
|
|
try {
|
|
body = JSON.parse(text);
|
|
} catch {
|
|
body = text;
|
|
}
|
|
}
|
|
if (!res.ok) throw this._error(method, path, res.status, body);
|
|
return body;
|
|
}
|
|
|
|
_error(method, path, status, body) {
|
|
const detail = (body && body.error) || (typeof body === 'string' ? body : '');
|
|
const error = new Error(`${method} ${path} failed (${status})${detail ? `: ${detail}` : ''}`);
|
|
error.status = status;
|
|
error.body = body;
|
|
return error;
|
|
}
|
|
}
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
/* Browser audio helpers */
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
/** AudioWorklet that converts mic audio to 16-bit LE 20 ms frames. */
|
|
const CAPTURE_WORKLET = `
|
|
class AaeCaptureProcessor extends AudioWorkletProcessor {
|
|
constructor(options) {
|
|
super();
|
|
const opts = (options && options.processorOptions) || {};
|
|
this.frameSamples = opts.frameSamples || 882;
|
|
this.buf = new Float32Array(this.frameSamples);
|
|
this.fill = 0;
|
|
}
|
|
process(inputs) {
|
|
const channel = inputs[0] && inputs[0][0];
|
|
if (!channel) return true;
|
|
for (let i = 0; i < channel.length; i++) {
|
|
this.buf[this.fill++] = channel[i];
|
|
if (this.fill === this.frameSamples) {
|
|
const out = new Int16Array(this.frameSamples);
|
|
for (let j = 0; j < this.frameSamples; j++) {
|
|
let s = this.buf[j];
|
|
s = s < -1 ? -1 : s > 1 ? 1 : s;
|
|
out[j] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
}
|
|
this.port.postMessage(out, [out.buffer]);
|
|
this.fill = 0;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
registerProcessor('aae-capture', AaeCaptureProcessor);
|
|
`;
|
|
|
|
let _captureWorkletUrl = null;
|
|
|
|
/** Lazily create the Blob URL for the capture worklet (keeps the SDK single-file). */
|
|
function captureWorkletUrl() {
|
|
if (!_captureWorkletUrl) {
|
|
_captureWorkletUrl = URL.createObjectURL(
|
|
new Blob([CAPTURE_WORKLET], { type: 'application/javascript' }),
|
|
);
|
|
}
|
|
return _captureWorkletUrl;
|
|
}
|
|
|
|
/** Resolve the browser's AudioContext constructor. */
|
|
function audioContextCtor() {
|
|
const Ctor = globalThis.AudioContext || globalThis.webkitAudioContext;
|
|
if (!Ctor) {
|
|
throw new Error('the Web Audio API is unavailable — MicCapture/PcmPlayer are browser-only helpers');
|
|
}
|
|
return Ctor;
|
|
}
|
|
|
|
/**
|
|
* Microphone capture: emits `frame` events carrying `Int16Array` chunks of
|
|
* exactly `frameSamples` samples (882 = 20 ms at 44.1 kHz), ready for
|
|
* {@link AudioAgentClient#sendAudio}.
|
|
*/
|
|
export class MicCapture extends Emitter {
|
|
/**
|
|
* @param {object} [options]
|
|
* @param {number} [options.sampleRate=44100]
|
|
* @param {number} [options.frameSamples=882]
|
|
* @param {MediaStreamConstraints} [options.constraints]
|
|
*/
|
|
constructor(options = {}) {
|
|
super();
|
|
this.sampleRate = options.sampleRate ?? AUDIO.sampleRate;
|
|
this.frameSamples = options.frameSamples ?? AUDIO.frameSamples;
|
|
this.constraints = options.constraints ?? {
|
|
audio: {
|
|
channelCount: 1,
|
|
echoCancellation: true,
|
|
noiseSuppression: true,
|
|
autoGainControl: true,
|
|
},
|
|
};
|
|
/** @type {AudioContext|null} */
|
|
this._ctx = null;
|
|
/** @type {MediaStream|null} */
|
|
this._stream = null;
|
|
/** @type {AudioWorkletNode|null} */
|
|
this._node = null;
|
|
}
|
|
|
|
/** Actual sample rate of the running AudioContext, or `null` when stopped. */
|
|
get contextSampleRate() {
|
|
return this._ctx ? this._ctx.sampleRate : null;
|
|
}
|
|
|
|
/** Request the microphone and start emitting `frame` events. */
|
|
async start() {
|
|
if (this._ctx) return;
|
|
const Ctor = audioContextCtor();
|
|
this._stream = await navigator.mediaDevices.getUserMedia(this.constraints);
|
|
this._ctx = new Ctor({ sampleRate: this.sampleRate, latencyHint: 'interactive' });
|
|
if (this._ctx.state === 'suspended') await this._ctx.resume();
|
|
await this._ctx.audioWorklet.addModule(captureWorkletUrl());
|
|
const node = new AudioWorkletNode(this._ctx, 'aae-capture', {
|
|
numberOfInputs: 1,
|
|
numberOfOutputs: 0,
|
|
processorOptions: { frameSamples: this.frameSamples },
|
|
});
|
|
node.port.onmessage = (ev) => {
|
|
const samples = ev.data instanceof Int16Array ? ev.data : new Int16Array(ev.data);
|
|
this.emit('frame', samples);
|
|
};
|
|
this._ctx.createMediaStreamSource(this._stream).connect(node);
|
|
this._node = node;
|
|
this.emit('start', { sampleRate: this._ctx.sampleRate });
|
|
}
|
|
|
|
/** Stop capture and release the microphone. */
|
|
stop() {
|
|
if (this._node) {
|
|
this._node.port.onmessage = null;
|
|
this._node.disconnect();
|
|
this._node = null;
|
|
}
|
|
if (this._stream) {
|
|
for (const track of this._stream.getTracks()) track.stop();
|
|
this._stream = null;
|
|
}
|
|
if (this._ctx) {
|
|
this._ctx.close().catch(() => {});
|
|
this._ctx = null;
|
|
}
|
|
this.emit('stop');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Jitter-buffered PCM player: feed it `Int16Array` chunks (the `audio` event of
|
|
* {@link AudioAgentClient}) and it schedules them back-to-back on a Web Audio
|
|
* timeline, waiting for `bufferMs` of audio before the first chunk to absorb
|
|
* network jitter.
|
|
*/
|
|
export class PcmPlayer extends Emitter {
|
|
/**
|
|
* @param {object} [options]
|
|
* @param {number} [options.sampleRate=44100]
|
|
* @param {number} [options.bufferMs=150] pre-roll before playback starts
|
|
* @param {number} [options.gain=1]
|
|
*/
|
|
constructor(options = {}) {
|
|
super();
|
|
this.sampleRate = options.sampleRate ?? AUDIO.sampleRate;
|
|
this.bufferMs = options.bufferMs ?? 150;
|
|
this.gain = options.gain ?? 1;
|
|
/** @type {AudioContext|null} */
|
|
this._ctx = null;
|
|
/** @type {GainNode|null} */
|
|
this._gainNode = null;
|
|
/** @type {Int16Array[]} */
|
|
this._queue = [];
|
|
this._queuedSamples = 0;
|
|
this._nextTime = 0;
|
|
this._started = false;
|
|
/** @type {Set<AudioBufferSourceNode>} */
|
|
this._sources = new Set();
|
|
/** @type {ReturnType<typeof setTimeout>|null} */
|
|
this._forceTimer = null;
|
|
}
|
|
|
|
/** Create the AudioContext (call from a user gesture to satisfy autoplay policy). */
|
|
async resume() {
|
|
await this._ensureContext();
|
|
if (this._ctx.state === 'suspended') await this._ctx.resume();
|
|
}
|
|
|
|
/**
|
|
* Queue a PCM chunk.
|
|
* @param {Int16Array|Float32Array|ArrayBuffer|ArrayBufferView|number[]} chunk
|
|
*/
|
|
push(chunk) {
|
|
const samples = toInt16(chunk);
|
|
if (!samples.length) return;
|
|
this._ensureContext().then(() => {
|
|
this._queue.push(samples);
|
|
this._queuedSamples += samples.length;
|
|
if (this._started) {
|
|
this._drain();
|
|
} else if (this._millisQueued() >= this.bufferMs) {
|
|
this._started = true;
|
|
this._drain();
|
|
} else if (!this._forceTimer) {
|
|
this._forceTimer = setTimeout(() => {
|
|
this._forceTimer = null;
|
|
if (!this._started) {
|
|
this._started = true;
|
|
this._drain();
|
|
}
|
|
}, this.bufferMs);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Set the playback gain.
|
|
* @param {number} value
|
|
*/
|
|
setGain(value) {
|
|
this.gain = value;
|
|
if (this._gainNode) this._gainNode.gain.value = value;
|
|
}
|
|
|
|
/** Drop everything queued and stop the scheduled audio (barge-in). */
|
|
flush() {
|
|
clearTimeout(this._forceTimer ?? undefined);
|
|
this._forceTimer = null;
|
|
for (const source of this._sources) {
|
|
try {
|
|
source.stop();
|
|
} catch {
|
|
/* already stopped */
|
|
}
|
|
}
|
|
this._sources.clear();
|
|
this._queue.length = 0;
|
|
this._queuedSamples = 0;
|
|
this._nextTime = 0;
|
|
this._started = false;
|
|
this.emit('flush');
|
|
this.emit('idle');
|
|
}
|
|
|
|
/** Stop playback and close the AudioContext. */
|
|
close() {
|
|
this.flush();
|
|
if (this._ctx) {
|
|
this._ctx.close().catch(() => {});
|
|
this._ctx = null;
|
|
this._gainNode = null;
|
|
}
|
|
}
|
|
|
|
_millisQueued() {
|
|
return (this._queuedSamples / this.sampleRate) * 1000;
|
|
}
|
|
|
|
/** Whether no PCM is queued or scheduled for playback. */
|
|
isIdle() {
|
|
return this._queue.length === 0 && this._sources.size === 0;
|
|
}
|
|
|
|
async _ensureContext() {
|
|
if (this._ctx) return;
|
|
const Ctor = audioContextCtor();
|
|
this._ctx = new Ctor({ sampleRate: this.sampleRate, latencyHint: 'interactive' });
|
|
this._gainNode = this._ctx.createGain();
|
|
this._gainNode.gain.value = this.gain;
|
|
this._gainNode.connect(this._ctx.destination);
|
|
}
|
|
|
|
_drain() {
|
|
if (!this._ctx || !this._gainNode) return;
|
|
while (this._queue.length) {
|
|
const samples = this._queue.shift();
|
|
this._queuedSamples -= samples.length;
|
|
const buffer = this._ctx.createBuffer(1, samples.length, this.sampleRate);
|
|
const channel = buffer.getChannelData(0);
|
|
for (let i = 0; i < samples.length; i++) channel[i] = samples[i] / 32768;
|
|
const source = this._ctx.createBufferSource();
|
|
source.buffer = buffer;
|
|
source.connect(this._gainNode);
|
|
const startAt = Math.max(this._ctx.currentTime, this._nextTime);
|
|
source.start(startAt);
|
|
source.onended = () => {
|
|
this._sources.delete(source);
|
|
if (this.isIdle()) this.emit('idle');
|
|
};
|
|
this._sources.add(source);
|
|
this._nextTime = startAt + buffer.duration;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* One-object convenience wrapper: connects an {@link AudioAgentClient}, pumps the
|
|
* microphone into it and plays the returned audio, re-emitting every client
|
|
* event on the session.
|
|
*
|
|
* ```js
|
|
* const session = new AudioSession({ baseUrl: 'http://localhost:8787' });
|
|
* session.on('assistant_delta', (e) => render(e.text));
|
|
* await session.start();
|
|
* ```
|
|
*/
|
|
export class AudioSession extends Emitter {
|
|
/**
|
|
* @param {object} [options] forwarded to {@link AudioAgentClient}; extra keys:
|
|
* @param {AudioAgentClient} [options.client] reuse an existing client
|
|
* @param {PcmPlayer} [options.player] reuse an existing player
|
|
* @param {number} [options.bufferMs=150] playback jitter buffer
|
|
* @param {boolean} [options.forwardMic=true] stream the microphone automatically
|
|
*/
|
|
constructor(options = {}) {
|
|
super();
|
|
this.client = options.client ?? new AudioAgentClient(options);
|
|
this.player = options.player ?? new PcmPlayer({ bufferMs: options.bufferMs ?? 150 });
|
|
this.forwardMic = options.forwardMic !== false;
|
|
/** @type {MicCapture|null} */
|
|
this.mic = null;
|
|
this._wire();
|
|
}
|
|
|
|
_wire() {
|
|
this.client.on('audio', (e) => this.player.push(e.samples));
|
|
this.client.on('barge_in', () => this.player.flush());
|
|
for (const type of [...CLIENT_EVENTS, 'open', 'close', 'state', 'reconnecting']) {
|
|
this.client.on(type, (payload) => this.emit(type, payload));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Connect, resume playback and (unless disabled) start the microphone.
|
|
* @param {object} [options]
|
|
* @param {boolean} [options.microphone=true]
|
|
*/
|
|
async start(options = {}) {
|
|
await this.player.resume();
|
|
await this.client.connect();
|
|
if (options.microphone !== false && this.forwardMic) {
|
|
this.mic = new MicCapture({ sampleRate: AUDIO.sampleRate });
|
|
this.mic.on('frame', (samples) => this.client.sendAudio(samples));
|
|
await this.mic.start();
|
|
}
|
|
}
|
|
|
|
/** Run a text turn through the LLM. */
|
|
sendUserText(text) {
|
|
return this.client.sendUserText(text);
|
|
}
|
|
|
|
/** Barge-in: cancel the in-flight reply and flush what is queued locally. */
|
|
interrupt() {
|
|
this.player.flush();
|
|
return this.client.interrupt();
|
|
}
|
|
|
|
/** Stop the microphone, close the socket and release the audio devices. */
|
|
async stop() {
|
|
if (this.mic) {
|
|
this.mic.stop();
|
|
this.mic = null;
|
|
}
|
|
this.client.close();
|
|
this.player.close();
|
|
}
|
|
}
|
|
|
|
export default {
|
|
AUDIO,
|
|
CLIENT_EVENTS,
|
|
int16ArrayFromBytes,
|
|
float32ToInt16,
|
|
int16ToFloat32,
|
|
toInt16,
|
|
AudioAgentClient,
|
|
AudioAgentRest,
|
|
MicCapture,
|
|
PcmPlayer,
|
|
AudioSession,
|
|
};
|