feat(frontend): port realtime audio and session handling to TypeScript
This commit is contained in:
@@ -0,0 +1,592 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { encodePcm } from "../audio/codec";
|
||||
import { AudioEngine, MIC_CONSTRAINTS } from "../audio/engine";
|
||||
import { ApiError, endReservation, post, request } from "../lib/api";
|
||||
import {
|
||||
grantSchema,
|
||||
realtimeEventSchema,
|
||||
sessionSchema,
|
||||
type ConnectionStatus,
|
||||
type FinishedResponse,
|
||||
type FunctionTool,
|
||||
type NoiseGate,
|
||||
type RealtimeEvent,
|
||||
type SessionGrant,
|
||||
type ToolCall,
|
||||
type Transcript,
|
||||
} from "../lib/contracts";
|
||||
|
||||
interface Events {
|
||||
status: ConnectionStatus;
|
||||
transcript: Transcript;
|
||||
finished: FinishedResponse;
|
||||
tool: ToolCall;
|
||||
level: number;
|
||||
queue: { position: number; queueId: string };
|
||||
grant: SessionGrant;
|
||||
readyToJoin: { grant: SessionGrant; expiresSec: number };
|
||||
error: Error;
|
||||
serverError: Error;
|
||||
}
|
||||
export interface ClientOptions {
|
||||
directUrl?: string;
|
||||
sessionUrl?: string;
|
||||
voice: string;
|
||||
instructions: string;
|
||||
tools: FunctionTool[];
|
||||
gate: NoiseGate;
|
||||
context: AudioContext;
|
||||
greeting?: string;
|
||||
}
|
||||
|
||||
export class SessionError extends Error {
|
||||
readonly code: "limit" | "queue-full" | "queue-expired" | "join-expired" | "aborted";
|
||||
constructor(code: SessionError["code"], message: string) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
this.name = "SessionError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Owns one complete session, including pending acquisition, audio, queue tickets and teardown. */
|
||||
export class RealtimeClient {
|
||||
private readonly options: ClientOptions;
|
||||
private readonly events = new EventTarget();
|
||||
private readonly abort = new AbortController();
|
||||
private readonly audio: AudioEngine;
|
||||
private socket: WebSocket | null = null;
|
||||
private currentStatus: ConnectionStatus = "idle";
|
||||
private configured = false;
|
||||
private closed = false;
|
||||
private closing: Promise<void> | null = null;
|
||||
private grant: SessionGrant | null = null;
|
||||
private queueId = "";
|
||||
private joinResolve: (() => void) | null = null;
|
||||
private joinReject: ((reason: Error) => void) | null = null;
|
||||
private joinTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
private heartbeat: ReturnType<typeof setInterval> | undefined;
|
||||
private incoming: Promise<void> = Promise.resolve();
|
||||
private activeResponses = new Set<string>();
|
||||
private activeResponse = "";
|
||||
private createInFlight = false;
|
||||
private pendingCreates = 0;
|
||||
private deferredTexts: string[] = [];
|
||||
private suppressed = new Set<string>();
|
||||
private suppressNext = false;
|
||||
private audible = new Set<string>();
|
||||
private segments = new Map<string, string>();
|
||||
private completed = new Map<string, string>();
|
||||
private userSegments = new Map<string, string>();
|
||||
private toolCalls = new Set<string>();
|
||||
private resolveReady: (() => void) | null = null;
|
||||
private rejectReady: ((reason: Error) => void) | null = null;
|
||||
private readyTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
constructor(options: ClientOptions) {
|
||||
this.options = options;
|
||||
this.audio = new AudioEngine(options.context, options.gate);
|
||||
}
|
||||
get status() {
|
||||
return this.currentStatus;
|
||||
}
|
||||
on<K extends keyof Events>(name: K, listener: (detail: Events[K]) => void) {
|
||||
const handler = (event: Event) => {
|
||||
listener((event as CustomEvent<Events[K]>).detail);
|
||||
};
|
||||
this.events.addEventListener(name, handler);
|
||||
return () => {
|
||||
this.events.removeEventListener(name, handler);
|
||||
};
|
||||
}
|
||||
private emit<K extends keyof Events>(name: K, detail: Events[K]) {
|
||||
if (!this.closed || name === "status") {
|
||||
this.events.dispatchEvent(new CustomEvent(name, { detail }));
|
||||
}
|
||||
}
|
||||
private setStatus(status: ConnectionStatus) {
|
||||
if (this.currentStatus !== status) {
|
||||
this.currentStatus = status;
|
||||
this.emit("status", status);
|
||||
}
|
||||
}
|
||||
private ensureOpen() {
|
||||
if (this.closed) {
|
||||
throw new SessionError("aborted", "La conexión se canceló.");
|
||||
}
|
||||
}
|
||||
|
||||
async connect() {
|
||||
this.setStatus("connecting");
|
||||
try {
|
||||
// Ask for permission while the click-created AudioContext is still resumed.
|
||||
const permission = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
|
||||
permission.getTracks().forEach((track) => {
|
||||
track.stop();
|
||||
});
|
||||
this.ensureOpen();
|
||||
let url = this.options.directUrl;
|
||||
if (!url) {
|
||||
const grant = await this.acquireSession();
|
||||
this.ensureOpen();
|
||||
url = grant.connect_url;
|
||||
}
|
||||
this.ensureOpen();
|
||||
const ready = new Promise<void>((resolve, reject) => {
|
||||
this.resolveReady = resolve;
|
||||
this.rejectReady = reject;
|
||||
this.readyTimer = setTimeout(() => {
|
||||
reject(new Error("El servidor de voz no respondió a tiempo."));
|
||||
}, 20000);
|
||||
});
|
||||
const audioReady = this.audio.start(
|
||||
(buffer) => {
|
||||
if (this.configured) {
|
||||
this.send({ type: "input_audio_buffer.append", audio: encodePcm(buffer) });
|
||||
}
|
||||
},
|
||||
(rms) => {
|
||||
this.emit("level", rms);
|
||||
},
|
||||
() => this.currentStatus === "ai-speaking",
|
||||
);
|
||||
this.socket = new WebSocket(url);
|
||||
this.socket.binaryType = "arraybuffer";
|
||||
this.socket.addEventListener("message", (event: MessageEvent<unknown>) => {
|
||||
this.incoming = this.incoming
|
||||
.then(async () => {
|
||||
await audioReady;
|
||||
if (!this.closed) {
|
||||
await this.receive(event.data);
|
||||
}
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (!this.closed) {
|
||||
this.fail(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
this.socket.addEventListener("error", () => {
|
||||
this.fail(new Error("No se pudo conectar con el servidor de voz."));
|
||||
});
|
||||
this.socket.addEventListener("close", (event) => {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
if (event.code === 1000 && this.configured) {
|
||||
void this.close();
|
||||
} else {
|
||||
this.fail(new Error(`Se cerró la conexión de voz (${event.code}). ${event.reason}`.trim()));
|
||||
}
|
||||
});
|
||||
await Promise.all([audioReady, ready]);
|
||||
this.ensureOpen();
|
||||
} catch (error) {
|
||||
await this.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async acquireSession() {
|
||||
let result: z.infer<typeof sessionSchema>;
|
||||
try {
|
||||
result = await post(this.options.sessionUrl ?? "/api/session", sessionSchema, {}, this.abort.signal);
|
||||
} catch (error) {
|
||||
throw this.sessionError(error);
|
||||
}
|
||||
let waited = false;
|
||||
if ("state" in result) {
|
||||
waited = true;
|
||||
const ticket = result;
|
||||
this.queueId = ticket.queue_id;
|
||||
this.setStatus("queued");
|
||||
this.emit("queue", { position: ticket.position, queueId: this.queueId });
|
||||
while ("state" in result) {
|
||||
await this.delay(Math.max(1, ticket.poll_interval_s) * 1000);
|
||||
this.ensureOpen();
|
||||
try {
|
||||
result = await request(`/api/queue/${encodeURIComponent(this.queueId)}`, sessionSchema, {
|
||||
signal: this.abort.signal,
|
||||
});
|
||||
} catch (error) {
|
||||
this.ensureOpen();
|
||||
if (error instanceof ApiError && (error.status === 402 || error.status === 404)) {
|
||||
throw this.sessionError(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ("state" in result) {
|
||||
this.emit("queue", { position: result.position, queueId: this.queueId });
|
||||
}
|
||||
}
|
||||
}
|
||||
const grant = grantSchema.parse(result);
|
||||
this.grant = grant;
|
||||
this.queueId = "";
|
||||
this.ensureOpen();
|
||||
if (waited) {
|
||||
const seconds = Math.max(3, grant.pending_timeout_s - 3);
|
||||
this.setStatus("your-turn");
|
||||
const joined = new Promise<void>((resolve, reject) => {
|
||||
this.joinResolve = resolve;
|
||||
this.joinReject = reject;
|
||||
this.joinTimer = setTimeout(() => {
|
||||
reject(new SessionError("join-expired", "Tu turno ha caducado. Vuelve a intentarlo."));
|
||||
}, seconds * 1000);
|
||||
});
|
||||
this.emit("readyToJoin", { grant, expiresSec: seconds });
|
||||
await joined;
|
||||
}
|
||||
this.emit("grant", grant);
|
||||
if (grant.limited) {
|
||||
this.heartbeat = setInterval(
|
||||
() => {
|
||||
void post(
|
||||
"/api/session/heartbeat",
|
||||
z.object({ expired: z.boolean() }),
|
||||
{ sessionId: grant.session_id },
|
||||
this.abort.signal,
|
||||
)
|
||||
.then((result) => {
|
||||
if (result.expired) {
|
||||
this.fail(new SessionError("limit", "Se ha agotado tu tiempo de conversación."));
|
||||
}
|
||||
})
|
||||
.catch(() => undefined);
|
||||
},
|
||||
Math.max(1, grant.heartbeatSec) * 1000,
|
||||
);
|
||||
}
|
||||
this.setStatus("connecting");
|
||||
return grant;
|
||||
}
|
||||
private sessionError(error: unknown): unknown {
|
||||
if (error instanceof ApiError) {
|
||||
if (error.status === 402) {
|
||||
return new SessionError("limit", "Se ha agotado tu tiempo de conversación.");
|
||||
}
|
||||
if (error.status === 404) {
|
||||
return new SessionError("queue-expired", "La espera ha caducado. Vuelve a intentarlo.");
|
||||
}
|
||||
const data = z.object({ state: z.string() }).safeParse(error.body);
|
||||
if (error.status === 503 && data.success && data.data.state === "at_capacity") {
|
||||
return new SessionError("queue-full", "Todos los lugares y la cola están ocupados. Vuelve en un minuto.");
|
||||
}
|
||||
}
|
||||
return error;
|
||||
}
|
||||
private delay(ms: number) {
|
||||
return new Promise<void>((resolve) => {
|
||||
const finish = () => {
|
||||
clearTimeout(timer);
|
||||
this.abort.signal.removeEventListener("abort", finish);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(finish, ms);
|
||||
this.abort.signal.addEventListener("abort", finish, { once: true });
|
||||
if (this.abort.signal.aborted) {
|
||||
finish();
|
||||
}
|
||||
});
|
||||
}
|
||||
join() {
|
||||
clearTimeout(this.joinTimer);
|
||||
void this.options.context.resume().catch(() => undefined);
|
||||
this.joinResolve?.();
|
||||
this.joinResolve = null;
|
||||
this.joinReject = null;
|
||||
}
|
||||
private fail(error: unknown) {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
const failure = error instanceof Error ? error : new Error("La sesión de voz falló.");
|
||||
this.rejectReady?.(failure);
|
||||
this.emit("error", failure);
|
||||
this.setStatus("error");
|
||||
void this.close();
|
||||
}
|
||||
private async receive(raw: unknown) {
|
||||
const text =
|
||||
typeof raw === "string"
|
||||
? raw
|
||||
: raw instanceof ArrayBuffer
|
||||
? new TextDecoder().decode(raw)
|
||||
: raw instanceof Blob
|
||||
? await raw.text()
|
||||
: "";
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const result = realtimeEventSchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
return;
|
||||
}
|
||||
this.handle(result.data);
|
||||
}
|
||||
private assistantText(id: string) {
|
||||
return [this.completed.get(id), this.segments.get(id)].filter(Boolean).join(" ");
|
||||
}
|
||||
private handle(event: RealtimeEvent) {
|
||||
const id = event.response_id ?? event.response?.id ?? this.activeResponse;
|
||||
switch (event.type) {
|
||||
case "session.created":
|
||||
this.send({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
instructions: this.options.instructions,
|
||||
audio: { output: { voice: this.options.voice } },
|
||||
tools: this.options.tools,
|
||||
tool_choice: this.options.tools.length ? "auto" : "none",
|
||||
},
|
||||
});
|
||||
break;
|
||||
case "session.updated":
|
||||
if (!this.configured) {
|
||||
this.configured = true;
|
||||
clearTimeout(this.readyTimer);
|
||||
this.resolveReady?.();
|
||||
this.resolveReady = null;
|
||||
this.rejectReady = null;
|
||||
this.setStatus("listening");
|
||||
if (this.options.greeting) {
|
||||
this.sendUserText(this.options.greeting);
|
||||
this.requestResponse();
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "input_audio_buffer.speech_started":
|
||||
this.audio.clear();
|
||||
this.setStatus("user-speaking");
|
||||
break;
|
||||
case "input_audio_buffer.speech_stopped":
|
||||
this.setStatus("processing");
|
||||
break;
|
||||
case "response.created":
|
||||
this.activeResponse = id;
|
||||
this.activeResponses.add(id);
|
||||
this.createInFlight = false;
|
||||
if (this.suppressNext) {
|
||||
this.suppressed.add(id);
|
||||
this.suppressNext = false;
|
||||
}
|
||||
this.setStatus("processing");
|
||||
break;
|
||||
case "response.audio.delta":
|
||||
case "response.output_audio.delta":
|
||||
if (event.delta && !this.suppressed.has(id)) {
|
||||
this.audio.play(event.delta);
|
||||
this.audible.add(id);
|
||||
this.setStatus("ai-speaking");
|
||||
}
|
||||
break;
|
||||
case "conversation.item.input_audio_transcription.delta": {
|
||||
const itemId = event.item_id ?? "user-current";
|
||||
const text = (this.userSegments.get(itemId) ?? "") + (event.delta ?? "");
|
||||
this.userSegments.set(itemId, text);
|
||||
this.emit("transcript", { role: "user", text, partial: true, id: itemId });
|
||||
break;
|
||||
}
|
||||
case "conversation.item.input_audio_transcription.completed": {
|
||||
const itemId = event.item_id ?? "user-current";
|
||||
this.emit("transcript", {
|
||||
role: "user",
|
||||
text: event.transcript ?? this.userSegments.get(itemId) ?? "",
|
||||
partial: false,
|
||||
id: itemId,
|
||||
});
|
||||
this.userSegments.delete(itemId);
|
||||
break;
|
||||
}
|
||||
case "response.audio_transcript.delta":
|
||||
case "response.output_audio_transcript.delta":
|
||||
case "response.text.delta":
|
||||
case "response.output_text.delta":
|
||||
if (!this.suppressed.has(id)) {
|
||||
this.segments.set(id, (this.segments.get(id) ?? "") + (event.delta ?? ""));
|
||||
this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: true, id });
|
||||
}
|
||||
break;
|
||||
case "response.audio_transcript.done":
|
||||
case "response.output_audio_transcript.done":
|
||||
case "response.text.done":
|
||||
case "response.output_text.done":
|
||||
if (!this.suppressed.has(id)) {
|
||||
const segment = event.transcript ?? event.text ?? this.segments.get(id) ?? "";
|
||||
this.completed.set(id, [this.completed.get(id), segment].filter(Boolean).join(" "));
|
||||
this.segments.delete(id);
|
||||
this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: false, id });
|
||||
}
|
||||
break;
|
||||
case "response.done": {
|
||||
const transcript =
|
||||
event.response?.output
|
||||
?.flatMap((item) => item.content ?? [])
|
||||
.map((part) => part.transcript ?? part.text ?? "")
|
||||
.filter(Boolean)
|
||||
.join(" ") || this.assistantText(id);
|
||||
if (!this.suppressed.has(id)) {
|
||||
this.emit("finished", {
|
||||
responseId: id,
|
||||
status: event.response?.status ?? "completed",
|
||||
transcript,
|
||||
audible: this.audible.has(id),
|
||||
});
|
||||
}
|
||||
this.activeResponses.delete(id);
|
||||
this.audible.delete(id);
|
||||
this.suppressed.delete(id);
|
||||
this.segments.delete(id);
|
||||
this.completed.delete(id);
|
||||
if (this.activeResponse === id) {
|
||||
this.activeResponse = "";
|
||||
}
|
||||
this.setStatus("listening");
|
||||
this.flush();
|
||||
break;
|
||||
}
|
||||
case "response.function_call_arguments.done":
|
||||
if (event.name && event.call_id && !this.toolCalls.has(event.call_id)) {
|
||||
this.toolCalls.add(event.call_id);
|
||||
this.emit("tool", { name: event.name, arguments: event.arguments ?? "{}", callId: event.call_id });
|
||||
}
|
||||
break;
|
||||
case "error":
|
||||
if (
|
||||
event.error?.code === "conversation_already_has_active_response" ||
|
||||
event.error?.type === "conversation_already_has_active_response"
|
||||
) {
|
||||
if (this.createInFlight) {
|
||||
this.createInFlight = false;
|
||||
this.pendingCreates++;
|
||||
}
|
||||
} else {
|
||||
this.emit("serverError", new Error(event.error?.message ?? "Vocero no pudo completar la respuesta."));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
private busy() {
|
||||
return this.activeResponses.size > 0 || this.createInFlight;
|
||||
}
|
||||
private send(event: Record<string, unknown>) {
|
||||
if (!this.closed && this.socket?.readyState === WebSocket.OPEN) {
|
||||
this.socket.send(JSON.stringify(event));
|
||||
}
|
||||
}
|
||||
updateSession(instructions: string, voice: string, tools: FunctionTool[]) {
|
||||
this.options.instructions = instructions;
|
||||
this.options.voice = voice;
|
||||
this.options.tools = tools;
|
||||
if (this.configured) {
|
||||
this.send({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
instructions,
|
||||
audio: { output: { voice } },
|
||||
tools,
|
||||
tool_choice: tools.length ? "auto" : "none",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
sendUserText(text: string) {
|
||||
if (!text.trim()) {
|
||||
return;
|
||||
}
|
||||
if (this.busy()) {
|
||||
this.deferredTexts.push(text.trim());
|
||||
return;
|
||||
}
|
||||
this.send({
|
||||
type: "conversation.item.create",
|
||||
item: { type: "message", role: "user", content: [{ type: "input_text", text: text.trim() }] },
|
||||
});
|
||||
}
|
||||
requestResponse() {
|
||||
if (this.busy()) {
|
||||
this.pendingCreates++;
|
||||
return;
|
||||
}
|
||||
this.createInFlight = true;
|
||||
this.send({ type: "response.create" });
|
||||
}
|
||||
private flush() {
|
||||
if (this.busy()) {
|
||||
return;
|
||||
}
|
||||
const text = this.deferredTexts.shift();
|
||||
if (text) {
|
||||
this.sendUserText(text);
|
||||
}
|
||||
if (this.pendingCreates) {
|
||||
this.pendingCreates--;
|
||||
this.requestResponse();
|
||||
}
|
||||
}
|
||||
sendToolOutput(callId: string, output: string) {
|
||||
this.send({ type: "conversation.item.create", item: { type: "function_call_output", call_id: callId, output } });
|
||||
this.requestResponse();
|
||||
}
|
||||
renderExternalAnswer(text: string) {
|
||||
if (this.activeResponse) {
|
||||
this.suppressed.add(this.activeResponse);
|
||||
} else if (this.createInFlight) {
|
||||
this.suppressNext = true;
|
||||
}
|
||||
this.audio.clear();
|
||||
this.pendingCreates = 0;
|
||||
this.deferredTexts = [];
|
||||
this.send({
|
||||
type: "conversation.item.create",
|
||||
item: { type: "message", role: "assistant", content: [{ type: "text", text }] },
|
||||
});
|
||||
}
|
||||
setMuted(muted: boolean) {
|
||||
this.audio.setMuted(muted);
|
||||
}
|
||||
setGate(gate: NoiseGate) {
|
||||
this.audio.setGate(gate);
|
||||
}
|
||||
close(): Promise<void> {
|
||||
if (this.closing) {
|
||||
return this.closing;
|
||||
}
|
||||
this.closed = true;
|
||||
this.abort.abort();
|
||||
clearInterval(this.heartbeat);
|
||||
clearTimeout(this.joinTimer);
|
||||
clearTimeout(this.readyTimer);
|
||||
const cancelled = new SessionError("aborted", "La conexión se canceló.");
|
||||
this.joinReject?.(cancelled);
|
||||
this.rejectReady?.(cancelled);
|
||||
this.joinResolve = null;
|
||||
this.joinReject = null;
|
||||
this.resolveReady = null;
|
||||
this.rejectReady = null;
|
||||
if (this.queueId) {
|
||||
endReservation("/api/queue/end", { queueId: this.queueId });
|
||||
this.queueId = "";
|
||||
}
|
||||
if (this.grant) {
|
||||
endReservation("/api/session/end", { sessionId: this.grant.session_id });
|
||||
this.grant = null;
|
||||
}
|
||||
if (this.socket && this.socket.readyState <= WebSocket.OPEN) {
|
||||
this.socket.close(1000, "client closed");
|
||||
}
|
||||
this.socket = null;
|
||||
this.closing = this.audio
|
||||
.close()
|
||||
.catch(() => undefined)
|
||||
.then(() => {
|
||||
this.setStatus("closed");
|
||||
});
|
||||
return this.closing;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user