import { z } from "zod"; import { encodePcm } from "../audio/codec"; import { AudioEngine, MIC_CONSTRAINTS } from "../audio/engine"; import { ApiError, endReservation, post, request } from "../lib/api"; import { grantSchema, realtimeEventSchema, sessionSchema, type ConnectionStatus, type FinishedResponse, type FunctionTool, type NoiseGate, type RealtimeEvent, type SessionGrant, type ToolCall, type Transcript, } from "../lib/contracts"; interface Events { status: ConnectionStatus; transcript: Transcript; finished: FinishedResponse; tool: ToolCall; level: number; queue: { position: number; queueId: string }; grant: SessionGrant; readyToJoin: { grant: SessionGrant; expiresSec: number }; error: Error; serverError: Error; } export interface ClientOptions { directUrl?: string; sessionUrl?: string; voice: string; instructions: string; tools: FunctionTool[]; gate: NoiseGate; context: AudioContext; greeting?: string; } export class SessionError extends Error { readonly code: "limit" | "queue-full" | "queue-expired" | "join-expired" | "aborted"; constructor(code: SessionError["code"], message: string) { super(message); this.code = code; this.name = "SessionError"; } } /** Owns one complete session, including pending acquisition, audio, queue tickets and teardown. */ export class RealtimeClient { private readonly options: ClientOptions; private readonly events = new EventTarget(); private readonly abort = new AbortController(); private readonly audio: AudioEngine; private socket: WebSocket | null = null; private currentStatus: ConnectionStatus = "idle"; private configured = false; private closed = false; private closing: Promise | null = null; private grant: SessionGrant | null = null; private queueId = ""; private joinResolve: (() => void) | null = null; private joinReject: ((reason: Error) => void) | null = null; private joinTimer: ReturnType | undefined; private heartbeat: ReturnType | undefined; private incoming: Promise = Promise.resolve(); private activeResponses = new Set(); private activeResponse = ""; private createInFlight = false; private pendingCreates = 0; private deferredTexts: string[] = []; private suppressed = new Set(); private suppressNext = false; private audible = new Set(); private segments = new Map(); private completed = new Map(); private userSegments = new Map(); private toolCalls = new Set(); private resolveReady: (() => void) | null = null; private rejectReady: ((reason: Error) => void) | null = null; private readyTimer: ReturnType | undefined; constructor(options: ClientOptions) { this.options = options; this.audio = new AudioEngine(options.context, options.gate); } get status() { return this.currentStatus; } on(name: K, listener: (detail: Events[K]) => void) { const handler = (event: Event) => { listener((event as CustomEvent).detail); }; this.events.addEventListener(name, handler); return () => { this.events.removeEventListener(name, handler); }; } private emit(name: K, detail: Events[K]) { if (!this.closed || name === "status") { this.events.dispatchEvent(new CustomEvent(name, { detail })); } } private setStatus(status: ConnectionStatus) { if (this.currentStatus !== status) { this.currentStatus = status; this.emit("status", status); } } private ensureOpen() { if (this.closed) { throw new SessionError("aborted", "La conexión se canceló."); } } async connect() { this.setStatus("connecting"); try { // Ask for permission while the click-created AudioContext is still resumed. const permission = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); permission.getTracks().forEach((track) => { track.stop(); }); this.ensureOpen(); let url = this.options.directUrl; if (!url) { const grant = await this.acquireSession(); this.ensureOpen(); url = grant.connect_url; } this.ensureOpen(); this.socket = new WebSocket(url); this.socket.binaryType = "arraybuffer"; const ready = new Promise((resolve, reject) => { this.resolveReady = resolve; this.rejectReady = reject; this.readyTimer = setTimeout(() => { reject(new Error("El servidor de voz no respondió a tiempo.")); }, 20000); }); const audioReady = this.audio.start( (buffer) => { if (this.configured) { this.send({ type: "input_audio_buffer.append", audio: encodePcm(buffer) }); } }, (rms) => { this.emit("level", rms); }, () => this.currentStatus === "ai-speaking", ); this.socket.addEventListener("message", (event: MessageEvent) => { this.incoming = this.incoming .then(async () => { await audioReady; if (!this.closed) { await this.receive(event.data); } }) .catch((error: unknown) => { if (!this.closed) { this.fail(error); } }); }); this.socket.addEventListener("error", () => { this.fail(new Error("No se pudo conectar con el servidor de voz.")); }); this.socket.addEventListener("close", (event) => { if (this.closed) { return; } if (event.code === 1000 && this.configured) { void this.close(); } else { this.fail(new Error(`Se cerró la conexión de voz (${event.code}). ${event.reason}`.trim())); } }); await Promise.all([audioReady, ready]); this.ensureOpen(); } catch (error) { const failure = this.closed ? new SessionError("aborted", "La conexión se canceló.") : error; await this.close(); throw failure; } } private async acquireSession() { let result: z.infer; try { result = await post(this.options.sessionUrl ?? "/api/session", sessionSchema, {}, this.abort.signal); } catch (error) { throw this.sessionError(error); } let waited = false; if ("state" in result) { waited = true; const ticket = result; this.queueId = ticket.queue_id; this.setStatus("queued"); this.emit("queue", { position: ticket.position, queueId: this.queueId }); while ("state" in result) { await this.delay(Math.max(1, ticket.poll_interval_s) * 1000); this.ensureOpen(); try { result = await request(`/api/queue/${encodeURIComponent(this.queueId)}`, sessionSchema, { signal: this.abort.signal, }); } catch (error) { this.ensureOpen(); if (error instanceof ApiError && (error.status === 402 || error.status === 404)) { throw this.sessionError(error); } continue; } if ("state" in result) { this.emit("queue", { position: result.position, queueId: this.queueId }); } } } const grant = grantSchema.parse(result); this.grant = grant; this.queueId = ""; this.ensureOpen(); if (waited) { const seconds = Math.max(3, grant.pending_timeout_s - 3); this.setStatus("your-turn"); const joined = new Promise((resolve, reject) => { this.joinResolve = resolve; this.joinReject = reject; this.joinTimer = setTimeout(() => { reject(new SessionError("join-expired", "Tu turno ha caducado. Vuelve a intentarlo.")); }, seconds * 1000); }); this.emit("readyToJoin", { grant, expiresSec: seconds }); await joined; } this.emit("grant", grant); if (grant.limited) { this.heartbeat = setInterval( () => { void post( "/api/session/heartbeat", z.object({ expired: z.boolean() }), { sessionId: grant.session_id }, this.abort.signal, ) .then((result) => { if (result.expired) { this.fail(new SessionError("limit", "Se ha agotado tu tiempo de conversación.")); } }) .catch(() => undefined); }, Math.max(1, grant.heartbeatSec) * 1000, ); } this.setStatus("connecting"); return grant; } private sessionError(error: unknown): unknown { if (error instanceof ApiError) { if (error.status === 402) { return new SessionError("limit", "Se ha agotado tu tiempo de conversación."); } if (error.status === 404) { return new SessionError("queue-expired", "La espera ha caducado. Vuelve a intentarlo."); } const data = z.object({ state: z.string() }).safeParse(error.body); if (error.status === 503 && data.success && data.data.state === "at_capacity") { return new SessionError("queue-full", "Todos los lugares y la cola están ocupados. Vuelve en un minuto."); } } return error; } private delay(ms: number) { return new Promise((resolve) => { const finish = () => { clearTimeout(timer); this.abort.signal.removeEventListener("abort", finish); resolve(); }; const timer = setTimeout(finish, ms); this.abort.signal.addEventListener("abort", finish, { once: true }); if (this.abort.signal.aborted) { finish(); } }); } join() { clearTimeout(this.joinTimer); void this.options.context.resume().catch(() => undefined); this.joinResolve?.(); this.joinResolve = null; this.joinReject = null; } private fail(error: unknown) { if (this.closed) { return; } const failure = error instanceof Error ? error : new Error("La sesión de voz falló."); this.rejectReady?.(failure); this.emit("error", failure); this.setStatus("error"); void this.close(); } private async receive(raw: unknown) { const text = typeof raw === "string" ? raw : raw instanceof ArrayBuffer ? new TextDecoder().decode(raw) : raw instanceof Blob ? await raw.text() : ""; let parsed: unknown; try { parsed = JSON.parse(text); } catch { return; } const result = realtimeEventSchema.safeParse(parsed); if (!result.success) { return; } this.handle(result.data); } private assistantText(id: string) { return [this.completed.get(id), this.segments.get(id)].filter(Boolean).join(" "); } private handle(event: RealtimeEvent) { const id = event.response_id ?? event.response?.id ?? this.activeResponse; switch (event.type) { case "session.created": this.send({ type: "session.update", session: { type: "realtime", instructions: this.options.instructions, audio: { output: { voice: this.options.voice } }, tools: this.options.tools, tool_choice: this.options.tools.length ? "auto" : "none", }, }); break; case "session.updated": if (!this.configured) { this.configured = true; clearTimeout(this.readyTimer); this.resolveReady?.(); this.resolveReady = null; this.rejectReady = null; this.setStatus("listening"); if (this.options.greeting) { this.sendUserText(this.options.greeting); this.requestResponse(); } } break; case "input_audio_buffer.speech_started": this.audio.clear(); this.setStatus("user-speaking"); break; case "input_audio_buffer.speech_stopped": this.setStatus("processing"); break; case "response.created": this.activeResponse = id; this.activeResponses.add(id); this.createInFlight = false; if (this.suppressNext) { this.suppressed.add(id); this.suppressNext = false; } this.setStatus("processing"); break; case "response.audio.delta": case "response.output_audio.delta": if (event.delta && !this.suppressed.has(id)) { this.audio.play(event.delta); this.audible.add(id); this.setStatus("ai-speaking"); } break; case "conversation.item.input_audio_transcription.delta": { const itemId = event.item_id ?? "user-current"; // This server sends the complete speculative transcription in each delta. const text = event.delta ?? ""; this.userSegments.set(itemId, text); this.emit("transcript", { role: "user", text, partial: true, id: itemId }); break; } case "conversation.item.input_audio_transcription.completed": { const itemId = event.item_id ?? "user-current"; this.emit("transcript", { role: "user", text: event.transcript ?? this.userSegments.get(itemId) ?? "", partial: false, id: itemId, }); this.userSegments.delete(itemId); break; } case "response.audio_transcript.delta": case "response.output_audio_transcript.delta": case "response.text.delta": case "response.output_text.delta": if (!this.suppressed.has(id)) { this.segments.set(id, (this.segments.get(id) ?? "") + (event.delta ?? "")); this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: true, id }); } break; case "response.audio_transcript.done": case "response.output_audio_transcript.done": case "response.text.done": case "response.output_text.done": if (!this.suppressed.has(id)) { const segment = event.transcript ?? event.text ?? this.segments.get(id) ?? ""; this.completed.set(id, [this.completed.get(id), segment].filter(Boolean).join(" ")); this.segments.delete(id); this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: false, id }); } break; case "response.done": { const transcript = event.response?.output ?.flatMap((item) => item.content ?? []) .map((part) => part.transcript ?? part.text ?? "") .filter(Boolean) .join(" ") || this.assistantText(id); if (!this.suppressed.has(id)) { this.emit("finished", { responseId: id, status: event.response?.status ?? "completed", transcript, audible: this.audible.has(id), }); } this.activeResponses.delete(id); this.audible.delete(id); this.suppressed.delete(id); this.segments.delete(id); this.completed.delete(id); if (this.activeResponse === id) { this.activeResponse = ""; } if (this.currentStatus === "ai-speaking" || this.currentStatus === "processing") { this.setStatus("listening"); } this.flush(); break; } case "response.function_call_arguments.done": if (event.name && event.call_id && !this.toolCalls.has(event.call_id)) { this.toolCalls.add(event.call_id); this.emit("tool", { name: event.name, arguments: event.arguments ?? "{}", callId: event.call_id }); } break; case "error": if ( event.error?.code === "conversation_already_has_active_response" || event.error?.type === "conversation_already_has_active_response" ) { if (this.createInFlight) { this.createInFlight = false; this.pendingCreates++; } } else { this.emit("serverError", new Error(event.error?.message ?? "Vocero no pudo completar la respuesta.")); } break; } } private busy() { return this.activeResponses.size > 0 || this.createInFlight; } private send(event: Record) { if (!this.closed && this.socket?.readyState === WebSocket.OPEN) { this.socket.send(JSON.stringify(event)); } } updateSession(instructions: string, voice: string, tools: FunctionTool[]) { this.options.instructions = instructions; this.options.voice = voice; this.options.tools = tools; if (this.configured) { this.send({ type: "session.update", session: { type: "realtime", instructions, audio: { output: { voice } }, tools, tool_choice: tools.length ? "auto" : "none", }, }); } } sendUserText(text: string) { if (!text.trim()) { return; } if (this.busy()) { this.deferredTexts.push(text.trim()); return; } this.send({ type: "conversation.item.create", item: { type: "message", role: "user", content: [{ type: "input_text", text: text.trim() }] }, }); } requestResponse() { if (this.closed || !this.configured) { return; } if (this.busy()) { this.pendingCreates++; return; } this.createInFlight = true; this.send({ type: "response.create" }); } private flush() { if (this.busy()) { return; } const text = this.deferredTexts.shift(); if (text) { this.sendUserText(text); } if (this.pendingCreates) { this.pendingCreates--; this.requestResponse(); } } sendToolOutput(callId: string, output: string) { this.send({ type: "conversation.item.create", item: { type: "function_call_output", call_id: callId, output } }); this.requestResponse(); } renderExternalAnswer(text: string) { if (this.activeResponse) { this.suppressed.add(this.activeResponse); } else { this.suppressNext = true; } this.audio.clear(); this.pendingCreates = 0; this.deferredTexts = []; this.send({ type: "conversation.item.create", item: { type: "message", role: "assistant", content: [{ type: "text", text }] }, }); } setMuted(muted: boolean) { this.audio.setMuted(muted); } setGate(gate: NoiseGate) { this.audio.setGate(gate); } close(): Promise { if (this.closing) { return this.closing; } this.closed = true; this.abort.abort(); clearInterval(this.heartbeat); clearTimeout(this.joinTimer); clearTimeout(this.readyTimer); const cancelled = new SessionError("aborted", "La conexión se canceló."); this.joinReject?.(cancelled); this.rejectReady?.(cancelled); this.joinResolve = null; this.joinReject = null; this.resolveReady = null; this.rejectReady = null; if (this.queueId) { endReservation("/api/queue/end", { queueId: this.queueId }); this.queueId = ""; } if (this.grant) { endReservation("/api/session/end", { sessionId: this.grant.session_id }); this.grant = null; } if (this.socket && this.socket.readyState <= WebSocket.OPEN) { this.socket.close(1000, "client closed"); } this.socket = null; this.closing = this.audio .close() .catch(() => undefined) .then(() => { this.setStatus("closed"); }); return this.closing; } }