600 lines
19 KiB
TypeScript
600 lines
19 KiB
TypeScript
import { z } from "zod";
|
|
|
|
import { encodePcm } from "../audio/codec";
|
|
import { AudioEngine, MIC_CONSTRAINTS } from "../audio/engine";
|
|
import { ApiError, endReservation, post, request } from "../lib/api";
|
|
import {
|
|
grantSchema,
|
|
realtimeEventSchema,
|
|
sessionSchema,
|
|
type ConnectionStatus,
|
|
type FinishedResponse,
|
|
type FunctionTool,
|
|
type NoiseGate,
|
|
type RealtimeEvent,
|
|
type SessionGrant,
|
|
type ToolCall,
|
|
type Transcript,
|
|
} from "../lib/contracts";
|
|
|
|
interface Events {
|
|
status: ConnectionStatus;
|
|
transcript: Transcript;
|
|
finished: FinishedResponse;
|
|
tool: ToolCall;
|
|
level: number;
|
|
queue: { position: number; queueId: string };
|
|
grant: SessionGrant;
|
|
readyToJoin: { grant: SessionGrant; expiresSec: number };
|
|
error: Error;
|
|
serverError: Error;
|
|
}
|
|
export interface ClientOptions {
|
|
directUrl?: string;
|
|
sessionUrl?: string;
|
|
voice: string;
|
|
instructions: string;
|
|
tools: FunctionTool[];
|
|
gate: NoiseGate;
|
|
context: AudioContext;
|
|
greeting?: string;
|
|
}
|
|
|
|
export class SessionError extends Error {
|
|
readonly code: "limit" | "queue-full" | "queue-expired" | "join-expired" | "aborted";
|
|
constructor(code: SessionError["code"], message: string) {
|
|
super(message);
|
|
this.code = code;
|
|
this.name = "SessionError";
|
|
}
|
|
}
|
|
|
|
/** Owns one complete session, including pending acquisition, audio, queue tickets and teardown. */
|
|
export class RealtimeClient {
|
|
private readonly options: ClientOptions;
|
|
private readonly events = new EventTarget();
|
|
private readonly abort = new AbortController();
|
|
private readonly audio: AudioEngine;
|
|
private socket: WebSocket | null = null;
|
|
private currentStatus: ConnectionStatus = "idle";
|
|
private configured = false;
|
|
private closed = false;
|
|
private closing: Promise<void> | null = null;
|
|
private grant: SessionGrant | null = null;
|
|
private queueId = "";
|
|
private joinResolve: (() => void) | null = null;
|
|
private joinReject: ((reason: Error) => void) | null = null;
|
|
private joinTimer: ReturnType<typeof setTimeout> | undefined;
|
|
private heartbeat: ReturnType<typeof setInterval> | undefined;
|
|
private incoming: Promise<void> = Promise.resolve();
|
|
private activeResponses = new Set<string>();
|
|
private activeResponse = "";
|
|
private createInFlight = false;
|
|
private pendingCreates = 0;
|
|
private deferredTexts: string[] = [];
|
|
private suppressed = new Set<string>();
|
|
private suppressNext = false;
|
|
private audible = new Set<string>();
|
|
private segments = new Map<string, string>();
|
|
private completed = new Map<string, string>();
|
|
private userSegments = new Map<string, string>();
|
|
private toolCalls = new Set<string>();
|
|
private resolveReady: (() => void) | null = null;
|
|
private rejectReady: ((reason: Error) => void) | null = null;
|
|
private readyTimer: ReturnType<typeof setTimeout> | undefined;
|
|
|
|
constructor(options: ClientOptions) {
|
|
this.options = options;
|
|
this.audio = new AudioEngine(options.context, options.gate);
|
|
}
|
|
get status() {
|
|
return this.currentStatus;
|
|
}
|
|
on<K extends keyof Events>(name: K, listener: (detail: Events[K]) => void) {
|
|
const handler = (event: Event) => {
|
|
listener((event as CustomEvent<Events[K]>).detail);
|
|
};
|
|
this.events.addEventListener(name, handler);
|
|
return () => {
|
|
this.events.removeEventListener(name, handler);
|
|
};
|
|
}
|
|
private emit<K extends keyof Events>(name: K, detail: Events[K]) {
|
|
if (!this.closed || name === "status") {
|
|
this.events.dispatchEvent(new CustomEvent(name, { detail }));
|
|
}
|
|
}
|
|
private setStatus(status: ConnectionStatus) {
|
|
if (this.currentStatus !== status) {
|
|
this.currentStatus = status;
|
|
this.emit("status", status);
|
|
}
|
|
}
|
|
private ensureOpen() {
|
|
if (this.closed) {
|
|
throw new SessionError("aborted", "La conexión se canceló.");
|
|
}
|
|
}
|
|
|
|
async connect() {
|
|
this.setStatus("connecting");
|
|
try {
|
|
// Ask for permission while the click-created AudioContext is still resumed.
|
|
const permission = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
|
|
permission.getTracks().forEach((track) => {
|
|
track.stop();
|
|
});
|
|
this.ensureOpen();
|
|
let url = this.options.directUrl;
|
|
if (!url) {
|
|
const grant = await this.acquireSession();
|
|
this.ensureOpen();
|
|
url = grant.connect_url;
|
|
}
|
|
this.ensureOpen();
|
|
this.socket = new WebSocket(url);
|
|
this.socket.binaryType = "arraybuffer";
|
|
const ready = new Promise<void>((resolve, reject) => {
|
|
this.resolveReady = resolve;
|
|
this.rejectReady = reject;
|
|
this.readyTimer = setTimeout(() => {
|
|
reject(new Error("El servidor de voz no respondió a tiempo."));
|
|
}, 20000);
|
|
});
|
|
const audioReady = this.audio.start(
|
|
(buffer) => {
|
|
if (this.configured) {
|
|
this.send({ type: "input_audio_buffer.append", audio: encodePcm(buffer) });
|
|
}
|
|
},
|
|
(rms) => {
|
|
this.emit("level", rms);
|
|
},
|
|
() => this.currentStatus === "ai-speaking",
|
|
);
|
|
this.socket.addEventListener("message", (event: MessageEvent<unknown>) => {
|
|
this.incoming = this.incoming
|
|
.then(async () => {
|
|
await audioReady;
|
|
if (!this.closed) {
|
|
await this.receive(event.data);
|
|
}
|
|
})
|
|
.catch((error: unknown) => {
|
|
if (!this.closed) {
|
|
this.fail(error);
|
|
}
|
|
});
|
|
});
|
|
this.socket.addEventListener("error", () => {
|
|
this.fail(new Error("No se pudo conectar con el servidor de voz."));
|
|
});
|
|
this.socket.addEventListener("close", (event) => {
|
|
if (this.closed) {
|
|
return;
|
|
}
|
|
if (event.code === 1000 && this.configured) {
|
|
void this.close();
|
|
} else {
|
|
this.fail(new Error(`Se cerró la conexión de voz (${event.code}). ${event.reason}`.trim()));
|
|
}
|
|
});
|
|
await Promise.all([audioReady, ready]);
|
|
this.ensureOpen();
|
|
} catch (error) {
|
|
const failure = this.closed ? new SessionError("aborted", "La conexión se canceló.") : error;
|
|
await this.close();
|
|
throw failure;
|
|
}
|
|
}
|
|
|
|
private async acquireSession() {
|
|
let result: z.infer<typeof sessionSchema>;
|
|
try {
|
|
result = await post(this.options.sessionUrl ?? "/api/session", sessionSchema, {}, this.abort.signal);
|
|
} catch (error) {
|
|
throw this.sessionError(error);
|
|
}
|
|
let waited = false;
|
|
if ("state" in result) {
|
|
waited = true;
|
|
const ticket = result;
|
|
this.queueId = ticket.queue_id;
|
|
this.setStatus("queued");
|
|
this.emit("queue", { position: ticket.position, queueId: this.queueId });
|
|
while ("state" in result) {
|
|
await this.delay(Math.max(1, ticket.poll_interval_s) * 1000);
|
|
this.ensureOpen();
|
|
try {
|
|
result = await request(`/api/queue/${encodeURIComponent(this.queueId)}`, sessionSchema, {
|
|
signal: this.abort.signal,
|
|
});
|
|
} catch (error) {
|
|
this.ensureOpen();
|
|
if (error instanceof ApiError && (error.status === 402 || error.status === 404)) {
|
|
throw this.sessionError(error);
|
|
}
|
|
continue;
|
|
}
|
|
if ("state" in result) {
|
|
this.emit("queue", { position: result.position, queueId: this.queueId });
|
|
}
|
|
}
|
|
}
|
|
const grant = grantSchema.parse(result);
|
|
this.grant = grant;
|
|
this.queueId = "";
|
|
this.ensureOpen();
|
|
if (waited) {
|
|
const seconds = Math.max(3, grant.pending_timeout_s - 3);
|
|
this.setStatus("your-turn");
|
|
const joined = new Promise<void>((resolve, reject) => {
|
|
this.joinResolve = resolve;
|
|
this.joinReject = reject;
|
|
this.joinTimer = setTimeout(() => {
|
|
reject(new SessionError("join-expired", "Tu turno ha caducado. Vuelve a intentarlo."));
|
|
}, seconds * 1000);
|
|
});
|
|
this.emit("readyToJoin", { grant, expiresSec: seconds });
|
|
await joined;
|
|
}
|
|
this.emit("grant", grant);
|
|
if (grant.limited) {
|
|
this.heartbeat = setInterval(
|
|
() => {
|
|
void post(
|
|
"/api/session/heartbeat",
|
|
z.object({ expired: z.boolean() }),
|
|
{ sessionId: grant.session_id },
|
|
this.abort.signal,
|
|
)
|
|
.then((result) => {
|
|
if (result.expired) {
|
|
this.fail(new SessionError("limit", "Se ha agotado tu tiempo de conversación."));
|
|
}
|
|
})
|
|
.catch(() => undefined);
|
|
},
|
|
Math.max(1, grant.heartbeatSec) * 1000,
|
|
);
|
|
}
|
|
this.setStatus("connecting");
|
|
return grant;
|
|
}
|
|
private sessionError(error: unknown): unknown {
|
|
if (error instanceof ApiError) {
|
|
if (error.status === 402) {
|
|
return new SessionError("limit", "Se ha agotado tu tiempo de conversación.");
|
|
}
|
|
if (error.status === 404) {
|
|
return new SessionError("queue-expired", "La espera ha caducado. Vuelve a intentarlo.");
|
|
}
|
|
const data = z.object({ state: z.string() }).safeParse(error.body);
|
|
if (error.status === 503 && data.success && data.data.state === "at_capacity") {
|
|
return new SessionError("queue-full", "Todos los lugares y la cola están ocupados. Vuelve en un minuto.");
|
|
}
|
|
}
|
|
return error;
|
|
}
|
|
private delay(ms: number) {
|
|
return new Promise<void>((resolve) => {
|
|
const finish = () => {
|
|
clearTimeout(timer);
|
|
this.abort.signal.removeEventListener("abort", finish);
|
|
resolve();
|
|
};
|
|
const timer = setTimeout(finish, ms);
|
|
this.abort.signal.addEventListener("abort", finish, { once: true });
|
|
if (this.abort.signal.aborted) {
|
|
finish();
|
|
}
|
|
});
|
|
}
|
|
join() {
|
|
clearTimeout(this.joinTimer);
|
|
void this.options.context.resume().catch(() => undefined);
|
|
this.joinResolve?.();
|
|
this.joinResolve = null;
|
|
this.joinReject = null;
|
|
}
|
|
private fail(error: unknown) {
|
|
if (this.closed) {
|
|
return;
|
|
}
|
|
const failure = error instanceof Error ? error : new Error("La sesión de voz falló.");
|
|
this.rejectReady?.(failure);
|
|
this.emit("error", failure);
|
|
this.setStatus("error");
|
|
void this.close();
|
|
}
|
|
private async receive(raw: unknown) {
|
|
const text =
|
|
typeof raw === "string"
|
|
? raw
|
|
: raw instanceof ArrayBuffer
|
|
? new TextDecoder().decode(raw)
|
|
: raw instanceof Blob
|
|
? await raw.text()
|
|
: "";
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(text);
|
|
} catch {
|
|
return;
|
|
}
|
|
const result = realtimeEventSchema.safeParse(parsed);
|
|
if (!result.success) {
|
|
return;
|
|
}
|
|
this.handle(result.data);
|
|
}
|
|
private assistantText(id: string) {
|
|
return [this.completed.get(id), this.segments.get(id)].filter(Boolean).join(" ");
|
|
}
|
|
private handle(event: RealtimeEvent) {
|
|
const id = event.response_id ?? event.response?.id ?? this.activeResponse;
|
|
switch (event.type) {
|
|
case "session.created":
|
|
this.send({
|
|
type: "session.update",
|
|
session: {
|
|
type: "realtime",
|
|
instructions: this.options.instructions,
|
|
audio: { output: { voice: this.options.voice } },
|
|
tools: this.options.tools,
|
|
tool_choice: this.options.tools.length ? "auto" : "none",
|
|
},
|
|
});
|
|
break;
|
|
case "session.updated":
|
|
if (!this.configured) {
|
|
this.configured = true;
|
|
clearTimeout(this.readyTimer);
|
|
this.resolveReady?.();
|
|
this.resolveReady = null;
|
|
this.rejectReady = null;
|
|
this.setStatus("listening");
|
|
if (this.options.greeting) {
|
|
this.sendUserText(this.options.greeting);
|
|
this.requestResponse();
|
|
}
|
|
}
|
|
break;
|
|
case "input_audio_buffer.speech_started":
|
|
this.audio.clear();
|
|
this.setStatus("user-speaking");
|
|
break;
|
|
case "input_audio_buffer.speech_stopped":
|
|
this.setStatus("processing");
|
|
break;
|
|
case "response.created":
|
|
this.activeResponse = id;
|
|
this.activeResponses.add(id);
|
|
this.createInFlight = false;
|
|
if (this.suppressNext) {
|
|
this.suppressed.add(id);
|
|
this.suppressNext = false;
|
|
}
|
|
this.setStatus("processing");
|
|
break;
|
|
case "response.audio.delta":
|
|
case "response.output_audio.delta":
|
|
if (event.delta && !this.suppressed.has(id)) {
|
|
this.audio.play(event.delta);
|
|
this.audible.add(id);
|
|
this.setStatus("ai-speaking");
|
|
}
|
|
break;
|
|
case "conversation.item.input_audio_transcription.delta": {
|
|
const itemId = event.item_id ?? "user-current";
|
|
// This server sends the complete speculative transcription in each delta.
|
|
const text = event.delta ?? "";
|
|
this.userSegments.set(itemId, text);
|
|
this.emit("transcript", { role: "user", text, partial: true, id: itemId });
|
|
break;
|
|
}
|
|
case "conversation.item.input_audio_transcription.completed": {
|
|
const itemId = event.item_id ?? "user-current";
|
|
this.emit("transcript", {
|
|
role: "user",
|
|
text: event.transcript ?? this.userSegments.get(itemId) ?? "",
|
|
partial: false,
|
|
id: itemId,
|
|
});
|
|
this.userSegments.delete(itemId);
|
|
break;
|
|
}
|
|
case "response.audio_transcript.delta":
|
|
case "response.output_audio_transcript.delta":
|
|
case "response.text.delta":
|
|
case "response.output_text.delta":
|
|
if (!this.suppressed.has(id)) {
|
|
this.segments.set(id, (this.segments.get(id) ?? "") + (event.delta ?? ""));
|
|
this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: true, id });
|
|
}
|
|
break;
|
|
case "response.audio_transcript.done":
|
|
case "response.output_audio_transcript.done":
|
|
case "response.text.done":
|
|
case "response.output_text.done":
|
|
if (!this.suppressed.has(id)) {
|
|
const segment = event.transcript ?? event.text ?? this.segments.get(id) ?? "";
|
|
this.completed.set(id, [this.completed.get(id), segment].filter(Boolean).join(" "));
|
|
this.segments.delete(id);
|
|
this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: false, id });
|
|
}
|
|
break;
|
|
case "response.done": {
|
|
const transcript =
|
|
event.response?.output
|
|
?.flatMap((item) => item.content ?? [])
|
|
.map((part) => part.transcript ?? part.text ?? "")
|
|
.filter(Boolean)
|
|
.join(" ") || this.assistantText(id);
|
|
if (!this.suppressed.has(id)) {
|
|
this.emit("finished", {
|
|
responseId: id,
|
|
status: event.response?.status ?? "completed",
|
|
transcript,
|
|
audible: this.audible.has(id),
|
|
});
|
|
}
|
|
this.activeResponses.delete(id);
|
|
this.audible.delete(id);
|
|
this.suppressed.delete(id);
|
|
this.segments.delete(id);
|
|
this.completed.delete(id);
|
|
if (this.activeResponse === id) {
|
|
this.activeResponse = "";
|
|
}
|
|
if (this.currentStatus === "ai-speaking" || this.currentStatus === "processing") {
|
|
this.setStatus("listening");
|
|
}
|
|
this.flush();
|
|
break;
|
|
}
|
|
case "response.function_call_arguments.done":
|
|
if (event.name && event.call_id && !this.toolCalls.has(event.call_id)) {
|
|
this.toolCalls.add(event.call_id);
|
|
this.emit("tool", { name: event.name, arguments: event.arguments ?? "{}", callId: event.call_id });
|
|
}
|
|
break;
|
|
case "error":
|
|
if (
|
|
event.error?.code === "conversation_already_has_active_response" ||
|
|
event.error?.type === "conversation_already_has_active_response"
|
|
) {
|
|
if (this.createInFlight) {
|
|
this.createInFlight = false;
|
|
this.pendingCreates++;
|
|
}
|
|
} else {
|
|
this.emit("serverError", new Error(event.error?.message ?? "Vocero no pudo completar la respuesta."));
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
private busy() {
|
|
return this.activeResponses.size > 0 || this.createInFlight;
|
|
}
|
|
private send(event: Record<string, unknown>) {
|
|
if (!this.closed && this.socket?.readyState === WebSocket.OPEN) {
|
|
this.socket.send(JSON.stringify(event));
|
|
}
|
|
}
|
|
updateSession(instructions: string, voice: string, tools: FunctionTool[]) {
|
|
this.options.instructions = instructions;
|
|
this.options.voice = voice;
|
|
this.options.tools = tools;
|
|
if (this.configured) {
|
|
this.send({
|
|
type: "session.update",
|
|
session: {
|
|
type: "realtime",
|
|
instructions,
|
|
audio: { output: { voice } },
|
|
tools,
|
|
tool_choice: tools.length ? "auto" : "none",
|
|
},
|
|
});
|
|
}
|
|
}
|
|
sendUserText(text: string) {
|
|
if (!text.trim()) {
|
|
return;
|
|
}
|
|
if (this.busy()) {
|
|
this.deferredTexts.push(text.trim());
|
|
return;
|
|
}
|
|
this.send({
|
|
type: "conversation.item.create",
|
|
item: { type: "message", role: "user", content: [{ type: "input_text", text: text.trim() }] },
|
|
});
|
|
}
|
|
requestResponse() {
|
|
if (this.closed || !this.configured) {
|
|
return;
|
|
}
|
|
if (this.busy()) {
|
|
this.pendingCreates++;
|
|
return;
|
|
}
|
|
this.createInFlight = true;
|
|
this.send({ type: "response.create" });
|
|
}
|
|
private flush() {
|
|
if (this.busy()) {
|
|
return;
|
|
}
|
|
const text = this.deferredTexts.shift();
|
|
if (text) {
|
|
this.sendUserText(text);
|
|
}
|
|
if (this.pendingCreates) {
|
|
this.pendingCreates--;
|
|
this.requestResponse();
|
|
}
|
|
}
|
|
sendToolOutput(callId: string, output: string) {
|
|
this.send({ type: "conversation.item.create", item: { type: "function_call_output", call_id: callId, output } });
|
|
this.requestResponse();
|
|
}
|
|
renderExternalAnswer(text: string) {
|
|
if (this.activeResponse) {
|
|
this.suppressed.add(this.activeResponse);
|
|
} else {
|
|
this.suppressNext = true;
|
|
}
|
|
this.audio.clear();
|
|
this.pendingCreates = 0;
|
|
this.deferredTexts = [];
|
|
this.send({
|
|
type: "conversation.item.create",
|
|
item: { type: "message", role: "assistant", content: [{ type: "text", text }] },
|
|
});
|
|
}
|
|
setMuted(muted: boolean) {
|
|
this.audio.setMuted(muted);
|
|
}
|
|
setGate(gate: NoiseGate) {
|
|
this.audio.setGate(gate);
|
|
}
|
|
close(): Promise<void> {
|
|
if (this.closing) {
|
|
return this.closing;
|
|
}
|
|
this.closed = true;
|
|
this.abort.abort();
|
|
clearInterval(this.heartbeat);
|
|
clearTimeout(this.joinTimer);
|
|
clearTimeout(this.readyTimer);
|
|
const cancelled = new SessionError("aborted", "La conexión se canceló.");
|
|
this.joinReject?.(cancelled);
|
|
this.rejectReady?.(cancelled);
|
|
this.joinResolve = null;
|
|
this.joinReject = null;
|
|
this.resolveReady = null;
|
|
this.rejectReady = null;
|
|
if (this.queueId) {
|
|
endReservation("/api/queue/end", { queueId: this.queueId });
|
|
this.queueId = "";
|
|
}
|
|
if (this.grant) {
|
|
endReservation("/api/session/end", { sessionId: this.grant.session_id });
|
|
this.grant = null;
|
|
}
|
|
if (this.socket && this.socket.readyState <= WebSocket.OPEN) {
|
|
this.socket.close(1000, "client closed");
|
|
}
|
|
this.socket = null;
|
|
this.closing = this.audio
|
|
.close()
|
|
.catch(() => undefined)
|
|
.then(() => {
|
|
this.setStatus("closed");
|
|
});
|
|
return this.closing;
|
|
}
|
|
}
|