467 lines
18 KiB
TypeScript
467 lines
18 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
|
|
import type { AudioEngine } from "../audio/engine";
|
|
|
|
import { RealtimeClient, SessionError, type ClientOptions } from "./client";
|
|
|
|
const audio = vi.hoisted(() => ({
|
|
start: vi.fn<AudioEngine["start"]>(),
|
|
close: vi.fn<() => Promise<void>>(),
|
|
play: vi.fn(),
|
|
clear: vi.fn(),
|
|
finishPlayback: vi.fn(),
|
|
setMuted: vi.fn(),
|
|
setGate: vi.fn(),
|
|
}));
|
|
vi.mock("../audio/engine", () => ({
|
|
MIC_CONSTRAINTS: { audio: true },
|
|
AudioEngine: class {
|
|
start = audio.start;
|
|
close = audio.close;
|
|
play = audio.play;
|
|
clear = audio.clear;
|
|
finishPlayback = audio.finishPlayback;
|
|
setMuted = audio.setMuted;
|
|
setGate = audio.setGate;
|
|
},
|
|
}));
|
|
|
|
class Socket extends EventTarget {
|
|
static OPEN = 1;
|
|
static instances: Socket[] = [];
|
|
readyState = 1;
|
|
binaryType = "";
|
|
sent: Record<string, unknown>[] = [];
|
|
readonly url: string;
|
|
constructor(url: string) {
|
|
super();
|
|
this.url = url;
|
|
Socket.instances.push(this);
|
|
}
|
|
send(text: string) {
|
|
this.sent.push(JSON.parse(text) as Record<string, unknown>);
|
|
}
|
|
close = vi.fn(() => {
|
|
this.readyState = 3;
|
|
});
|
|
receive(value: unknown) {
|
|
this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(value) }));
|
|
}
|
|
}
|
|
const stopTrack = vi.fn();
|
|
const getUserMedia = vi.fn();
|
|
const sendBeacon = vi.fn(() => true);
|
|
const options = (): ClientOptions => ({
|
|
directUrl: "ws://localhost:8420/v1/realtime",
|
|
context: { resume: vi.fn().mockResolvedValue(undefined) } as unknown as AudioContext,
|
|
voice: "Aiden",
|
|
instructions: "Habla español",
|
|
tools: [],
|
|
gate: { enabled: false, thresholdDb: -66 },
|
|
});
|
|
|
|
async function connect(client: RealtimeClient, pending = client.connect()) {
|
|
await vi.waitFor(() => {
|
|
expect(Socket.instances).toHaveLength(1);
|
|
});
|
|
const socket = Socket.instances[0];
|
|
if (!socket) {
|
|
throw new Error("Missing socket");
|
|
}
|
|
socket.receive({ type: "session.created" });
|
|
await vi.waitFor(() => {
|
|
expect(socket.sent[0]?.type).toBe("session.update");
|
|
});
|
|
await pending;
|
|
return socket;
|
|
}
|
|
beforeEach(() => {
|
|
vi.useFakeTimers();
|
|
vi.clearAllMocks();
|
|
Socket.instances = [];
|
|
audio.start.mockResolvedValue(undefined);
|
|
audio.close.mockResolvedValue(undefined);
|
|
getUserMedia.mockResolvedValue({ getTracks: () => [{ stop: stopTrack }] });
|
|
vi.stubGlobal("navigator", { mediaDevices: { getUserMedia }, sendBeacon });
|
|
vi.stubGlobal("WebSocket", Socket);
|
|
});
|
|
afterEach(() => {
|
|
vi.useRealTimers();
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
describe("realtime session", () => {
|
|
it("waits for buffered audio to drain after response.done", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const socket = await connect(client);
|
|
socket.receive({ type: "response.created", response: { id: "r1" } });
|
|
socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" });
|
|
socket.receive({ type: "response.done", response: { id: "r1", status: "completed" } });
|
|
await vi.waitFor(() => {
|
|
expect(audio.finishPlayback).toHaveBeenCalledOnce();
|
|
});
|
|
expect(client.status).toBe("ai-speaking");
|
|
audio.start.mock.calls[0]?.[3]();
|
|
expect(client.status).toBe("listening");
|
|
await client.close();
|
|
});
|
|
it("does not let a playback drain override a user interruption", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const socket = await connect(client);
|
|
socket.receive({ type: "response.created", response: { id: "r1" } });
|
|
socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" });
|
|
socket.receive({ type: "input_audio_buffer.speech_started" });
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("user-speaking");
|
|
});
|
|
audio.start.mock.calls[0]?.[3]();
|
|
expect(client.status).toBe("user-speaking");
|
|
await client.close();
|
|
});
|
|
it("starts microphone streaming without a session.updated acknowledgment", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const pending = client.connect();
|
|
const settled = pending.catch(() => undefined);
|
|
try {
|
|
await vi.waitFor(() => {
|
|
expect(audio.start).toHaveBeenCalledOnce();
|
|
});
|
|
const socket = Socket.instances[0];
|
|
const capture = audio.start.mock.calls[0]?.[0];
|
|
if (!socket || !capture) {
|
|
throw new Error("Missing socket or microphone callback");
|
|
}
|
|
const pcm = new ArrayBuffer(2);
|
|
capture(pcm);
|
|
expect(socket.sent).toHaveLength(0);
|
|
|
|
socket.receive({ type: "session.created" });
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("listening");
|
|
});
|
|
await expect(pending).resolves.toBeUndefined();
|
|
capture(pcm);
|
|
expect(socket.sent).toEqual([
|
|
expect.objectContaining({ type: "session.update" }),
|
|
{ type: "input_audio_buffer.append", audio: "AAA=" },
|
|
]);
|
|
await vi.advanceTimersByTimeAsync(21000);
|
|
expect(client.status).toBe("listening");
|
|
expect(socket.close).not.toHaveBeenCalled();
|
|
} finally {
|
|
await client.close();
|
|
await settled;
|
|
}
|
|
expect(audio.close).toHaveBeenCalledOnce();
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
});
|
|
|
|
it("times out and releases the microphone when the server never creates a session", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const rejected = expect(client.connect()).rejects.toThrow("El servidor de voz no respondió a tiempo.");
|
|
await vi.waitFor(() => {
|
|
expect(audio.start).toHaveBeenCalledOnce();
|
|
});
|
|
await vi.advanceTimersByTimeAsync(20000);
|
|
await rejected;
|
|
expect(Socket.instances[0]?.sent).toHaveLength(0);
|
|
expect(Socket.instances[0]?.close).toHaveBeenCalledExactlyOnceWith(1000, "client closed");
|
|
expect(audio.close).toHaveBeenCalledOnce();
|
|
expect(client.status).toBe("closed");
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
});
|
|
|
|
it("connects with a plain load-balancer grant and closes without further HTTP requests", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValueOnce(
|
|
new Response(
|
|
JSON.stringify({
|
|
session_id: "local-session",
|
|
connect_url: "ws://localhost/v1/realtime",
|
|
}),
|
|
),
|
|
);
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
const client = new RealtimeClient({ ...options(), directUrl: undefined, sessionUrl: "/api/session" });
|
|
const error = vi.fn();
|
|
client.on("error", error);
|
|
const socket = await connect(client);
|
|
await vi.advanceTimersByTimeAsync(60000);
|
|
expect(client.status).toBe("listening");
|
|
expect(error).not.toHaveBeenCalled();
|
|
expect(socket.url).toBe("ws://localhost/v1/realtime");
|
|
await client.close();
|
|
expect(socket.close).toHaveBeenCalledOnce();
|
|
expect(audio.close).toHaveBeenCalledOnce();
|
|
expect(fetchMock).toHaveBeenCalledOnce();
|
|
expect(fetchMock).toHaveBeenCalledWith("/api/session", expect.objectContaining({ method: "POST" }));
|
|
expect(sendBeacon).not.toHaveBeenCalled();
|
|
expect(client.status).toBe("closed");
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
});
|
|
|
|
it.each(["ws://voice.test/v1/realtime", "ws://localhost:8420/v1/realtime"])(
|
|
"starts and cleans up the direct or proxied socket at %s",
|
|
async (directUrl) => {
|
|
const fetchMock = vi.fn();
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
const client = new RealtimeClient({ ...options(), directUrl });
|
|
const socket = await connect(client);
|
|
expect(socket.url).toBe(directUrl);
|
|
expect(audio.start).toHaveBeenCalledOnce();
|
|
await client.close();
|
|
await client.close();
|
|
expect(socket.close).toHaveBeenCalledExactlyOnceWith(1000, "client closed");
|
|
expect(audio.close).toHaveBeenCalledOnce();
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
expect(sendBeacon).not.toHaveBeenCalled();
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
},
|
|
);
|
|
|
|
it("polls a queue ticket and joins the granted slot before its deadline", async () => {
|
|
const fetchMock = vi
|
|
.fn<(path: string, init?: RequestInit) => Promise<Response>>()
|
|
.mockResolvedValueOnce(new Response(JSON.stringify({ state: "queued", queue_id: "q1", position: 2 })))
|
|
.mockResolvedValueOnce(new Response(JSON.stringify({ state: "queued", queue_id: "q1", position: 1 })))
|
|
.mockResolvedValueOnce(
|
|
new Response(JSON.stringify({ session_id: "s1", connect_url: "ws://voice.test", pending_timeout_s: 10 })),
|
|
);
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
const client = new RealtimeClient({ ...options(), directUrl: undefined });
|
|
const readyToJoin = vi.fn();
|
|
client.on("readyToJoin", readyToJoin);
|
|
const pending = client.connect();
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("queued");
|
|
});
|
|
await vi.advanceTimersByTimeAsync(4000);
|
|
expect(fetchMock.mock.calls.map(([path]) => path)).toEqual(["/api/session", "/api/queue/q1", "/api/queue/q1"]);
|
|
expect(client.status).toBe("your-turn");
|
|
expect(readyToJoin).toHaveBeenCalledWith(expect.objectContaining({ expiresSec: 7 }));
|
|
expect(Socket.instances).toHaveLength(0);
|
|
client.join();
|
|
await connect(client, pending);
|
|
await client.close();
|
|
expect(sendBeacon).not.toHaveBeenCalled();
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
});
|
|
|
|
it("expires an unclaimed join offer after releasing the permission probe's microphone", async () => {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi
|
|
.fn()
|
|
.mockResolvedValueOnce(new Response(JSON.stringify({ state: "queued", queue_id: "q1" })))
|
|
.mockResolvedValueOnce(
|
|
new Response(JSON.stringify({ session_id: "s1", connect_url: "ws://voice.test", pending_timeout_s: 10 })),
|
|
),
|
|
);
|
|
const client = new RealtimeClient({ ...options(), directUrl: undefined });
|
|
const rejected = expect(client.connect()).rejects.toMatchObject({ code: "join-expired" });
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("queued");
|
|
});
|
|
await vi.advanceTimersByTimeAsync(9000);
|
|
await rejected;
|
|
expect(stopTrack).toHaveBeenCalledOnce();
|
|
expect(audio.start).not.toHaveBeenCalled();
|
|
expect(Socket.instances).toHaveLength(0);
|
|
expect(client.status).toBe("closed");
|
|
expect(sendBeacon).not.toHaveBeenCalled();
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
});
|
|
|
|
it("stops polling when the load balancer expires a queue ticket", async () => {
|
|
const fetchMock = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(new Response(JSON.stringify({ state: "queued", queue_id: "q1" })))
|
|
.mockResolvedValueOnce(new Response(JSON.stringify({ state: "expired" }), { status: 404 }));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
const client = new RealtimeClient({ ...options(), directUrl: undefined });
|
|
const rejected = expect(client.connect()).rejects.toMatchObject({ code: "queue-expired" });
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("queued");
|
|
});
|
|
await vi.advanceTimersByTimeAsync(2000);
|
|
await rejected;
|
|
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
expect(Socket.instances).toHaveLength(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
});
|
|
|
|
it("uses a keepalive request to cancel a queue ticket when sendBeacon declines it", async () => {
|
|
sendBeacon.mockReturnValueOnce(false);
|
|
const fetchMock = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(new Response(JSON.stringify({ state: "queued", queue_id: "q1" })))
|
|
.mockResolvedValueOnce(new Response(JSON.stringify({ ok: true })));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
const client = new RealtimeClient({ ...options(), directUrl: undefined });
|
|
const rejected = expect(client.connect()).rejects.toMatchObject({ code: "aborted" });
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("queued");
|
|
});
|
|
await client.close();
|
|
await rejected;
|
|
expect(fetchMock).toHaveBeenLastCalledWith(
|
|
"/api/queue/end",
|
|
expect.objectContaining({ method: "POST", body: JSON.stringify({ queueId: "q1" }), keepalive: true }),
|
|
);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
});
|
|
|
|
it("replaces cumulative user transcript deltas instead of duplicating words", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const transcript = vi.fn();
|
|
client.on("transcript", transcript);
|
|
const socket = await connect(client);
|
|
socket.receive({ type: "conversation.item.input_audio_transcription.delta", item_id: "user1", delta: "Hola" });
|
|
socket.receive({
|
|
type: "conversation.item.input_audio_transcription.delta",
|
|
item_id: "user1",
|
|
delta: "Hola Vocero",
|
|
});
|
|
await vi.waitFor(() => {
|
|
expect(transcript).toHaveBeenCalledTimes(2);
|
|
});
|
|
expect(transcript).toHaveBeenLastCalledWith({ role: "user", text: "Hola Vocero", partial: true, id: "user1" });
|
|
await client.close();
|
|
});
|
|
|
|
it("suppresses the next native reply when external reasoning wins before response.created", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const transcript = vi.fn();
|
|
client.on("transcript", transcript);
|
|
const socket = await connect(client);
|
|
client.renderExternalAnswer("Respuesta externa");
|
|
socket.receive({ type: "response.created", response: { id: "native" } });
|
|
socket.receive({ type: "response.audio.delta", response_id: "native", delta: "AAAA" });
|
|
socket.receive({ type: "response.audio_transcript.delta", response_id: "native", delta: "Respuesta duplicada" });
|
|
socket.receive({ type: "response.done", response: { id: "native", status: "completed" } });
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("listening");
|
|
});
|
|
expect(audio.play).not.toHaveBeenCalled();
|
|
expect(transcript).not.toHaveBeenCalled();
|
|
expect(socket.sent.at(-1)).toMatchObject({ item: { role: "assistant", content: [{ text: "Respuesta externa" }] } });
|
|
await client.close();
|
|
});
|
|
it("sends configuration before greeting without an acknowledgment and serializes response requests", async () => {
|
|
const client = new RealtimeClient({ ...options(), greeting: "hola vocero" });
|
|
const socket = await connect(client);
|
|
expect(socket.sent.map((event) => event.type)).toEqual([
|
|
"session.update",
|
|
"conversation.item.create",
|
|
"response.create",
|
|
]);
|
|
socket.receive({ type: "response.created", response: { id: "r1" } });
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("processing");
|
|
});
|
|
const sentCount = socket.sent.length;
|
|
socket.receive({ type: "session.updated" });
|
|
socket.receive({ type: "session.updated" });
|
|
await vi.advanceTimersByTimeAsync(0);
|
|
expect(client.status).toBe("processing");
|
|
expect(socket.sent).toHaveLength(sentCount);
|
|
client.sendUserText("Otra pregunta");
|
|
client.requestResponse();
|
|
expect(socket.sent.filter((event) => event.type === "response.create")).toHaveLength(1);
|
|
socket.receive({ type: "response.done", response: { id: "r1", status: "completed" } });
|
|
await vi.waitFor(() => {
|
|
expect(socket.sent.filter((event) => event.type === "response.create")).toHaveLength(2);
|
|
});
|
|
expect(socket.sent.at(-2)).toMatchObject({
|
|
type: "conversation.item.create",
|
|
item: { role: "user", content: [{ text: "Otra pregunta" }] },
|
|
});
|
|
await client.close();
|
|
expect(audio.close).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("assembles streaming transcripts and preserves an interrupted response", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const transcripts = vi.fn();
|
|
const finished = vi.fn();
|
|
client.on("transcript", transcripts);
|
|
client.on("finished", finished);
|
|
const socket = await connect(client);
|
|
socket.receive({ type: "response.created", response: { id: "reply" } });
|
|
socket.receive({ type: "response.audio_transcript.delta", response_id: "reply", delta: "Hola " });
|
|
socket.receive({ type: "response.audio_transcript.delta", response_id: "reply", delta: "mundo" });
|
|
socket.receive({ type: "response.audio.delta", response_id: "reply", delta: "AAAA" });
|
|
socket.receive({ type: "input_audio_buffer.speech_started" });
|
|
socket.receive({ type: "response.done", response: { id: "reply", status: "cancelled" } });
|
|
await vi.waitFor(() => {
|
|
expect(finished).toHaveBeenCalledOnce();
|
|
});
|
|
expect(transcripts).toHaveBeenLastCalledWith({ role: "assistant", id: "reply", text: "Hola mundo", partial: true });
|
|
expect(finished).toHaveBeenCalledWith({
|
|
responseId: "reply",
|
|
transcript: "Hola mundo",
|
|
status: "cancelled",
|
|
audible: true,
|
|
});
|
|
expect(audio.clear).toHaveBeenCalled();
|
|
await client.close();
|
|
});
|
|
|
|
it("does not open a socket after cancellation during microphone permission", async () => {
|
|
let allow: ((value: unknown) => void) | undefined;
|
|
getUserMedia.mockReturnValue(
|
|
new Promise((resolve) => {
|
|
allow = resolve;
|
|
}),
|
|
);
|
|
const client = new RealtimeClient(options());
|
|
const pending = client.connect();
|
|
const rejected = expect(pending).rejects.toMatchObject({ code: "aborted" });
|
|
await client.close();
|
|
allow?.({ getTracks: () => [{ stop: stopTrack }] });
|
|
await rejected;
|
|
expect(stopTrack).toHaveBeenCalledOnce();
|
|
expect(Socket.instances).toHaveLength(0);
|
|
});
|
|
|
|
it("releases a queue ticket when leaving before admission", async () => {
|
|
const fetchMock = vi
|
|
.fn()
|
|
.mockResolvedValue(new Response(JSON.stringify({ state: "queued", queue_id: "q1", position: 2 })));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
const client = new RealtimeClient({ ...options(), directUrl: undefined, sessionUrl: "/api/session" });
|
|
const pending = client.connect();
|
|
const rejected = expect(pending).rejects.toBeInstanceOf(SessionError);
|
|
await vi.waitFor(() => {
|
|
expect(client.status).toBe("queued");
|
|
});
|
|
await client.close();
|
|
await rejected;
|
|
expect(sendBeacon).toHaveBeenCalledWith("/api/queue/end", expect.any(Blob));
|
|
expect(fetchMock).toHaveBeenCalledOnce();
|
|
expect(Socket.instances).toHaveLength(0);
|
|
});
|
|
|
|
it("sends each tool call once and returns its result using the realtime protocol", async () => {
|
|
const client = new RealtimeClient(options());
|
|
const tool = vi.fn();
|
|
client.on("tool", tool);
|
|
const socket = await connect(client);
|
|
const call = {
|
|
type: "response.function_call_arguments.done",
|
|
name: "web_search",
|
|
arguments: '{"query":"hoy"}',
|
|
call_id: "call1",
|
|
};
|
|
socket.receive(call);
|
|
socket.receive(call);
|
|
await vi.waitFor(() => {
|
|
expect(tool).toHaveBeenCalledOnce();
|
|
});
|
|
client.sendToolOutput("call1", "Resultado");
|
|
expect(socket.sent.at(-2)).toMatchObject({
|
|
item: { type: "function_call_output", call_id: "call1", output: "Resultado" },
|
|
});
|
|
await client.close();
|
|
});
|
|
});
|