diff --git a/s2s-frontend/README.md b/s2s-frontend/README.md index 96cd1ab..22d5f2e 100644 --- a/s2s-frontend/README.md +++ b/s2s-frontend/README.md @@ -147,6 +147,13 @@ credentials, and ApiPilot execution. Browser `SpeechSynthesis` is used only when the configured external reasoning service returns an answer instead of `log_only`. +Session startup waits for audio initialization and `session.created`, then sends +`session.update` before microphone audio or the initial greeting. The speech +server processes those frames in order and does not emit `session.updated` on +success. That acknowledgment is therefore optional; receiving it later does not +restart the greeting or reset an active response. The startup timeout still +closes the microphone and socket if the server never creates a session. + ## References - [Vite getting started](https://vite.dev/guide/) diff --git a/s2s-frontend/src/realtime/client.test.ts b/s2s-frontend/src/realtime/client.test.ts index aaead41..c5e4f9a 100644 --- a/s2s-frontend/src/realtime/client.test.ts +++ b/s2s-frontend/src/realtime/client.test.ts @@ -1,9 +1,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AudioEngine } from "../audio/engine"; + import { RealtimeClient, SessionError, type ClientOptions } from "./client"; const audio = vi.hoisted(() => ({ - start: vi.fn<(...callbacks: unknown[]) => Promise>(), + start: vi.fn(), close: vi.fn<() => Promise>(), play: vi.fn(), clear: vi.fn(), @@ -68,7 +70,6 @@ async function connect(client: RealtimeClient, pending = client.connect()) { await vi.waitFor(() => { expect(socket.sent[0]?.type).toBe("session.update"); }); - socket.receive({ type: "session.updated" }); await pending; return socket; } @@ -88,6 +89,59 @@ afterEach(() => { }); describe("realtime session", () => { + it("starts microphone streaming without a session.updated acknowledgment", async () => { + const client = new RealtimeClient(options()); + const pending = client.connect(); + const settled = pending.catch(() => undefined); + try { + await vi.waitFor(() => { + expect(audio.start).toHaveBeenCalledOnce(); + }); + const socket = Socket.instances[0]; + const capture = audio.start.mock.calls[0]?.[0]; + if (!socket || !capture) { + throw new Error("Missing socket or microphone callback"); + } + const pcm = new ArrayBuffer(2); + capture(pcm); + expect(socket.sent).toHaveLength(0); + + socket.receive({ type: "session.created" }); + await vi.waitFor(() => { + expect(client.status).toBe("listening"); + }); + await expect(pending).resolves.toBeUndefined(); + capture(pcm); + expect(socket.sent).toEqual([ + expect.objectContaining({ type: "session.update" }), + { type: "input_audio_buffer.append", audio: "AAA=" }, + ]); + await vi.advanceTimersByTimeAsync(21000); + expect(client.status).toBe("listening"); + expect(socket.close).not.toHaveBeenCalled(); + } finally { + await client.close(); + await settled; + } + expect(audio.close).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("times out and releases the microphone when the server never creates a session", async () => { + const client = new RealtimeClient(options()); + const rejected = expect(client.connect()).rejects.toThrow("El servidor de voz no respondió a tiempo."); + await vi.waitFor(() => { + expect(audio.start).toHaveBeenCalledOnce(); + }); + await vi.advanceTimersByTimeAsync(20000); + await rejected; + expect(Socket.instances[0]?.sent).toHaveLength(0); + expect(Socket.instances[0]?.close).toHaveBeenCalledExactlyOnceWith(1000, "client closed"); + expect(audio.close).toHaveBeenCalledOnce(); + expect(client.status).toBe("closed"); + expect(vi.getTimerCount()).toBe(0); + }); + it("connects with a plain load-balancer grant and closes without further HTTP requests", async () => { const fetchMock = vi.fn().mockResolvedValueOnce( new Response( @@ -263,7 +317,7 @@ describe("realtime session", () => { expect(socket.sent.at(-1)).toMatchObject({ item: { role: "assistant", content: [{ text: "Respuesta externa" }] } }); await client.close(); }); - it("waits for configuration before greeting and serializes response requests", async () => { + it("sends configuration before greeting without an acknowledgment and serializes response requests", async () => { const client = new RealtimeClient({ ...options(), greeting: "hola vocero" }); const socket = await connect(client); expect(socket.sent.map((event) => event.type)).toEqual([ @@ -275,6 +329,12 @@ describe("realtime session", () => { await vi.waitFor(() => { expect(client.status).toBe("processing"); }); + const sentCount = socket.sent.length; + socket.receive({ type: "session.updated" }); + socket.receive({ type: "session.updated" }); + await vi.advanceTimersByTimeAsync(0); + expect(client.status).toBe("processing"); + expect(socket.sent).toHaveLength(sentCount); client.sendUserText("Otra pregunta"); client.requestResponse(); expect(socket.sent.filter((event) => event.type === "response.create")).toHaveLength(1); diff --git a/s2s-frontend/src/realtime/client.ts b/s2s-frontend/src/realtime/client.ts index b618ec6..11bcfa9 100644 --- a/s2s-frontend/src/realtime/client.ts +++ b/s2s-frontend/src/realtime/client.ts @@ -320,8 +320,8 @@ export class RealtimeClient { tool_choice: this.options.tools.length ? "auto" : "none", }, }); - break; - case "session.updated": + // The speech server applies frames in order without acknowledging session.update. + // Send configuration before microphone audio or the greeting, then consider it ready. if (!this.configured) { this.configured = true; clearTimeout(this.readyTimer);