From 09bf6ece00fe84994e6217fb85d446a4285ece64 Mon Sep 17 00:00:00 2001 From: Alvaro Parker Date: Wed, 7 Oct 2026 09:27:54 -0300 Subject: [PATCH] fix(frontend): track playback completion and meaningful inactivity --- vocero-api/CHANGELOG.md | 3 + .../src/audio/audio-playback.test.ts | 56 +++++++++++++++++++ .../src/audio/audio-playback.worklet.ts | 20 ++++++- vocero-frontend/src/audio/engine.ts | 20 ++++++- vocero-frontend/src/realtime/client.test.ts | 29 ++++++++++ vocero-frontend/src/realtime/client.ts | 20 ++++++- .../src/realtime/controller.test.ts | 14 +++++ vocero-frontend/src/realtime/controller.ts | 21 ++++++- 8 files changed, 173 insertions(+), 10 deletions(-) create mode 100644 vocero-frontend/src/audio/audio-playback.test.ts diff --git a/vocero-api/CHANGELOG.md b/vocero-api/CHANGELOG.md index 68eea7b..2c9809f 100644 --- a/vocero-api/CHANGELOG.md +++ b/vocero-api/CHANGELOG.md @@ -9,6 +9,9 @@ the implementation at the recorded date; current operating instructions live in ### Fixed +- Keep the voice UI speaking until buffered playback finishes, flush short final audio chunks, + and exclude response processing from the user inactivity timeout. + - Make frontend configuration failures retryable, bound startup requests, and load optional reports independently while waiting for caller context before starting a session. Cancel superseded external reasoning and reject stale diff --git a/vocero-frontend/src/audio/audio-playback.test.ts b/vocero-frontend/src/audio/audio-playback.test.ts new file mode 100644 index 0000000..6d7e344 --- /dev/null +++ b/vocero-frontend/src/audio/audio-playback.test.ts @@ -0,0 +1,56 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +let processor: AudioWorkletProcessor; +const port = { onmessage: undefined as ((event: MessageEvent) => void) | undefined, postMessage: vi.fn() }; +beforeEach(async () => { + vi.resetModules(); + port.postMessage.mockClear(); + vi.stubGlobal("sampleRate", 48000); + vi.stubGlobal( + "AudioWorkletProcessor", + class { + port = port; + }, + ); + vi.stubGlobal("registerProcessor", (_name: string, Processor: new () => AudioWorkletProcessor) => { + processor = new Processor(); + }); + await import("./audio-playback.worklet"); +}); +afterEach(() => { + vi.unstubAllGlobals(); +}); +function send(data: unknown) { + port.onmessage?.(new MessageEvent("message", { data })); +} +function render(blocks: number) { + let audible = false; + for (let index = 0; index < blocks; index++) { + const output = new Float32Array(128); + processor.process([], [[output]]); + audible ||= output.some((value) => value !== 0); + } + return audible; +} +it("flushes a final chunk shorter than the prebuffer and signals after playback finishes", () => { + send({ kind: "audio", samples: new Float32Array(160).fill(0.5), sequence: 1 }); + expect(render(1)).toBe(false); + expect(port.postMessage).not.toHaveBeenCalled(); + send({ kind: "finish", sequence: 1 }); + expect(render(1)).toBe(true); + expect(port.postMessage).not.toHaveBeenCalled(); + render(12); + expect(port.postMessage).toHaveBeenCalledExactlyOnceWith({ kind: "drained", sequence: 1 }); +}); +it("discards cleared samples and identifies the latest playback generation", () => { + send({ kind: "audio", samples: new Float32Array(16000).fill(0.5), sequence: 1 }); + expect(render(1)).toBe(true); + send({ kind: "clear", sequence: 2 }); + render(2); + expect(port.postMessage).toHaveBeenLastCalledWith({ kind: "drained", sequence: 2 }); + expect(render(10)).toBe(false); + send({ kind: "audio", samples: new Float32Array(160).fill(0.25), sequence: 3 }); + send({ kind: "finish", sequence: 3 }); + expect(render(12)).toBe(true); + expect(port.postMessage).toHaveBeenLastCalledWith({ kind: "drained", sequence: 3 }); +}); diff --git a/vocero-frontend/src/audio/audio-playback.worklet.ts b/vocero-frontend/src/audio/audio-playback.worklet.ts index 8ef9364..823dbe5 100644 --- a/vocero-frontend/src/audio/audio-playback.worklet.ts +++ b/vocero-frontend/src/audio/audio-playback.worklet.ts @@ -1,5 +1,7 @@ type PlaybackMessage = - { kind: "config"; inputRate: number } | { kind: "audio"; samples: Float32Array } | { kind: "clear" }; + | { kind: "config"; inputRate: number } + | { kind: "audio"; samples: Float32Array; sequence: number } + | { kind: "clear" | "finish"; sequence: number }; class AudioPlaybackProcessor extends AudioWorkletProcessor { private rate = 16000; @@ -10,6 +12,8 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor { private fadeIn = 0; private fadeOut = 0; private lastSample = 0; + private sequence = 0; + private finished = false; constructor() { super(); @@ -22,16 +26,24 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor { } break; case "audio": + this.sequence = data.sequence; + this.finished = false; if (data.samples.length) { this.queue.push(data.samples); } break; case "clear": + this.sequence = data.sequence; + this.finished = true; this.queue = []; this.offset = 0; this.fraction = 0; this.fadeOut = 32; break; + case "finish": + this.sequence = data.sequence; + this.finished = true; + break; } }; } @@ -66,7 +78,7 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor { } for (let index = 0; index < output.length; index++) { let value = 0; - if (!this.playing && this.queue.length && this.bufferedMs() >= 90) { + if (!this.playing && this.queue.length && (this.finished || this.bufferedMs() >= 90)) { this.playing = true; this.fadeIn = 32; this.fadeOut = 0; @@ -102,6 +114,10 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor { stereo[index] = value; } } + if (this.finished && !this.playing && !this.queue.length) { + this.finished = false; + this.port.postMessage({ kind: "drained", sequence: this.sequence }); + } return true; } } diff --git a/vocero-frontend/src/audio/engine.ts b/vocero-frontend/src/audio/engine.ts index ec3cb8a..2c92880 100644 --- a/vocero-frontend/src/audio/engine.ts +++ b/vocero-frontend/src/audio/engine.ts @@ -26,13 +26,19 @@ export class AudioEngine { private closed = false; private gate: NoiseGate; private muted = false; + private playbackSequence = 0; constructor(context: AudioContext, gate: NoiseGate) { this.context = context; this.gate = gate; } - async start(onChunk: (chunk: ArrayBuffer) => void, onLevel: (rms: number) => void, speaking: () => boolean) { + async start( + onChunk: (chunk: ArrayBuffer) => void, + onLevel: (rms: number) => void, + speaking: () => boolean, + onDrained: () => void, + ) { const stream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); if (this.closed) { stream.getTracks().forEach((track) => { @@ -58,6 +64,11 @@ export class AudioEngine { playback.port.postMessage({ kind: "config", inputRate: 16000 }); this.capture = capture; this.playback = playback; + playback.port.onmessage = (event: MessageEvent<{ kind: string; sequence: number }>) => { + if (!this.closed && event.data.kind === "drained" && event.data.sequence === this.playbackSequence) { + onDrained(); + } + }; this.setGate(this.gate); this.setMuted(this.muted); capture.port.onmessage = (event: MessageEvent) => { @@ -110,10 +121,13 @@ export class AudioEngine { return; } const samples = decodePcm(base64); - this.playback?.port.postMessage({ kind: "audio", samples }, [samples.buffer]); + this.playback?.port.postMessage({ kind: "audio", samples, sequence: ++this.playbackSequence }, [samples.buffer]); } clear() { - this.playback?.port.postMessage({ kind: "clear" }); + this.playback?.port.postMessage({ kind: "clear", sequence: ++this.playbackSequence }); + } + finishPlayback() { + this.playback?.port.postMessage({ kind: "finish", sequence: this.playbackSequence }); } setGate(gate: NoiseGate) { this.gate = gate; diff --git a/vocero-frontend/src/realtime/client.test.ts b/vocero-frontend/src/realtime/client.test.ts index c5e4f9a..7233e0a 100644 --- a/vocero-frontend/src/realtime/client.test.ts +++ b/vocero-frontend/src/realtime/client.test.ts @@ -9,6 +9,7 @@ const audio = vi.hoisted(() => ({ close: vi.fn<() => Promise>(), play: vi.fn(), clear: vi.fn(), + finishPlayback: vi.fn(), setMuted: vi.fn(), setGate: vi.fn(), })); @@ -19,6 +20,7 @@ vi.mock("../audio/engine", () => ({ close = audio.close; play = audio.play; clear = audio.clear; + finishPlayback = audio.finishPlayback; setMuted = audio.setMuted; setGate = audio.setGate; }, @@ -89,6 +91,33 @@ afterEach(() => { }); describe("realtime session", () => { + it("waits for buffered audio to drain after response.done", async () => { + const client = new RealtimeClient(options()); + const socket = await connect(client); + socket.receive({ type: "response.created", response: { id: "r1" } }); + socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" }); + socket.receive({ type: "response.done", response: { id: "r1", status: "completed" } }); + await vi.waitFor(() => { + expect(audio.finishPlayback).toHaveBeenCalledOnce(); + }); + expect(client.status).toBe("ai-speaking"); + audio.start.mock.calls[0]?.[3](); + expect(client.status).toBe("listening"); + await client.close(); + }); + it("does not let a playback drain override a user interruption", async () => { + const client = new RealtimeClient(options()); + const socket = await connect(client); + socket.receive({ type: "response.created", response: { id: "r1" } }); + socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" }); + socket.receive({ type: "input_audio_buffer.speech_started" }); + await vi.waitFor(() => { + expect(client.status).toBe("user-speaking"); + }); + audio.start.mock.calls[0]?.[3](); + expect(client.status).toBe("user-speaking"); + await client.close(); + }); it("starts microphone streaming without a session.updated acknowledgment", async () => { const client = new RealtimeClient(options()); const pending = client.connect(); diff --git a/vocero-frontend/src/realtime/client.ts b/vocero-frontend/src/realtime/client.ts index 11bcfa9..ba09b6a 100644 --- a/vocero-frontend/src/realtime/client.ts +++ b/vocero-frontend/src/realtime/client.ts @@ -58,6 +58,7 @@ export class RealtimeClient { private socket: WebSocket | null = null; private currentStatus: ConnectionStatus = "idle"; private configured = false; + private playbackPending = false; private closed = false; private closing: Promise | null = null; private queueId = ""; @@ -149,6 +150,10 @@ export class RealtimeClient { this.emit("level", rms); }, () => this.currentStatus === "ai-speaking", + () => { + this.playbackPending = false; + this.settleResponseStatus(); + }, ); this.socket.addEventListener("message", (event: MessageEvent) => { this.incoming = this.incoming @@ -336,6 +341,7 @@ export class RealtimeClient { } break; case "input_audio_buffer.speech_started": + this.playbackPending = false; this.audio.clear(); this.setStatus("user-speaking"); break; @@ -350,11 +356,12 @@ export class RealtimeClient { this.suppressed.add(id); this.suppressNext = false; } - this.setStatus("processing"); + this.setStatus(this.playbackPending ? "ai-speaking" : "processing"); break; case "response.audio.delta": case "response.output_audio.delta": if (event.delta && !this.suppressed.has(id)) { + this.playbackPending = true; this.audio.play(event.delta); this.audible.add(id); this.setStatus("ai-speaking"); @@ -422,10 +429,11 @@ export class RealtimeClient { if (this.activeResponse === id) { this.activeResponse = ""; } - if (this.currentStatus === "ai-speaking" || this.currentStatus === "processing") { - this.setStatus("listening"); + if (!this.activeResponses.size) { + this.audio.finishPlayback(); } this.flush(); + this.settleResponseStatus(); break; } case "response.function_call_arguments.done": @@ -449,6 +457,11 @@ export class RealtimeClient { break; } } + private settleResponseStatus() { + if (!this.closed && (this.currentStatus === "ai-speaking" || this.currentStatus === "processing")) { + this.setStatus(this.playbackPending ? "ai-speaking" : this.busy() ? "processing" : "listening"); + } + } private busy() { return this.activeResponses.size > 0 || this.createInFlight; } @@ -521,6 +534,7 @@ export class RealtimeClient { } else { this.suppressNext = true; } + this.playbackPending = false; this.audio.clear(); this.pendingCreates = 0; this.deferredTexts = []; diff --git a/vocero-frontend/src/realtime/controller.test.ts b/vocero-frontend/src/realtime/controller.test.ts index 9b01014..3512bc4 100644 --- a/vocero-frontend/src/realtime/controller.test.ts +++ b/vocero-frontend/src/realtime/controller.test.ts @@ -247,4 +247,18 @@ describe("voice application lifecycle", () => { expect(client.close).toHaveBeenCalledOnce(); expect(controller.getSnapshot().status).toBe("closed"); }); + it("does not count native processing or an external answer request as user inactivity", async () => { + const client = await start(); + client.emit("status", "processing"); + await vi.advanceTimersByTimeAsync(60000); + expect(client.close).not.toHaveBeenCalled(); + client.emit("transcript", { id: "u1", role: "user", text: "Consulta lenta", partial: false }); + client.emit("status", "listening"); + await vi.advanceTimersByTimeAsync(60000); + expect(client.close).not.toHaveBeenCalled(); + expect(controller.getSnapshot().status).toBe("processing"); + pendingReasoning?.(new Response(JSON.stringify({ answer: "Respuesta" }))); + await vi.advanceTimersByTimeAsync(1); + expect(controller.getSnapshot().status).toBe("listening"); + }); }); diff --git a/vocero-frontend/src/realtime/controller.ts b/vocero-frontend/src/realtime/controller.ts index bbc09fa..43e4328 100644 --- a/vocero-frontend/src/realtime/controller.ts +++ b/vocero-frontend/src/realtime/controller.ts @@ -71,6 +71,7 @@ export class VoiceController { private initialization: AbortController | null = null; private reasoningRequest: AbortController | null = null; private reasoningTurn = 0; + private externalSpeaking = false; private idleTimer: ReturnType | undefined; private lastActivity = Date.now(); private lastMeter = 0; @@ -308,7 +309,12 @@ export class VoiceController { ), }); } - this.update({ status }); + if ( + !["listening", "processing", "ai-speaking"].includes(status) || + (!this.reasoningRequest && !this.externalSpeaking) + ) { + this.update({ status }); + } }), client.on("transcript", (transcript) => { this.lastActivity = Date.now(); @@ -357,7 +363,9 @@ export class VoiceController { this.lastActivity = Date.now(); this.idleTimer = setInterval(() => { if ( - ["listening", "user-speaking", "processing"].includes(this.state.status) && + this.state.status === "listening" && + !this.reasoningRequest && + !this.externalSpeaking && Date.now() - this.lastActivity >= 30000 ) { void this.stop(); @@ -413,6 +421,7 @@ export class VoiceController { this.reasoningTurn++; this.reasoningRequest?.abort(); this.reasoningRequest = null; + this.externalSpeaking = false; } private async reason(transcript: Transcript, client: RealtimeClient) { const { config, profile, rag } = this.state; @@ -423,6 +432,7 @@ export class VoiceController { const turn = this.reasoningTurn; const request = new AbortController(); this.reasoningRequest = request; + this.update({ status: "processing" }); const current = () => this.client === client && turn === this.reasoningTurn && !request.signal.aborted; window.speechSynthesis?.cancel(); try { @@ -449,6 +459,7 @@ export class VoiceController { client.renderExternalAnswer(answer); this.transcript({ role: "assistant", id: `reasoning:${transcript.id}`, text: answer, partial: false }); if ("speechSynthesis" in window) { + this.externalSpeaking = true; const utterance = new SpeechSynthesisUtterance(answer); utterance.lang = "es-ES"; const voice = window.speechSynthesis.getVoices().find((voice) => voice.lang.startsWith("es")); @@ -462,6 +473,7 @@ export class VoiceController { }; utterance.onend = utterance.onerror = () => { if (current()) { + this.externalSpeaking = false; this.lastActivity = Date.now(); this.update({ status: "listening" }); } @@ -471,11 +483,16 @@ export class VoiceController { } } catch (error) { if (current()) { + this.externalSpeaking = false; this.update({ error: errorMessage(error) }); } } finally { if (turn === this.reasoningTurn) { this.reasoningRequest = null; + if (!this.externalSpeaking && current()) { + this.lastActivity = Date.now(); + this.update({ status: "listening" }); + } } } }