fix(frontend): track playback completion and meaningful inactivity

vocero-api/CHANGELOG.md

@@ -9,6 +9,9 @@ the implementation at the recorded date; current operating instructions live in
 
 ### Fixed
 
+- Keep the voice UI speaking until buffered playback finishes, flush short final audio chunks,
+  and exclude response processing from the user inactivity timeout.
+
 - Make frontend configuration failures retryable, bound startup requests, and
   load optional reports independently while waiting for caller context before
   starting a session. Cancel superseded external reasoning and reject stale

vocero-frontend/src/audio/audio-playback.test.ts

@@ -0,0 +1,56 @@
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+
+let processor: AudioWorkletProcessor;
+const port = { onmessage: undefined as ((event: MessageEvent) => void) | undefined, postMessage: vi.fn() };
+beforeEach(async () => {
+  vi.resetModules();
+  port.postMessage.mockClear();
+  vi.stubGlobal("sampleRate", 48000);
+  vi.stubGlobal(
+    "AudioWorkletProcessor",
+    class {
+      port = port;
+    },
+  );
+  vi.stubGlobal("registerProcessor", (_name: string, Processor: new () => AudioWorkletProcessor) => {
+    processor = new Processor();
+  });
+  await import("./audio-playback.worklet");
+});
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+function send(data: unknown) {
+  port.onmessage?.(new MessageEvent("message", { data }));
+}
+function render(blocks: number) {
+  let audible = false;
+  for (let index = 0; index < blocks; index++) {
+    const output = new Float32Array(128);
+    processor.process([], [[output]]);
+    audible ||= output.some((value) => value !== 0);
+  }
+  return audible;
+}
+it("flushes a final chunk shorter than the prebuffer and signals after playback finishes", () => {
+  send({ kind: "audio", samples: new Float32Array(160).fill(0.5), sequence: 1 });
+  expect(render(1)).toBe(false);
+  expect(port.postMessage).not.toHaveBeenCalled();
+  send({ kind: "finish", sequence: 1 });
+  expect(render(1)).toBe(true);
+  expect(port.postMessage).not.toHaveBeenCalled();
+  render(12);
+  expect(port.postMessage).toHaveBeenCalledExactlyOnceWith({ kind: "drained", sequence: 1 });
+});
+it("discards cleared samples and identifies the latest playback generation", () => {
+  send({ kind: "audio", samples: new Float32Array(16000).fill(0.5), sequence: 1 });
+  expect(render(1)).toBe(true);
+  send({ kind: "clear", sequence: 2 });
+  render(2);
+  expect(port.postMessage).toHaveBeenLastCalledWith({ kind: "drained", sequence: 2 });
+  expect(render(10)).toBe(false);
+  send({ kind: "audio", samples: new Float32Array(160).fill(0.25), sequence: 3 });
+  send({ kind: "finish", sequence: 3 });
+  expect(render(12)).toBe(true);
+  expect(port.postMessage).toHaveBeenLastCalledWith({ kind: "drained", sequence: 3 });
+});

vocero-frontend/src/audio/audio-playback.worklet.ts

@@ -1,5 +1,7 @@
 type PlaybackMessage =
-  { kind: "config"; inputRate: number } | { kind: "audio"; samples: Float32Array } | { kind: "clear" };
+  | { kind: "config"; inputRate: number }
+  | { kind: "audio"; samples: Float32Array; sequence: number }
+  | { kind: "clear" | "finish"; sequence: number };
 
 class AudioPlaybackProcessor extends AudioWorkletProcessor {
   private rate = 16000;
@@ -10,6 +12,8 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
   private fadeIn = 0;
   private fadeOut = 0;
   private lastSample = 0;
+  private sequence = 0;
+  private finished = false;
 
   constructor() {
     super();
@@ -22,16 +26,24 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
           }
           break;
         case "audio":
+          this.sequence = data.sequence;
+          this.finished = false;
           if (data.samples.length) {
             this.queue.push(data.samples);
           }
           break;
         case "clear":
+          this.sequence = data.sequence;
+          this.finished = true;
           this.queue = [];
           this.offset = 0;
           this.fraction = 0;
           this.fadeOut = 32;
           break;
+        case "finish":
+          this.sequence = data.sequence;
+          this.finished = true;
+          break;
       }
     };
   }
@@ -66,7 +78,7 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
     }
     for (let index = 0; index < output.length; index++) {
       let value = 0;
-      if (!this.playing && this.queue.length && this.bufferedMs() >= 90) {
+      if (!this.playing && this.queue.length && (this.finished || this.bufferedMs() >= 90)) {
         this.playing = true;
         this.fadeIn = 32;
         this.fadeOut = 0;
@@ -102,6 +114,10 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
         stereo[index] = value;
       }
     }
+    if (this.finished && !this.playing && !this.queue.length) {
+      this.finished = false;
+      this.port.postMessage({ kind: "drained", sequence: this.sequence });
+    }
     return true;
   }
 }

vocero-frontend/src/audio/engine.ts

@@ -26,13 +26,19 @@ export class AudioEngine {
   private closed = false;
   private gate: NoiseGate;
   private muted = false;
+  private playbackSequence = 0;
 
   constructor(context: AudioContext, gate: NoiseGate) {
     this.context = context;
     this.gate = gate;
   }
 
-  async start(onChunk: (chunk: ArrayBuffer) => void, onLevel: (rms: number) => void, speaking: () => boolean) {
+  async start(
+    onChunk: (chunk: ArrayBuffer) => void,
+    onLevel: (rms: number) => void,
+    speaking: () => boolean,
+    onDrained: () => void,
+  ) {
     const stream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
     if (this.closed) {
       stream.getTracks().forEach((track) => {
@@ -58,6 +64,11 @@ export class AudioEngine {
     playback.port.postMessage({ kind: "config", inputRate: 16000 });
     this.capture = capture;
     this.playback = playback;
+    playback.port.onmessage = (event: MessageEvent<{ kind: string; sequence: number }>) => {
+      if (!this.closed && event.data.kind === "drained" && event.data.sequence === this.playbackSequence) {
+        onDrained();
+      }
+    };
     this.setGate(this.gate);
     this.setMuted(this.muted);
     capture.port.onmessage = (event: MessageEvent<ArrayBuffer | { kind: "level"; rms: number }>) => {
@@ -110,10 +121,13 @@ export class AudioEngine {
       return;
     }
     const samples = decodePcm(base64);
-    this.playback?.port.postMessage({ kind: "audio", samples }, [samples.buffer]);
+    this.playback?.port.postMessage({ kind: "audio", samples, sequence: ++this.playbackSequence }, [samples.buffer]);
   }
   clear() {
-    this.playback?.port.postMessage({ kind: "clear" });
+    this.playback?.port.postMessage({ kind: "clear", sequence: ++this.playbackSequence });
+  }
+  finishPlayback() {
+    this.playback?.port.postMessage({ kind: "finish", sequence: this.playbackSequence });
   }
   setGate(gate: NoiseGate) {
     this.gate = gate;

vocero-frontend/src/realtime/client.test.ts

@@ -9,6 +9,7 @@ const audio = vi.hoisted(() => ({
   close: vi.fn<() => Promise<void>>(),
   play: vi.fn(),
   clear: vi.fn(),
+  finishPlayback: vi.fn(),
   setMuted: vi.fn(),
   setGate: vi.fn(),
 }));
@@ -19,6 +20,7 @@ vi.mock("../audio/engine", () => ({
     close = audio.close;
     play = audio.play;
     clear = audio.clear;
+    finishPlayback = audio.finishPlayback;
     setMuted = audio.setMuted;
     setGate = audio.setGate;
   },
@@ -89,6 +91,33 @@ afterEach(() => {
 });
 
 describe("realtime session", () => {
+  it("waits for buffered audio to drain after response.done", async () => {
+    const client = new RealtimeClient(options());
+    const socket = await connect(client);
+    socket.receive({ type: "response.created", response: { id: "r1" } });
+    socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" });
+    socket.receive({ type: "response.done", response: { id: "r1", status: "completed" } });
+    await vi.waitFor(() => {
+      expect(audio.finishPlayback).toHaveBeenCalledOnce();
+    });
+    expect(client.status).toBe("ai-speaking");
+    audio.start.mock.calls[0]?.[3]();
+    expect(client.status).toBe("listening");
+    await client.close();
+  });
+  it("does not let a playback drain override a user interruption", async () => {
+    const client = new RealtimeClient(options());
+    const socket = await connect(client);
+    socket.receive({ type: "response.created", response: { id: "r1" } });
+    socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" });
+    socket.receive({ type: "input_audio_buffer.speech_started" });
+    await vi.waitFor(() => {
+      expect(client.status).toBe("user-speaking");
+    });
+    audio.start.mock.calls[0]?.[3]();
+    expect(client.status).toBe("user-speaking");
+    await client.close();
+  });
   it("starts microphone streaming without a session.updated acknowledgment", async () => {
     const client = new RealtimeClient(options());
     const pending = client.connect();

vocero-frontend/src/realtime/client.ts

@@ -58,6 +58,7 @@ export class RealtimeClient {
   private socket: WebSocket | null = null;
   private currentStatus: ConnectionStatus = "idle";
   private configured = false;
+  private playbackPending = false;
   private closed = false;
   private closing: Promise<void> | null = null;
   private queueId = "";
@@ -149,6 +150,10 @@ export class RealtimeClient {
           this.emit("level", rms);
         },
         () => this.currentStatus === "ai-speaking",
+        () => {
+          this.playbackPending = false;
+          this.settleResponseStatus();
+        },
       );
       this.socket.addEventListener("message", (event: MessageEvent<unknown>) => {
         this.incoming = this.incoming
@@ -336,6 +341,7 @@ export class RealtimeClient {
         }
         break;
       case "input_audio_buffer.speech_started":
+        this.playbackPending = false;
         this.audio.clear();
         this.setStatus("user-speaking");
         break;
@@ -350,11 +356,12 @@ export class RealtimeClient {
           this.suppressed.add(id);
           this.suppressNext = false;
         }
-        this.setStatus("processing");
+        this.setStatus(this.playbackPending ? "ai-speaking" : "processing");
         break;
       case "response.audio.delta":
       case "response.output_audio.delta":
         if (event.delta && !this.suppressed.has(id)) {
+          this.playbackPending = true;
           this.audio.play(event.delta);
           this.audible.add(id);
           this.setStatus("ai-speaking");
@@ -422,10 +429,11 @@ export class RealtimeClient {
         if (this.activeResponse === id) {
           this.activeResponse = "";
         }
-        if (this.currentStatus === "ai-speaking" || this.currentStatus === "processing") {
+        if (!this.activeResponses.size) {
-          this.setStatus("listening");
+          this.audio.finishPlayback();
         }
         this.flush();
+        this.settleResponseStatus();
         break;
       }
       case "response.function_call_arguments.done":
@@ -449,6 +457,11 @@ export class RealtimeClient {
         break;
     }
   }
+  private settleResponseStatus() {
+    if (!this.closed && (this.currentStatus === "ai-speaking" || this.currentStatus === "processing")) {
+      this.setStatus(this.playbackPending ? "ai-speaking" : this.busy() ? "processing" : "listening");
+    }
+  }
   private busy() {
     return this.activeResponses.size > 0 || this.createInFlight;
   }
@@ -521,6 +534,7 @@ export class RealtimeClient {
     } else {
       this.suppressNext = true;
     }
+    this.playbackPending = false;
     this.audio.clear();
     this.pendingCreates = 0;
     this.deferredTexts = [];

vocero-frontend/src/realtime/controller.test.ts

@@ -247,4 +247,18 @@ describe("voice application lifecycle", () => {
     expect(client.close).toHaveBeenCalledOnce();
     expect(controller.getSnapshot().status).toBe("closed");
   });
+  it("does not count native processing or an external answer request as user inactivity", async () => {
+    const client = await start();
+    client.emit("status", "processing");
+    await vi.advanceTimersByTimeAsync(60000);
+    expect(client.close).not.toHaveBeenCalled();
+    client.emit("transcript", { id: "u1", role: "user", text: "Consulta lenta", partial: false });
+    client.emit("status", "listening");
+    await vi.advanceTimersByTimeAsync(60000);
+    expect(client.close).not.toHaveBeenCalled();
+    expect(controller.getSnapshot().status).toBe("processing");
+    pendingReasoning?.(new Response(JSON.stringify({ answer: "Respuesta" })));
+    await vi.advanceTimersByTimeAsync(1);
+    expect(controller.getSnapshot().status).toBe("listening");
+  });
 });

vocero-frontend/src/realtime/controller.ts

@@ -71,6 +71,7 @@ export class VoiceController {
   private initialization: AbortController | null = null;
   private reasoningRequest: AbortController | null = null;
   private reasoningTurn = 0;
+  private externalSpeaking = false;
   private idleTimer: ReturnType<typeof setInterval> | undefined;
   private lastActivity = Date.now();
   private lastMeter = 0;
@@ -308,7 +309,12 @@ export class VoiceController {
               ),
             });
           }
+          if (
+            !["listening", "processing", "ai-speaking"].includes(status) ||
+            (!this.reasoningRequest && !this.externalSpeaking)
+          ) {
             this.update({ status });
+          }
         }),
         client.on("transcript", (transcript) => {
           this.lastActivity = Date.now();
@@ -357,7 +363,9 @@ export class VoiceController {
       this.lastActivity = Date.now();
       this.idleTimer = setInterval(() => {
         if (
-          ["listening", "user-speaking", "processing"].includes(this.state.status) &&
+          this.state.status === "listening" &&
+          !this.reasoningRequest &&
+          !this.externalSpeaking &&
           Date.now() - this.lastActivity >= 30000
         ) {
           void this.stop();
@@ -413,6 +421,7 @@ export class VoiceController {
     this.reasoningTurn++;
     this.reasoningRequest?.abort();
     this.reasoningRequest = null;
+    this.externalSpeaking = false;
   }
   private async reason(transcript: Transcript, client: RealtimeClient) {
     const { config, profile, rag } = this.state;
@@ -423,6 +432,7 @@ export class VoiceController {
     const turn = this.reasoningTurn;
     const request = new AbortController();
     this.reasoningRequest = request;
+    this.update({ status: "processing" });
     const current = () => this.client === client && turn === this.reasoningTurn && !request.signal.aborted;
     window.speechSynthesis?.cancel();
     try {
@@ -449,6 +459,7 @@ export class VoiceController {
       client.renderExternalAnswer(answer);
       this.transcript({ role: "assistant", id: `reasoning:${transcript.id}`, text: answer, partial: false });
       if ("speechSynthesis" in window) {
+        this.externalSpeaking = true;
         const utterance = new SpeechSynthesisUtterance(answer);
         utterance.lang = "es-ES";
         const voice = window.speechSynthesis.getVoices().find((voice) => voice.lang.startsWith("es"));
@@ -462,6 +473,7 @@ export class VoiceController {
         };
         utterance.onend = utterance.onerror = () => {
           if (current()) {
+            this.externalSpeaking = false;
             this.lastActivity = Date.now();
             this.update({ status: "listening" });
           }
@@ -471,11 +483,16 @@ export class VoiceController {
       }
     } catch (error) {
       if (current()) {
+        this.externalSpeaking = false;
         this.update({ error: errorMessage(error) });
       }
     } finally {
       if (turn === this.reasoningTurn) {
         this.reasoningRequest = null;
+        if (!this.externalSpeaking && current()) {
+          this.lastActivity = Date.now();
+          this.update({ status: "listening" });
+        }
       }
     }
   }