valenti/vocero-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(frontend): track playback completion and meaningful inactivity

Browse Source
This commit is contained in:
alvaro
2026-10-07 09:27:54 -03:00
parent ff32c6a605
commit 09bf6ece00
8 changed files with 173 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vocero-frontend/src/audio/audio-playback.test.ts
+56
View File
@@ -0,0 +1,56 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
let processor: AudioWorkletProcessor;
const port = { onmessage: undefined as ((event: MessageEvent) => void) | undefined, postMessage: vi.fn() };
beforeEach(async () => {
vi.resetModules();
port.postMessage.mockClear();
vi.stubGlobal("sampleRate", 48000);
vi.stubGlobal(
"AudioWorkletProcessor",
class {
port = port;
},
);
vi.stubGlobal("registerProcessor", (_name: string, Processor: new () => AudioWorkletProcessor) => {
processor = new Processor();
});
await import("./audio-playback.worklet");
});
afterEach(() => {
vi.unstubAllGlobals();
});
function send(data: unknown) {
port.onmessage?.(new MessageEvent("message", { data }));
}
function render(blocks: number) {
let audible = false;
for (let index = 0; index < blocks; index++) {
const output = new Float32Array(128);
processor.process([], [[output]]);
audible ||= output.some((value) => value !== 0);
}
return audible;
}
it("flushes a final chunk shorter than the prebuffer and signals after playback finishes", () => {
send({ kind: "audio", samples: new Float32Array(160).fill(0.5), sequence: 1 });
expect(render(1)).toBe(false);
expect(port.postMessage).not.toHaveBeenCalled();
send({ kind: "finish", sequence: 1 });
expect(render(1)).toBe(true);
expect(port.postMessage).not.toHaveBeenCalled();
render(12);
expect(port.postMessage).toHaveBeenCalledExactlyOnceWith({ kind: "drained", sequence: 1 });
});
it("discards cleared samples and identifies the latest playback generation", () => {
send({ kind: "audio", samples: new Float32Array(16000).fill(0.5), sequence: 1 });
expect(render(1)).toBe(true);
send({ kind: "clear", sequence: 2 });
render(2);
expect(port.postMessage).toHaveBeenLastCalledWith({ kind: "drained", sequence: 2 });
expect(render(10)).toBe(false);
send({ kind: "audio", samples: new Float32Array(160).fill(0.25), sequence: 3 });
send({ kind: "finish", sequence: 3 });
expect(render(12)).toBe(true);
expect(port.postMessage).toHaveBeenLastCalledWith({ kind: "drained", sequence: 3 });
});
vocero-frontend/src/audio/audio-playback.worklet.ts
+18 -2
View File
@@ -1,5 +1,7 @@
type PlaybackMessage =
{ kind: "config"; inputRate: number } | { kind: "audio"; samples: Float32Array } | { kind: "clear" };
| { kind: "config"; inputRate: number }
| { kind: "audio"; samples: Float32Array; sequence: number }
| { kind: "clear" | "finish"; sequence: number };
class AudioPlaybackProcessor extends AudioWorkletProcessor {
private rate = 16000;
@@ -10,6 +12,8 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
private fadeIn = 0;
private fadeOut = 0;
private lastSample = 0;
private sequence = 0;
private finished = false;
constructor() {
super();
@@ -22,16 +26,24 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
}
break;
case "audio":
this.sequence = data.sequence;
this.finished = false;
if (data.samples.length) {
this.queue.push(data.samples);
}
break;
case "clear":
this.sequence = data.sequence;
this.finished = true;
this.queue = [];
this.offset = 0;
this.fraction = 0;
this.fadeOut = 32;
break;
case "finish":
this.sequence = data.sequence;
this.finished = true;
break;
}
};
}
@@ -66,7 +78,7 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
}
for (let index = 0; index < output.length; index++) {
let value = 0;
if (!this.playing && this.queue.length && this.bufferedMs() >= 90) {
if (!this.playing && this.queue.length && (this.finished || this.bufferedMs() >= 90)) {
this.playing = true;
this.fadeIn = 32;
this.fadeOut = 0;
@@ -102,6 +114,10 @@ class AudioPlaybackProcessor extends AudioWorkletProcessor {
stereo[index] = value;
}
}
if (this.finished && !this.playing && !this.queue.length) {
this.finished = false;
this.port.postMessage({ kind: "drained", sequence: this.sequence });
}
return true;
}
}
vocero-frontend/src/audio/engine.ts
+17 -3
View File
@@ -26,13 +26,19 @@ export class AudioEngine {
private closed = false;
private gate: NoiseGate;
private muted = false;
private playbackSequence = 0;
constructor(context: AudioContext, gate: NoiseGate) {
this.context = context;
this.gate = gate;
}
async start(onChunk: (chunk: ArrayBuffer) => void, onLevel: (rms: number) => void, speaking: () => boolean) {
async start(
onChunk: (chunk: ArrayBuffer) => void,
onLevel: (rms: number) => void,
speaking: () => boolean,
onDrained: () => void,
) {
const stream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
if (this.closed) {
stream.getTracks().forEach((track) => {
@@ -58,6 +64,11 @@ export class AudioEngine {
playback.port.postMessage({ kind: "config", inputRate: 16000 });
this.capture = capture;
this.playback = playback;
playback.port.onmessage = (event: MessageEvent<{ kind: string; sequence: number }>) => {
if (!this.closed && event.data.kind === "drained" && event.data.sequence === this.playbackSequence) {
onDrained();
}
};
this.setGate(this.gate);
this.setMuted(this.muted);
capture.port.onmessage = (event: MessageEvent<ArrayBuffer | { kind: "level"; rms: number }>) => {
@@ -110,10 +121,13 @@ export class AudioEngine {
return;
}
const samples = decodePcm(base64);
this.playback?.port.postMessage({ kind: "audio", samples }, [samples.buffer]);
this.playback?.port.postMessage({ kind: "audio", samples, sequence: ++this.playbackSequence }, [samples.buffer]);
}
clear() {
this.playback?.port.postMessage({ kind: "clear" });
this.playback?.port.postMessage({ kind: "clear", sequence: ++this.playbackSequence });
}
finishPlayback() {
this.playback?.port.postMessage({ kind: "finish", sequence: this.playbackSequence });
}
setGate(gate: NoiseGate) {
this.gate = gate;
vocero-frontend/src/realtime/client.test.ts
+29
View File
@@ -9,6 +9,7 @@ const audio = vi.hoisted(() => ({
close: vi.fn<() => Promise<void>>(),
play: vi.fn(),
clear: vi.fn(),
finishPlayback: vi.fn(),
setMuted: vi.fn(),
setGate: vi.fn(),
}));
@@ -19,6 +20,7 @@ vi.mock("../audio/engine", () => ({
close = audio.close;
play = audio.play;
clear = audio.clear;
finishPlayback = audio.finishPlayback;
setMuted = audio.setMuted;
setGate = audio.setGate;
},
@@ -89,6 +91,33 @@ afterEach(() => {
});
describe("realtime session", () => {
it("waits for buffered audio to drain after response.done", async () => {
const client = new RealtimeClient(options());
const socket = await connect(client);
socket.receive({ type: "response.created", response: { id: "r1" } });
socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" });
socket.receive({ type: "response.done", response: { id: "r1", status: "completed" } });
await vi.waitFor(() => {
expect(audio.finishPlayback).toHaveBeenCalledOnce();
});
expect(client.status).toBe("ai-speaking");
audio.start.mock.calls[0]?.[3]();
expect(client.status).toBe("listening");
await client.close();
});
it("does not let a playback drain override a user interruption", async () => {
const client = new RealtimeClient(options());
const socket = await connect(client);
socket.receive({ type: "response.created", response: { id: "r1" } });
socket.receive({ type: "response.audio.delta", response_id: "r1", delta: "AAAA" });
socket.receive({ type: "input_audio_buffer.speech_started" });
await vi.waitFor(() => {
expect(client.status).toBe("user-speaking");
});
audio.start.mock.calls[0]?.[3]();
expect(client.status).toBe("user-speaking");
await client.close();
});
it("starts microphone streaming without a session.updated acknowledgment", async () => {
const client = new RealtimeClient(options());
const pending = client.connect();
vocero-frontend/src/realtime/client.ts
+17 -3
View File
@@ -58,6 +58,7 @@ export class RealtimeClient {
private socket: WebSocket | null = null;
private currentStatus: ConnectionStatus = "idle";
private configured = false;
private playbackPending = false;
private closed = false;
private closing: Promise<void> | null = null;
private queueId = "";
@@ -149,6 +150,10 @@ export class RealtimeClient {
this.emit("level", rms);
},
() => this.currentStatus === "ai-speaking",
() => {
this.playbackPending = false;
this.settleResponseStatus();
},
);
this.socket.addEventListener("message", (event: MessageEvent<unknown>) => {
this.incoming = this.incoming
@@ -336,6 +341,7 @@ export class RealtimeClient {
}
break;
case "input_audio_buffer.speech_started":
this.playbackPending = false;
this.audio.clear();
this.setStatus("user-speaking");
break;
@@ -350,11 +356,12 @@ export class RealtimeClient {
this.suppressed.add(id);
this.suppressNext = false;
}
this.setStatus("processing");
this.setStatus(this.playbackPending ? "ai-speaking" : "processing");
break;
case "response.audio.delta":
case "response.output_audio.delta":
if (event.delta && !this.suppressed.has(id)) {
this.playbackPending = true;
this.audio.play(event.delta);
this.audible.add(id);
this.setStatus("ai-speaking");
@@ -422,10 +429,11 @@ export class RealtimeClient {
if (this.activeResponse === id) {
this.activeResponse = "";
}
if (this.currentStatus === "ai-speaking" || this.currentStatus === "processing") {
this.setStatus("listening");
if (!this.activeResponses.size) {
this.audio.finishPlayback();
}
this.flush();
this.settleResponseStatus();
break;
}
case "response.function_call_arguments.done":
@@ -449,6 +457,11 @@ export class RealtimeClient {
break;
}
}
private settleResponseStatus() {
if (!this.closed && (this.currentStatus === "ai-speaking" || this.currentStatus === "processing")) {
this.setStatus(this.playbackPending ? "ai-speaking" : this.busy() ? "processing" : "listening");
}
}
private busy() {
return this.activeResponses.size > 0 || this.createInFlight;
}
@@ -521,6 +534,7 @@ export class RealtimeClient {
} else {
this.suppressNext = true;
}
this.playbackPending = false;
this.audio.clear();
this.pendingCreates = 0;
this.deferredTexts = [];
vocero-frontend/src/realtime/controller.test.ts
+14
View File
@@ -247,4 +247,18 @@ describe("voice application lifecycle", () => {
expect(client.close).toHaveBeenCalledOnce();
expect(controller.getSnapshot().status).toBe("closed");
});
it("does not count native processing or an external answer request as user inactivity", async () => {
const client = await start();
client.emit("status", "processing");
await vi.advanceTimersByTimeAsync(60000);
expect(client.close).not.toHaveBeenCalled();
client.emit("transcript", { id: "u1", role: "user", text: "Consulta lenta", partial: false });
client.emit("status", "listening");
await vi.advanceTimersByTimeAsync(60000);
expect(client.close).not.toHaveBeenCalled();
expect(controller.getSnapshot().status).toBe("processing");
pendingReasoning?.(new Response(JSON.stringify({ answer: "Respuesta" })));
await vi.advanceTimersByTimeAsync(1);
expect(controller.getSnapshot().status).toBe("listening");
});
});
vocero-frontend/src/realtime/controller.ts
+19 -2
View File
@@ -71,6 +71,7 @@ export class VoiceController {
private initialization: AbortController | null = null;
private reasoningRequest: AbortController | null = null;
private reasoningTurn = 0;
private externalSpeaking = false;
private idleTimer: ReturnType<typeof setInterval> | undefined;
private lastActivity = Date.now();
private lastMeter = 0;
@@ -308,7 +309,12 @@ export class VoiceController {
),
});
}
this.update({ status });
if (
!["listening", "processing", "ai-speaking"].includes(status) ||
(!this.reasoningRequest && !this.externalSpeaking)
) {
this.update({ status });
}
}),
client.on("transcript", (transcript) => {
this.lastActivity = Date.now();
@@ -357,7 +363,9 @@ export class VoiceController {
this.lastActivity = Date.now();
this.idleTimer = setInterval(() => {
if (
["listening", "user-speaking", "processing"].includes(this.state.status) &&
this.state.status === "listening" &&
!this.reasoningRequest &&
!this.externalSpeaking &&
Date.now() - this.lastActivity >= 30000
) {
void this.stop();
@@ -413,6 +421,7 @@ export class VoiceController {
this.reasoningTurn++;
this.reasoningRequest?.abort();
this.reasoningRequest = null;
this.externalSpeaking = false;
}
private async reason(transcript: Transcript, client: RealtimeClient) {
const { config, profile, rag } = this.state;
@@ -423,6 +432,7 @@ export class VoiceController {
const turn = this.reasoningTurn;
const request = new AbortController();
this.reasoningRequest = request;
this.update({ status: "processing" });
const current = () => this.client === client && turn === this.reasoningTurn && !request.signal.aborted;
window.speechSynthesis?.cancel();
try {
@@ -449,6 +459,7 @@ export class VoiceController {
client.renderExternalAnswer(answer);
this.transcript({ role: "assistant", id: `reasoning:${transcript.id}`, text: answer, partial: false });
if ("speechSynthesis" in window) {
this.externalSpeaking = true;
const utterance = new SpeechSynthesisUtterance(answer);
utterance.lang = "es-ES";
const voice = window.speechSynthesis.getVoices().find((voice) => voice.lang.startsWith("es"));
@@ -462,6 +473,7 @@ export class VoiceController {
};
utterance.onend = utterance.onerror = () => {
if (current()) {
this.externalSpeaking = false;
this.lastActivity = Date.now();
this.update({ status: "listening" });
}
@@ -471,11 +483,16 @@ export class VoiceController {
}
} catch (error) {
if (current()) {
this.externalSpeaking = false;
this.update({ error: errorMessage(error) });
}
} finally {
if (turn === this.reasoningTurn) {
this.reasoningRequest = null;
if (!this.externalSpeaking && current()) {
this.lastActivity = Date.now();
this.update({ status: "listening" });
}
}
}
}