fix(frontend): track playback completion and meaningful inactivity

vocero-frontend/src/realtime/controller.ts

@@ -71,6 +71,7 @@ export class VoiceController {
   private initialization: AbortController | null = null;
   private reasoningRequest: AbortController | null = null;
   private reasoningTurn = 0;
+  private externalSpeaking = false;
   private idleTimer: ReturnType<typeof setInterval> | undefined;
   private lastActivity = Date.now();
   private lastMeter = 0;
@@ -308,7 +309,12 @@ export class VoiceController {
               ),
             });
           }
-          this.update({ status });
+          if (
+            !["listening", "processing", "ai-speaking"].includes(status) ||
+            (!this.reasoningRequest && !this.externalSpeaking)
+          ) {
+            this.update({ status });
+          }
         }),
         client.on("transcript", (transcript) => {
           this.lastActivity = Date.now();
@@ -357,7 +363,9 @@ export class VoiceController {
       this.lastActivity = Date.now();
       this.idleTimer = setInterval(() => {
         if (
-          ["listening", "user-speaking", "processing"].includes(this.state.status) &&
+          this.state.status === "listening" &&
+          !this.reasoningRequest &&
+          !this.externalSpeaking &&
           Date.now() - this.lastActivity >= 30000
         ) {
           void this.stop();
@@ -413,6 +421,7 @@ export class VoiceController {
     this.reasoningTurn++;
     this.reasoningRequest?.abort();
     this.reasoningRequest = null;
+    this.externalSpeaking = false;
   }
   private async reason(transcript: Transcript, client: RealtimeClient) {
     const { config, profile, rag } = this.state;
@@ -423,6 +432,7 @@ export class VoiceController {
     const turn = this.reasoningTurn;
     const request = new AbortController();
     this.reasoningRequest = request;
+    this.update({ status: "processing" });
     const current = () => this.client === client && turn === this.reasoningTurn && !request.signal.aborted;
     window.speechSynthesis?.cancel();
     try {
@@ -449,6 +459,7 @@ export class VoiceController {
       client.renderExternalAnswer(answer);
       this.transcript({ role: "assistant", id: `reasoning:${transcript.id}`, text: answer, partial: false });
       if ("speechSynthesis" in window) {
+        this.externalSpeaking = true;
         const utterance = new SpeechSynthesisUtterance(answer);
         utterance.lang = "es-ES";
         const voice = window.speechSynthesis.getVoices().find((voice) => voice.lang.startsWith("es"));
@@ -462,6 +473,7 @@ export class VoiceController {
         };
         utterance.onend = utterance.onerror = () => {
           if (current()) {
+            this.externalSpeaking = false;
             this.lastActivity = Date.now();
             this.update({ status: "listening" });
           }
@@ -471,11 +483,16 @@ export class VoiceController {
       }
     } catch (error) {
       if (current()) {
+        this.externalSpeaking = false;
         this.update({ error: errorMessage(error) });
       }
     } finally {
       if (turn === this.reasoningTurn) {
         this.reasoningRequest = null;
+        if (!this.externalSpeaking && current()) {
+          this.lastActivity = Date.now();
+          this.update({ status: "listening" });
+        }
       }
     }
   }