From 62c2692082ca8553b8e67b7b8b416d13fe4022f1 Mon Sep 17 00:00:00 2001 From: Alvaro Parker Date: Wed, 7 Oct 2026 10:46:15 -0300 Subject: [PATCH] fix(frontend): start conversations without a welcome modal --- docs/client/troubleshooting.md | 9 +- docs/client/usage.md | 13 +-- vocero-api/CHANGELOG.md | 4 + vocero-frontend/src/App.test.tsx | 48 +++++++++++ vocero-frontend/src/App.tsx | 44 +--------- vocero-frontend/src/components/VoiceStage.tsx | 10 ++- .../src/realtime/controller.test.ts | 85 ++++++++++++++++++- vocero-frontend/src/realtime/controller.ts | 18 +--- 8 files changed, 162 insertions(+), 69 deletions(-) diff --git a/docs/client/troubleshooting.md b/docs/client/troubleshooting.md index aa46a92..56e53b8 100644 --- a/docs/client/troubleshooting.md +++ b/docs/client/troubleshooting.md @@ -113,10 +113,11 @@ before changing audio timing based on those observations. In `OPENAI`, check the configured endpoint/model/key and provider logs; HTTP reasoning does not use the native session's tool array. -For `TALK`/`TALKFIRST`, the two-second warmup does not override browser -microphone/audio permissions. If automatic startup fails, use the unlocker -gesture and inspect the permission error rather than increasing the timer -without evidence. +`TALK`/`TALKFIRST` attempt startup as soon as configuration and caller context +are ready. Browser microphone/audio policies can still require a gesture. If +audio is blocked, tap the main microphone button when the inline hint appears. +If microphone access fails, allow it in the browser's site permissions and use +the same button to retry. No startup dialog is shown. ## Local validation diff --git a/docs/client/usage.md b/docs/client/usage.md index b7095ad..9ff2b10 100644 --- a/docs/client/usage.md +++ b/docs/client/usage.md @@ -148,12 +148,15 @@ loading window before the interface allows a generic session. | Value | Behavior | |---|---| | Empty/default | User taps to start | -| `TALK` | After a 2-second warmup, show a dismissible startup dialog; a user gesture starts the session and greeting | -| `TALKFIRST` | After a 2-second warmup, attempt automatic start and inject the Spanish wake phrase; microphone failure falls back to the startup dialog | +| `TALK` | Attempt automatic start and inject the Spanish wake phrase as soon as configuration and caller context are ready | +| `TALKFIRST` | Same automatic startup behavior as `TALK` | -The warmup starts after configuration and caller-context loading settle; -optional reports and tools can still be loading. Browser permission and audio -policies can still require a gesture. +Automatic startup runs once per page load, without a startup dialog or artificial +delay; optional reports and tools can still be loading. Browser permission and +audio policies can still require a gesture. When audio is blocked, an inline hint +asks the user to tap the microphone button. Permission or connection failures +show the error and leave the same button available to retry. Stopping a session +does not start it again automatically. The greeting uses the same `hola vocero` input as wake-word handling. On browsers with speech recognition, **Activar con «hola vocero»** opts into diff --git a/vocero-api/CHANGELOG.md b/vocero-api/CHANGELOG.md index 5a6124a..46a6526 100644 --- a/vocero-api/CHANGELOG.md +++ b/vocero-api/CHANGELOG.md @@ -31,6 +31,10 @@ the implementation at the recorded date; current operating instructions live in ### Changed +- Start `TALK` and `TALKFIRST` conversations as soon as configuration and caller + context are ready. Remove the startup dialog and two-second delay; use an + inline hint and the existing microphone button when a browser requires a tap. + - Replace the frontend legacy stylesheet with Tailwind v4 semantic theme tokens, shared form/dialog controls and component-owned layout utilities. Preserve both configured themes, responsive layouts, sanitized Markdown and audio-driven diff --git a/vocero-frontend/src/App.test.tsx b/vocero-frontend/src/App.test.tsx index 68df72c..e367e52 100644 --- a/vocero-frontend/src/App.test.tsx +++ b/vocero-frontend/src/App.test.tsx @@ -4,6 +4,7 @@ import { StrictMode } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import App from "./App"; +import { VoiceController } from "./realtime/controller"; const bootstrap = { theme: "white", s2sUrl: "ws://localhost:8420/v1/realtime", apipilot: true }; beforeEach(() => { @@ -43,6 +44,53 @@ afterEach(() => { }); describe("React application", () => { + it.each(["TALK", "TALKFIRST"])( + "attempts %s immediately without a startup dialog under StrictMode", + async (startSessionMode) => { + const original = vi.mocked(fetch).getMockImplementation(); + if (!original) { + throw new Error("Missing fetch mock"); + } + vi.mocked(fetch).mockImplementation((input, init) => + input === "/api/config" + ? Promise.resolve(new Response(JSON.stringify({ ...bootstrap, startSessionMode }))) + : original(input, init), + ); + const start = vi.spyOn(VoiceController.prototype, "start").mockResolvedValue(undefined); + render( + + + , + ); + await waitFor(() => { + expect(start).toHaveBeenCalledExactlyOnceWith(true); + }); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }, + ); + it.each(["white", "dark"])( + "shows an inline microphone fallback in the %s theme when autoplay is blocked", + async (theme) => { + vi.mocked(fetch).mockResolvedValueOnce( + new Response(JSON.stringify({ ...bootstrap, theme, startSessionMode: "TALK" })), + ); + const getUserMedia = vi.fn(); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + vi.stubGlobal( + "AudioContext", + class { + state = "suspended"; + resume = vi.fn().mockResolvedValue(undefined); + close = vi.fn().mockResolvedValue(undefined); + }, + ); + render(); + expect(await screen.findByText("Toca el micrófono para empezar a conversar.")).toHaveAttribute("role", "status"); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Iniciar conversación de voz" })).toBeEnabled(); + expect(getUserMedia).not.toHaveBeenCalled(); + }, + ); it("offers configuration retry without exposing a broken start control", async () => { vi.mocked(fetch).mockRejectedValueOnce(new Error("offline")); const user = userEvent.setup(); diff --git a/vocero-frontend/src/App.tsx b/vocero-frontend/src/App.tsx index 963d9db..a514363 100644 --- a/vocero-frontend/src/App.tsx +++ b/vocero-frontend/src/App.tsx @@ -3,12 +3,11 @@ import { useEffect, useLayoutEffect, useState, useSyncExternalStore } from "reac import { Bubbles } from "./components/Bubbles"; import { Conversation } from "./components/Conversation"; -import { Dialog } from "./components/Dialog"; import { ReportsPanel } from "./components/ReportsPanel"; import { AboutDialog, ProfileDialog, RagDialog, SettingsDialog, ToolsDialog } from "./components/SettingsDialogs"; import { VoiceStage } from "./components/VoiceStage"; import { WakeWordControl } from "./components/WakeWordControl"; -import { Button, DialogFooter, DialogIntro, IconButton } from "./components/ui"; +import { Button, IconButton } from "./components/ui"; import { useMediaQuery } from "./hooks/use-media-query"; import { initials, profileName } from "./lib/context"; import { isDebug } from "./lib/settings"; @@ -45,15 +44,9 @@ export default function App() { } }, [state.loading, white]); useEffect(() => { - if (!ready) { - return; - } - const timer = setTimeout(() => { + if (ready) { controller.offerAutoStart(); - }, 2000); - return () => { - clearTimeout(timer); - }; + } }, [ready, controller]); const close = () => { setModal(null); @@ -88,35 +81,6 @@ export default function App() { } return ( <> - {state.unlock && ( - { - controller.dismissUnlock(); - }} - > - - Activa el micrófono para hablar con Vocero. Puedes detener la conversación en cualquier momento. - - - - - - - )}
@@ -230,7 +194,7 @@ export default function App() {
)} - {ready && !ACTIVE_STATES.has(state.status) && !modal && !state.unlock && !(reportsOpen && !desktop) && ( + {ready && !ACTIVE_STATES.has(state.status) && !modal && !(reportsOpen && !desktop) && ( )} {state.notice && ( -

{state.notice}

+

+ {state.notice} +

)} {active && (
{caption}

- {state.notice &&

{state.notice}

} + {state.notice && ( +

+ {state.notice} +

+ )} )} {(status === "queued" || status === "your-turn") && ( diff --git a/vocero-frontend/src/realtime/controller.test.ts b/vocero-frontend/src/realtime/controller.test.ts index da557a5..fb24a77 100644 --- a/vocero-frontend/src/realtime/controller.test.ts +++ b/vocero-frontend/src/realtime/controller.test.ts @@ -3,8 +3,9 @@ import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vite import type { ClientOptions } from "./client"; import { VoiceController } from "./controller"; -const { MockClient, closeContext } = vi.hoisted(() => { +const { MockClient, closeContext, createContext } = vi.hoisted(() => { const closeContext = vi.fn().mockResolvedValue(undefined); + const createContext = vi.fn(() => ({ state: "running", close: closeContext })); class MockClient { static instances: MockClient[] = []; readonly options: ClientOptions; @@ -41,16 +42,17 @@ const { MockClient, closeContext } = vi.hoisted(() => { sendToolOutput = vi.fn(); join = vi.fn(); } - return { MockClient, closeContext }; + return { MockClient, closeContext, createContext }; }); vi.mock("./client", async (importOriginal) => ({ ...(await importOriginal()), RealtimeClient: MockClient })); -vi.mock("../audio/engine", () => ({ createAudioContext: () => ({ state: "running", close: closeContext }) })); +vi.mock("../audio/engine", () => ({ createAudioContext: createContext })); let pendingReasoning: ((value: Response) => void) | undefined; let controller: VoiceController; let fetchMock: Mock<(path: string, init?: RequestInit) => Promise>; beforeEach(() => { vi.useFakeTimers(); vi.clearAllMocks(); + createContext.mockReturnValue({ state: "running", close: closeContext }); MockClient.instances = []; pendingReasoning = undefined; vi.stubGlobal("AudioContext", vi.fn()); @@ -87,6 +89,83 @@ async function start() { return client; } describe("voice application lifecycle", () => { + it.each(["TALK", "TALKFIRST"])( + "automatically starts %s once with the greeting after initialization", + async (startSessionMode) => { + fetchMock.mockResolvedValueOnce( + new Response(JSON.stringify({ startSessionMode, s2sUrl: "ws://localhost/v1/realtime" })), + ); + controller.offerAutoStart(); + expect(MockClient.instances).toHaveLength(0); + await controller.initialize(); + controller.offerAutoStart(); + controller.offerAutoStart(); + expect(MockClient.instances).toHaveLength(1); + expect(MockClient.instances[0]?.options.greeting).toBe("hola vocero"); + expect(controller.getSnapshot().status).toBe("listening"); + await controller.stop(); + controller.offerAutoStart(); + expect(MockClient.instances).toHaveLength(1); + expect(controller.getSnapshot().status).toBe("closed"); + }, + ); + it.each(["", "UNSUPPORTED"])("leaves startup manual for mode %s", async (startSessionMode) => { + fetchMock.mockResolvedValueOnce( + new Response(JSON.stringify({ startSessionMode, s2sUrl: "ws://localhost/v1/realtime" })), + ); + await controller.initialize(); + controller.offerAutoStart(); + expect(createContext).not.toHaveBeenCalled(); + expect(MockClient.instances).toHaveLength(0); + await controller.start(); + expect(MockClient.instances).toHaveLength(1); + expect(MockClient.instances[0]?.options.greeting).toBeUndefined(); + }); + it("waits for caller context before the automatic attempt", async () => { + fetchMock.mockResolvedValueOnce( + new Response(JSON.stringify({ startSessionMode: "TALK", s2sUrl: "ws://localhost/v1/realtime" })), + ); + let finishProfile: ((value: Response) => void) | undefined; + fetchMock.mockImplementationOnce( + () => + new Promise((resolve) => { + finishProfile = resolve; + }), + ); + const initializing = controller.initialize(); + await vi.advanceTimersByTimeAsync(1); + expect(controller.getSnapshot()).toMatchObject({ configReady: true, contextLoading: true }); + controller.offerAutoStart(); + expect(MockClient.instances).toHaveLength(0); + finishProfile?.(new Response(JSON.stringify({ result: { name: "Ana" } }))); + await initializing; + controller.offerAutoStart(); + expect(MockClient.instances).toHaveLength(1); + }); + it.each(["TALK", "TALKFIRST"])( + "leaves a manual fallback when the browser blocks %s audio", + async (startSessionMode) => { + fetchMock.mockResolvedValueOnce( + new Response(JSON.stringify({ startSessionMode, s2sUrl: "ws://localhost/v1/realtime" })), + ); + createContext.mockReturnValueOnce({ state: "suspended", close: closeContext }); + await controller.initialize(); + controller.offerAutoStart(); + expect(closeContext).toHaveBeenCalledOnce(); + expect(MockClient.instances).toHaveLength(0); + expect(controller.getSnapshot()).toMatchObject({ + status: "idle", + error: "", + notice: "Toca el micrófono para empezar a conversar.", + }); + controller.offerAutoStart(); + expect(createContext).toHaveBeenCalledOnce(); + await controller.start(); + expect(MockClient.instances).toHaveLength(1); + expect(MockClient.instances[0]?.options.greeting).toBe("hola vocero"); + expect(controller.getSnapshot()).toMatchObject({ status: "listening", notice: "" }); + }, + ); it("notifies only meter subscribers when microphone level changes", async () => { const client = await start(); const app = vi.fn(); diff --git a/vocero-frontend/src/realtime/controller.ts b/vocero-frontend/src/realtime/controller.ts index f3ce2a5..c4cf9d7 100644 --- a/vocero-frontend/src/realtime/controller.ts +++ b/vocero-frontend/src/realtime/controller.ts @@ -56,7 +56,6 @@ export interface AppState { rag: string; queuePosition: number; joinDeadline: number; - unlock: boolean; wakeWordEnabled: boolean; } @@ -103,7 +102,6 @@ export class VoiceController { rag: "", queuePosition: 0, joinDeadline: 0, - unlock: false, wakeWordEnabled: false, }; } @@ -263,9 +261,6 @@ export class VoiceController { dismissError() { this.update({ error: "" }); } - dismissUnlock() { - this.update({ unlock: false }); - } setWakeWordEnabled(wakeWordEnabled: boolean) { this.update({ wakeWordEnabled }); } @@ -275,9 +270,7 @@ export class VoiceController { } this.autoStarted = true; const mode = this.state.config.startSessionMode.trim().toUpperCase(); - if (mode === "TALK") { - this.update({ unlock: true }); - } else if (mode === "TALKFIRST") { + if (mode === "TALK" || mode === "TALKFIRST") { void this.start(true); } } @@ -289,7 +282,6 @@ export class VoiceController { return; } this.update({ - unlock: false, error: "", notice: "", status: "connecting", @@ -304,8 +296,8 @@ export class VoiceController { const target = connectionTarget(this.state.config, this.state.settings, this.caller); context = preparedContext ?? createAudioContext(); if (automatic && context.state !== "running") { - await context.close(); - this.update({ status: "idle", unlock: true }); + void context.close().catch(() => undefined); + this.update({ status: "idle", notice: "Toca el micrófono para empezar a conversar." }); return; } this.requests = new AbortController(); @@ -417,9 +409,6 @@ export class VoiceController { // A closed event may already have cleared the client; explicit cancellation is silent. if (!(error instanceof SessionError && error.code === "aborted") && (!this.client || this.client === client)) { this.reportError(error); - if (automatic) { - this.update({ unlock: true }); - } } } } catch (error) { @@ -619,7 +608,6 @@ export class VoiceController { wakeWordEnabled: false, muted: false, joinDeadline: 0, - unlock: false, messages: this.state.messages.map((message) => message.partial ? { ...message, partial: false, interrupted: message.role === "assistant" } : message, ),