feat(frontend): port realtime audio and session handling to TypeScript

frontend-s2s/src/audio/audio-playback.worklet.ts

@@ -0,0 +1,108 @@
+type PlaybackMessage =
+  { kind: "config"; inputRate: number } | { kind: "audio"; samples: Float32Array } | { kind: "clear" };
+
+class AudioPlaybackProcessor extends AudioWorkletProcessor {
+  private rate = 16000;
+  private queue: Float32Array[] = [];
+  private offset = 0;
+  private fraction = 0;
+  private playing = false;
+  private fadeIn = 0;
+  private fadeOut = 0;
+  private lastSample = 0;
+
+  constructor() {
+    super();
+    this.port.onmessage = (event: MessageEvent<PlaybackMessage>) => {
+      const data = event.data;
+      switch (data.kind) {
+        case "config":
+          if (data.inputRate > 0) {
+            this.rate = data.inputRate;
+          }
+          break;
+        case "audio":
+          if (data.samples.length) {
+            this.queue.push(data.samples);
+          }
+          break;
+        case "clear":
+          this.queue = [];
+          this.offset = 0;
+          this.fraction = 0;
+          this.fadeOut = 32;
+          break;
+      }
+    };
+  }
+  private bufferedMs() {
+    return (this.queue.reduce((sum, buffer) => sum + buffer.length, -this.offset) / this.rate) * 1000;
+  }
+  private read(): number | null {
+    const head = this.queue[0];
+    if (!head) {
+      return null;
+    }
+    const a = head[this.offset] ?? 0;
+    const b = head[this.offset + 1] ?? this.queue[1]?.[0] ?? a;
+    return a + (b - a) * this.fraction;
+  }
+  private advance() {
+    this.fraction += this.rate / sampleRate;
+    while (this.fraction >= 1) {
+      this.fraction--;
+      this.offset++;
+    }
+    while (this.queue[0] && this.offset >= this.queue[0].length) {
+      this.offset -= this.queue[0].length;
+      this.queue.shift();
+    }
+  }
+  process(_inputs: Float32Array[][], outputs: Float32Array[][]) {
+    const channels = outputs[0];
+    const output = channels?.[0];
+    if (!output) {
+      return true;
+    }
+    for (let index = 0; index < output.length; index++) {
+      let value = 0;
+      if (!this.playing && this.queue.length && this.bufferedMs() >= 90) {
+        this.playing = true;
+        this.fadeIn = 32;
+        this.fadeOut = 0;
+      }
+      if (this.playing) {
+        const sample = this.read();
+        if (sample === null) {
+          value = this.lastSample * (1 - 1 / 32);
+          this.lastSample = value;
+          if (Math.abs(value) < 1e-4) {
+            this.playing = false;
+            this.lastSample = 0;
+          }
+        } else {
+          value = sample;
+          this.lastSample = sample;
+          this.advance();
+        }
+        if (this.fadeIn > 0) {
+          value *= 1 - this.fadeIn-- / 32;
+        }
+        if (this.fadeOut > 0) {
+          value *= this.fadeOut-- / 32;
+          if (!this.fadeOut) {
+            this.playing = false;
+            this.lastSample = 0;
+          }
+        }
+      }
+      output[index] = value;
+      const stereo = channels?.[1];
+      if (stereo) {
+        stereo[index] = value;
+      }
+    }
+    return true;
+  }
+}
+registerProcessor("audio-playback", AudioPlaybackProcessor);

frontend-s2s/src/audio/codec.ts

@@ -0,0 +1,23 @@
+export function encodePcm(buffer: ArrayBuffer) {
+  const bytes = new Uint8Array(buffer);
+  let binary = "";
+  for (let index = 0; index < bytes.length; index += 0x8000) {
+    binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000));
+  }
+  return btoa(binary);
+}
+export function decodePcm(base64: string) {
+  const binary = atob(base64);
+  const buffer = new ArrayBuffer(binary.length);
+  const bytes = new Uint8Array(buffer);
+  for (let index = 0; index < binary.length; index++) {
+    bytes[index] = binary.charCodeAt(index);
+  }
+  const view = new DataView(buffer);
+  const samples = new Float32Array(Math.floor(binary.length / 2));
+  for (let index = 0; index < samples.length; index++) {
+    const value = view.getInt16(index * 2, true);
+    samples[index] = value < 0 ? value / 0x8000 : value / 0x7fff;
+  }
+  return samples;
+}

frontend-s2s/src/audio/engine.ts

@@ -0,0 +1,147 @@
+import type { NoiseGate } from "../lib/contracts";
+import playbackUrl from "./audio-playback.worklet.ts?worker&url";
+import { decodePcm } from "./codec";
+import captureUrl from "./mic-capture.worklet.ts?worker&url";
+
+const BANDS = [2, 5, 9, 16, 28, 52];
+export const MIC_CONSTRAINTS: MediaStreamConstraints = {
+  audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
+  video: false,
+};
+
+/** Construct inside the click handler, before awaiting microphone permission. */
+export function createAudioContext() {
+  const context = new AudioContext({ latencyHint: "interactive" });
+  void context.resume().catch(() => undefined);
+  return context;
+}
+
+export class AudioEngine {
+  private readonly context: AudioContext;
+  private stream: MediaStream | null = null;
+  private capture: AudioWorkletNode | null = null;
+  private playback: AudioWorkletNode | null = null;
+  private nodes: AudioNode[] = [];
+  private frame = 0;
+  private closed = false;
+  private gate: NoiseGate;
+  private muted = false;
+
+  constructor(context: AudioContext, gate: NoiseGate) {
+    this.context = context;
+    this.gate = gate;
+  }
+
+  async start(onChunk: (chunk: ArrayBuffer) => void, onLevel: (rms: number) => void, speaking: () => boolean) {
+    const stream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
+    if (this.closed) {
+      stream.getTracks().forEach((track) => {
+        track.stop();
+      });
+      return;
+    }
+    this.stream = stream;
+    await Promise.all([
+      this.context.audioWorklet.addModule(captureUrl),
+      this.context.audioWorklet.addModule(playbackUrl),
+    ]);
+    if (this.closed) {
+      return;
+    }
+    const capture = new AudioWorkletNode(this.context, "mic-capture", { numberOfInputs: 1, numberOfOutputs: 0 });
+    const playback = new AudioWorkletNode(this.context, "audio-playback", {
+      numberOfInputs: 0,
+      numberOfOutputs: 1,
+      outputChannelCount: [1],
+    });
+    // The checked-in speech pipeline emits native 16 kHz PCM, despite older 24 kHz comments.
+    playback.port.postMessage({ kind: "config", inputRate: 16000 });
+    this.capture = capture;
+    this.playback = playback;
+    this.setGate(this.gate);
+    this.setMuted(this.muted);
+    capture.port.onmessage = (event: MessageEvent<ArrayBuffer | { kind: "level"; rms: number }>) => {
+      if (this.closed) {
+        return;
+      }
+      if (event.data instanceof ArrayBuffer) {
+        onChunk(event.data);
+      } else {
+        onLevel(event.data.rms);
+      }
+    };
+    const input = this.context.createMediaStreamSource(stream);
+    const micAnalyser = this.context.createAnalyser();
+    const outAnalyser = this.context.createAnalyser();
+    micAnalyser.fftSize = outAnalyser.fftSize = 256;
+    micAnalyser.smoothingTimeConstant = 0;
+    outAnalyser.smoothingTimeConstant = 0.3;
+    input.connect(capture);
+    input.connect(micAnalyser);
+    playback.connect(outAnalyser);
+    outAnalyser.connect(this.context.destination);
+    this.nodes = [input, capture, playback, micAnalyser, outAnalyser];
+    const buffer = new Uint8Array(micAnalyser.frequencyBinCount);
+    const levels = new Float32Array(5);
+    const tick = () => {
+      const source = speaking() ? outAnalyser : micAnalyser;
+      source.getByteFrequencyData(buffer);
+      for (let band = 0; band < 5; band++) {
+        const lo = BANDS[band] ?? 0;
+        const hi = BANDS[band + 1] ?? 52;
+        let sum = 0;
+        for (let index = lo; index < hi; index++) {
+          sum += buffer[index] ?? 0;
+        }
+        const target = sum / ((hi - lo) * 255);
+        const previous = levels[band] ?? 0;
+        levels[band] = previous + (target - previous) * (target > previous ? 0.6 : 0.18);
+        document.documentElement.style.setProperty(`--bar${band}`, String(levels[band]));
+      }
+      outAnalyser.getByteFrequencyData(buffer);
+      document.documentElement.style.setProperty("--ai-audio-level", String(Math.max(...buffer.subarray(0, 52)) / 255));
+      this.frame = requestAnimationFrame(tick);
+    };
+    this.frame = requestAnimationFrame(tick);
+  }
+  play(base64: string) {
+    if (!base64 || this.closed) {
+      return;
+    }
+    const samples = decodePcm(base64);
+    this.playback?.port.postMessage({ kind: "audio", samples }, [samples.buffer]);
+  }
+  clear() {
+    this.playback?.port.postMessage({ kind: "clear" });
+  }
+  setGate(gate: NoiseGate) {
+    this.gate = gate;
+    this.capture?.port.postMessage({ kind: "gate", ...gate });
+  }
+  setMuted(muted: boolean) {
+    this.muted = muted;
+    this.stream?.getAudioTracks().forEach((track) => {
+      track.enabled = !muted;
+    });
+    this.capture?.port.postMessage({ kind: "enable", value: !muted });
+  }
+  async close() {
+    this.closed = true;
+    cancelAnimationFrame(this.frame);
+    this.stream?.getTracks().forEach((track) => {
+      track.stop();
+    });
+    this.capture?.port.close();
+    this.playback?.port.close();
+    this.nodes.forEach((node) => {
+      node.disconnect();
+    });
+    if (this.context.state !== "closed") {
+      await this.context.close();
+    }
+    for (let index = 0; index < 5; index++) {
+      document.documentElement.style.removeProperty(`--bar${index}`);
+    }
+    document.documentElement.style.removeProperty("--ai-audio-level");
+  }
+}

frontend-s2s/src/audio/mic-capture.worklet.ts

@@ -0,0 +1,80 @@
+const TARGET_RATE = 16000;
+type Control = { kind: "enable"; value: boolean } | { kind: "gate"; enabled: boolean; thresholdDb: number };
+
+/** Preserve the existing 40 ms PCM16 frames and smoothed gate envelope. */
+class MicCaptureProcessor extends AudioWorkletProcessor {
+  private readonly ratio = sampleRate / TARGET_RATE;
+  private readonly chunkSize = 640;
+  private scratch = new Float32Array(0);
+  private readonly decimated = new Float32Array(this.chunkSize);
+  private enabled = true;
+  private gateEnabled = false;
+  private threshold = 0;
+  private gain = 1;
+  private hold = 0;
+  private readonly attack = Math.exp(-1 / (0.005 * TARGET_RATE));
+  private readonly release = Math.exp(-1 / (0.08 * TARGET_RATE));
+
+  constructor() {
+    super();
+    this.port.onmessage = (event: MessageEvent<Control>) => {
+      if (event.data.kind === "enable") {
+        this.enabled = event.data.value;
+      } else {
+        this.gateEnabled = event.data.enabled;
+        this.threshold = event.data.enabled ? 10 ** (event.data.thresholdDb / 20) : 0;
+      }
+    };
+  }
+
+  process(inputs: Float32Array[][]) {
+    const incoming = inputs[0]?.[0];
+    if (!incoming?.length) {
+      return true;
+    }
+    const next = new Float32Array(this.scratch.length + incoming.length);
+    next.set(this.scratch);
+    next.set(incoming, this.scratch.length);
+    this.scratch = next;
+    const needed = Math.ceil(this.chunkSize * this.ratio);
+    while (this.scratch.length >= needed) {
+      let sum = 0;
+      for (let index = 0; index < this.chunkSize; index++) {
+        const position = index * this.ratio;
+        const start = Math.floor(position);
+        const a = this.scratch[start] ?? 0;
+        const value =
+          Math.abs(this.ratio - 3) < 1e-6
+            ? (a + (this.scratch[start + 1] ?? 0) + (this.scratch[start + 2] ?? 0)) / 3
+            : a + ((this.scratch[start + 1] ?? a) - a) * (position - start);
+        this.decimated[index] = value;
+        sum += value * value;
+      }
+      const rms = Math.sqrt(sum / this.chunkSize);
+      let target = 1;
+      if (this.gateEnabled) {
+        if (rms >= this.threshold) {
+          this.hold = TARGET_RATE * 0.25;
+        } else if (this.hold > 0) {
+          this.hold -= this.chunkSize;
+        } else {
+          target = 0;
+        }
+      }
+      const output = new Int16Array(this.chunkSize);
+      for (let index = 0; index < this.chunkSize; index++) {
+        const coefficient = target > this.gain ? this.attack : this.release;
+        this.gain = target + (this.gain - target) * coefficient;
+        const value = Math.max(-1, Math.min(1, (this.decimated[index] ?? 0) * this.gain));
+        output[index] = value < 0 ? value * 0x8000 : value * 0x7fff;
+      }
+      this.scratch = this.scratch.slice(Math.floor(this.chunkSize * this.ratio));
+      this.port.postMessage({ kind: "level", rms });
+      if (this.enabled) {
+        this.port.postMessage(output.buffer, [output.buffer]);
+      }
+    }
+    return true;
+  }
+}
+registerProcessor("mic-capture", MicCaptureProcessor);

frontend-s2s/src/audio/worklet-globals.d.ts

@@ -0,0 +1,13 @@
+export {};
+
+declare global {
+  const sampleRate: number;
+  abstract class AudioWorkletProcessor {
+    readonly port: MessagePort;
+    abstract process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean;
+  }
+  function registerProcessor(
+    name: string,
+    processor: new (options?: AudioWorkletNodeOptions) => AudioWorkletProcessor,
+  ): void;
+}

frontend-s2s/src/lib/api.ts

@@ -0,0 +1,110 @@
+import { z } from "zod";
+
+import { accountSchema, configSchema, recordSchema, reportsSchema, toolsEnvelopeSchema } from "./contracts";
+
+export class ApiError extends Error {
+  readonly status: number;
+  readonly body: unknown;
+
+  constructor(status: number, body: unknown) {
+    const data = recordSchema.safeParse(body);
+    super(data.success && typeof data.data.detail === "string" ? data.data.detail : `Error del servidor (${status}).`);
+    this.name = "ApiError";
+    this.status = status;
+    this.body = body;
+  }
+}
+
+export async function request<T>(path: string, schema: z.ZodType<T>, init?: RequestInit): Promise<T> {
+  const response = await fetch(path, init);
+  const body: unknown = await response.json().catch(() => null);
+  if (!response.ok) {
+    throw new ApiError(response.status, body);
+  }
+  return schema.parse(body);
+}
+
+export function post<T>(path: string, schema: z.ZodType<T>, body: unknown, signal?: AbortSignal) {
+  return request(path, schema, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal,
+  });
+}
+
+export async function optionalRequest<T>(path: string, schema: z.ZodType<T>, signal: AbortSignal) {
+  try {
+    return await request(path, schema, { signal });
+  } catch (error) {
+    if (signal.aborted) {
+      throw error;
+    }
+    // Profile, reports, and account features are optional in generic deployments.
+    return null;
+  }
+}
+
+export function getConfig(signal: AbortSignal) {
+  return request("/api/config", configSchema, { signal });
+}
+export function getAccount(signal: AbortSignal) {
+  return optionalRequest("/api/me", accountSchema, signal);
+}
+export async function getProfile(phone: string, signal: AbortSignal) {
+  if (!phone) {
+    return null;
+  }
+  const data = await optionalRequest(
+    `/api/user/phone?${new URLSearchParams({ phoneNumber: phone })}`,
+    recordSchema,
+    signal,
+  );
+  if (!data) {
+    return null;
+  }
+  const result = recordSchema.safeParse(data.result);
+  return result.success ? result.data : data;
+}
+export function getReports(phone: string, signal: AbortSignal) {
+  const query = phone ? `?${new URLSearchParams({ phoneNumber: phone })}` : "";
+  return optionalRequest(`/api/user/reports${query}`, reportsSchema, signal);
+}
+export async function getTools(signal: AbortSignal, refresh = false) {
+  const result = await request(
+    `/api/apipilot/tools?include_unauthorized=true&refresh=${String(refresh)}`,
+    toolsEnvelopeSchema,
+    { signal },
+  );
+  if (!result.success || !result.data) {
+    throw new Error("No se pudieron cargar las herramientas.");
+  }
+  return result.data.tools;
+}
+
+export const reasoningSchema = z.object({ answer: z.string().default(""), log_only: z.boolean().default(false) });
+export const searchSchema = z.object({
+  answer: z.string().nullish(),
+  results: z.array(z.object({ title: z.string(), snippet: z.string().default(""), url: z.string() })).default([]),
+});
+
+/** Teardown must survive pagehide and must not hold up a new session. */
+export function endReservation(path: "/api/session/end" | "/api/queue/end", body: Record<string, string>) {
+  const payload = JSON.stringify(body);
+  if (
+    typeof navigator.sendBeacon === "function" &&
+    navigator.sendBeacon(path, new Blob([payload], { type: "application/json" }))
+  ) {
+    return;
+  }
+  void fetch(path, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: payload,
+    keepalive: true,
+  }).catch(() => undefined);
+}
+
+export function errorMessage(error: unknown) {
+  return error instanceof Error ? error.message : "No se pudo completar la operación.";
+}

frontend-s2s/src/lib/context.ts

@@ -0,0 +1,90 @@
+import { recordSchema, type DynamicTool, type FunctionTool, type Profile, type ServerConfig } from "./contracts";
+import type { Settings } from "./settings";
+
+export function profileName(profile: Profile | null): string {
+  if (!profile) {
+    return "";
+  }
+  for (const key of ["name", "fullName", "full_name", "displayName", "display_name", "username"]) {
+    const value = profile[key];
+    if (typeof value === "string" && value.trim()) {
+      return value.trim();
+    }
+  }
+  const full = [profile.firstName, profile.lastName]
+    .filter((part): part is string => typeof part === "string")
+    .join(" ")
+    .trim();
+  if (full) {
+    return full;
+  }
+  for (const key of ["user", "customer", "profile", "contact"]) {
+    const nested = recordSchema.safeParse(profile[key]);
+    if (nested.success) {
+      const name = profileName(nested.data);
+      if (name) {
+        return name;
+      }
+    }
+  }
+  return "";
+}
+export function initials(name: string) {
+  const words = name.trim().split(/\s+/);
+  return (words.length > 1 ? `${words[0]?.[0] ?? ""}${words.at(-1)?.[0] ?? ""}` : name.slice(0, 2)).toUpperCase();
+}
+export function profileContext(profile: Profile | null) {
+  if (!profile) {
+    return "";
+  }
+  const serialized = JSON.stringify(profile);
+  return `Caller profile resolved from the page phoneNumber: ${serialized.length > 1600 ? `${serialized.slice(0, 1600)}...` : serialized}. Use this as trusted caller context when relevant.`;
+}
+export function ragContext(text: string) {
+  return text.trim() ? `<<<CUSTOM_RAG_BEGIN>>>\n${text.trim()}\n<<<CUSTOM_RAG_END>>>` : "";
+}
+export const TOOL_MARKER = "<<<APIPILOT_TOOLS_LIST_BEGIN>>>";
+export function toolContext(tools: DynamicTool[]) {
+  if (!tools.length) {
+    return "";
+  }
+  const entries = tools.slice(0, 100).map((tool) => {
+    const required = Array.isArray(tool.parameters.required)
+      ? tool.parameters.required.filter((key): key is string => typeof key === "string")
+      : [];
+    return `- name: ${tool.name}\n  requires_auth: ${String(tool.requires_auth)}${required.length ? `\n  required_parameters: [${required.slice(0, 40).join(", ")}]` : ""}\n  description: ${tool.description.slice(0, 500)}`;
+  });
+  return `${TOOL_MARKER}\n### Available dynamic tools (ApiPilot AquaPortal)\nEach tool name is a unique identifier: DO NOT change, camelCase or translate it.\nMANDATORY EXECUTION RULES:\n1. If the user request matches a tool description, output ONLY one line as the FIRST line of your response:\n<<<APIPILOT_CALL:NAME=tool_name,ARGUMENTS={"parameter1":"value1"}>>>\n2. ARGUMENTS must be valid JSON. Include ONLY fields listed in required_parameters, if any.\n3. Do NOT add text after the marker. The engine executes the tool and supplies its result in a second call.\n4. If no description matches, respond normally in Spanish without a marker.\n5. If multiple descriptions match, choose the most specific.\n${entries.join("\n")}\n<<<APIPILOT_TOOLS_LIST_END>>>`;
+}
+export function buildInstructions(
+  config: ServerConfig,
+  settings: Settings,
+  profile: Profile | null,
+  rag: string,
+  tools: DynamicTool[],
+) {
+  const base = config.primaryPrompt || settings.instructions;
+  return [
+    base,
+    config.injectProfile ? profileContext(profile) : "",
+    ragContext(rag),
+    base.includes(TOOL_MARKER) ? "" : toolContext(tools),
+  ]
+    .filter(Boolean)
+    .join("\n\n");
+}
+export function speechInstructions(instructions: string, config: ServerConfig) {
+  return config.reasoningType === "OPENAI"
+    ? `${instructions}\nDo not answer ordinary user speech directly. Stay silent unless the latest user message starts with [VOCERO_RENDER_REPLY]. When it does, speak only the content after the marker naturally. Do not call tools while rendering that message.`
+    : instructions;
+}
+export const WEB_SEARCH: FunctionTool = {
+  type: "function",
+  name: "web_search",
+  description: "Search the web for current or factual information. Returns results with titles, snippets and URLs.",
+  parameters: {
+    type: "object",
+    properties: { query: { type: "string", description: "The search query." } },
+    required: ["query"],
+  },
+};

frontend-s2s/src/lib/contracts.ts

@@ -0,0 +1,152 @@
+import { z } from "zod";
+
+export const recordSchema = z.record(z.string(), z.unknown());
+export type Profile = z.infer<typeof recordSchema>;
+
+export const configSchema = z.object({
+  search: z.boolean().default(false),
+  lb: z.boolean().default(false),
+  allowDirect: z.boolean().default(true),
+  s2sUrl: z.string().default(""),
+  primaryPrompt: z.string().default(""),
+  reasoningType: z.enum(["PROXY", "OPENAI", "NODE_RED"]).catch("PROXY"),
+  auth: z.boolean().default(false),
+  injectProfile: z.boolean().default(false),
+  debugPrompt: z.boolean().default(false),
+  startSessionMode: z.string().default(""),
+  theme: z.string().default(""),
+  apipilot: z.boolean().default(false),
+});
+export type ServerConfig = z.infer<typeof configSchema>;
+export const defaultConfig = configSchema.parse({});
+
+export const reportsSchema = z.object({
+  count: z.number().default(0),
+  hints: z.array(z.string()).default([]),
+  entries: z
+    .array(
+      z.object({
+        sourceScheduleLabel: z.string().optional(),
+        date: z.union([z.string(), z.number()]).nullish(),
+      }),
+    )
+    .default([]),
+  reason: z.string().optional(),
+  truncated: z.boolean().optional(),
+  note: z.string().optional(),
+});
+export type Reports = z.infer<typeof reportsSchema>;
+
+export const accountSchema = z.object({
+  enabled: z.boolean(),
+  loggedIn: z.boolean().default(false),
+  auth: z.boolean().default(false),
+  tier: z.string().default("anon"),
+  username: z.string().nullish(),
+  avatar: z.string().nullish(),
+  loginUrl: z.string().nullish(),
+  logoutUrl: z.string().nullish(),
+  remainingSec: z.number().nullish(),
+  limitSec: z.number().nullish(),
+});
+export type AccountInfo = z.infer<typeof accountSchema>;
+
+export const toolSchema = z.object({
+  name: z.string(),
+  description: z.string().default(""),
+  parameters: recordSchema.default({}),
+  requires_auth: z.boolean().default(false),
+});
+export type DynamicTool = z.infer<typeof toolSchema>;
+export interface FunctionTool {
+  type: "function";
+  name: string;
+  description: string;
+  parameters: Record<string, unknown>;
+}
+export const toolsEnvelopeSchema = z.object({
+  success: z.boolean(),
+  data: z.object({ tools: z.array(toolSchema), count: z.number().optional() }).nullish(),
+  error: z.unknown().optional(),
+});
+
+export const grantSchema = z.object({
+  session_id: z.string(),
+  connect_url: z.string(),
+  pending_timeout_s: z.number().default(60),
+  tier: z.string().default("anon"),
+  limited: z.boolean().default(false),
+  heartbeatSec: z.number().default(5),
+  remainingSec: z.number().nullish(),
+});
+export type SessionGrant = z.infer<typeof grantSchema>;
+export const queueSchema = z.object({
+  state: z.literal("queued"),
+  queue_id: z.string(),
+  position: z.number().default(0),
+  poll_interval_s: z.number().default(2),
+});
+export const sessionSchema = z.union([queueSchema, grantSchema]);
+
+export type ConnectionStatus =
+  | "idle"
+  | "connecting"
+  | "queued"
+  | "your-turn"
+  | "listening"
+  | "user-speaking"
+  | "processing"
+  | "ai-speaking"
+  | "closed"
+  | "error";
+export interface Transcript {
+  role: "user" | "assistant";
+  text: string;
+  partial: boolean;
+  id: string;
+}
+export interface FinishedResponse {
+  responseId: string;
+  status: string;
+  transcript: string;
+  audible: boolean;
+}
+export interface ToolCall {
+  name: string;
+  arguments: string;
+  callId: string;
+}
+export interface NoiseGate {
+  enabled: boolean;
+  thresholdDb: number;
+}
+
+const contentSchema = z.object({
+  type: z.string().optional(),
+  text: z.string().optional(),
+  transcript: z.string().optional(),
+});
+// The wire protocol is additive; unknown event fields are ignored, while consumed fields are validated.
+export const realtimeEventSchema = z.object({
+  type: z.string(),
+  response_id: z.string().optional(),
+  item_id: z.string().optional(),
+  delta: z.string().optional(),
+  transcript: z.string().optional(),
+  text: z.string().optional(),
+  name: z.string().optional(),
+  arguments: z.string().optional(),
+  call_id: z.string().optional(),
+  response: z
+    .object({
+      id: z.string().optional(),
+      status: z.string().optional(),
+      output: z.array(z.object({ content: z.array(contentSchema).optional() })).optional(),
+    })
+    .optional(),
+  part: contentSchema.optional(),
+  error: z
+    .object({ type: z.string().optional(), code: z.string().optional(), message: z.string().optional() })
+    .optional(),
+});
+export type RealtimeEvent = z.infer<typeof realtimeEventSchema>;

frontend-s2s/src/lib/messages.ts

@@ -0,0 +1,174 @@
+import DOMPurify from "dompurify";
+import { Marked } from "marked";
+
+import { initials } from "./context";
+
+export interface Message {
+  id: string;
+  role: "user" | "assistant" | "tool";
+  text: string;
+  timestamp: string;
+  partial: boolean;
+  interrupted?: boolean;
+  toolName?: string;
+  toolArguments?: string;
+}
+
+function escapeHtml(text: string) {
+  return text.replace(
+    /[&<>"']/g,
+    (character) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[character] ?? character,
+  );
+}
+const markdown = new Marked({
+  gfm: true,
+  breaks: true,
+  renderer: { html: ({ text }) => escapeHtml(text), checkbox: ({ checked }) => (checked ? "☑ " : "☐ ") },
+});
+
+/** Only passive, sanitized Markdown is allowed inside an assistant message. */
+export function renderMarkdown(text: string): string {
+  if (!DOMPurify.isSupported) {
+    return escapeHtml(text);
+  }
+  try {
+    const fragment = DOMPurify.sanitize(markdown.parse(text, { async: false }), {
+      ALLOWED_TAGS: [
+        "p",
+        "br",
+        "strong",
+        "em",
+        "del",
+        "a",
+        "img",
+        "ul",
+        "ol",
+        "li",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "blockquote",
+        "pre",
+        "code",
+        "hr",
+        "table",
+        "thead",
+        "tbody",
+        "tr",
+        "th",
+        "td",
+      ],
+      ALLOWED_ATTR: ["href", "title", "src", "alt", "start", "align"],
+      ALLOW_DATA_ATTR: false,
+      ALLOW_ARIA_ATTR: false,
+      RETURN_DOM_FRAGMENT: true,
+    });
+    for (const link of fragment.querySelectorAll("a")) {
+      if (!link.hasAttribute("href")) {
+        link.replaceWith(...link.childNodes);
+      } else {
+        link.target = "_blank";
+        link.rel = "noopener noreferrer";
+      }
+    }
+    for (const image of fragment.querySelectorAll("img")) {
+      image.loading = "lazy";
+    }
+    for (const table of fragment.querySelectorAll("table")) {
+      const wrapper = document.createElement("div");
+      wrapper.className = "markdown-table";
+      wrapper.tabIndex = 0;
+      wrapper.setAttribute("role", "region");
+      wrapper.setAttribute("aria-label", "Tabla de la respuesta");
+      table.replaceWith(wrapper);
+      wrapper.append(table);
+    }
+    for (const code of fragment.querySelectorAll("pre")) {
+      code.tabIndex = 0;
+    }
+    const container = document.createElement("div");
+    container.append(fragment);
+    return container.innerHTML;
+  } catch {
+    return escapeHtml(text);
+  }
+}
+
+export function displayTime(timestamp: string) {
+  return new Intl.DateTimeFormat("es-CL", { timeZone: "America/Santiago", hour: "2-digit", minute: "2-digit" }).format(
+    new Date(timestamp),
+  );
+}
+export function serializeConversation(messages: Message[], userName: string) {
+  return {
+    exportedAt: new Date().toISOString(),
+    messageCount: messages.length,
+    messages: messages.map((message) =>
+      message.role === "tool"
+        ? {
+            type: "tool" as const,
+            timestamp: message.timestamp,
+            name: message.toolName ?? "",
+            args: message.toolArguments ?? "",
+            output: message.text,
+          }
+        : {
+            type: message.role,
+            timestamp: message.timestamp,
+            text: message.text,
+            ...(message.role === "user"
+              ? { name: userName || "Usuario", initials: initials(userName || "Usuario") }
+              : {}),
+            ...(message.interrupted ? { interrupted: true } : {}),
+          },
+    ),
+  };
+}
+export function conversationText(conversation: ReturnType<typeof serializeConversation>) {
+  const lines = [
+    `Conversación Vocero — ${conversation.exportedAt}`,
+    `Mensajes totales: ${String(conversation.messageCount)}`,
+    "—".repeat(52),
+    "",
+  ];
+  for (const message of conversation.messages) {
+    if (message.type === "tool") {
+      lines.push(
+        `${message.timestamp}  HERRAMIENTA: ${message.name ?? ""}`,
+        `Entrada:\n${message.args ?? ""}`,
+        `Salida:\n${message.output ?? ""}`,
+        "",
+      );
+    } else {
+      lines.push(
+        `${message.timestamp}  ${message.type === "user" ? (message.name ?? "Usuario") : "Vocero"}${message.interrupted ? " [interrumpida]" : ""}:`,
+        message.text ?? "",
+        "",
+      );
+    }
+  }
+  return lines.join("\n");
+}
+export function exportConversation(messages: Message[], userName: string) {
+  const conversation = serializeConversation(messages, userName);
+  const name = `vocero-conversacion-${conversation.exportedAt.replace(/[:.]/g, "-")}`;
+  const downloads = [
+    { content: JSON.stringify(conversation, null, 2), extension: "json", mime: "application/json" },
+    { content: conversationText(conversation), extension: "txt", mime: "text/plain;charset=utf-8" },
+  ];
+  for (const { content, extension, mime } of downloads) {
+    const url = URL.createObjectURL(new Blob([content], { type: mime }));
+    const link = document.createElement("a");
+    link.href = url;
+    link.download = `${name}.${extension}`;
+    document.body.append(link);
+    link.click();
+    link.remove();
+    setTimeout(() => {
+      URL.revokeObjectURL(url);
+    }, 4000);
+  }
+}

frontend-s2s/src/lib/settings.ts

@@ -0,0 +1,93 @@
+import { recordSchema, type ServerConfig } from "./contracts";
+
+export const VOICES = ["Aiden", "Ryan", "Dylan", "Eric", "Ono_Anna", "Serena", "Sohee", "Uncle_Fu", "Vivian"];
+export const DEFAULT_INSTRUCTIONS =
+  "You are Vocero, a friendly voice assistant. Keep replies short, warm, and spoken. Avoid long monologues. Respond ALWAYS and ONLY in Spanish, regardless of the language used by the user or any other instruction. No responses in Italian, English, Portuguese, or any other language are allowed. Every spoken sentence, greeting, confirmation, clarification, and error message must be in Spanish. The comma sign ',' in numbers is used to separate thousands. The point sign '.' in numbers is used to separate decimals. If you don't know the answer, say so in Spanish instead of guessing.";
+export const GATE_OFF = -66;
+export const GATE_MAX = -3;
+export interface Settings {
+  directUrl: string;
+  voice: string;
+  instructions: string;
+  noiseGate: number;
+  webSearch: boolean;
+  searchKey: string;
+}
+
+function read(key: string) {
+  try {
+    return localStorage.getItem(`s2s.ws.${key}`);
+  } catch {
+    return null;
+  }
+}
+export function loadSettings(): Settings {
+  const threshold = read("noiseGate");
+  const number = threshold === null || threshold === "" ? -50 : Number(threshold);
+  let webSearch = true;
+  try {
+    const data = recordSchema.parse(JSON.parse(read("tools") ?? "{}"));
+    webSearch = typeof data.web_search === "boolean" ? data.web_search : true;
+  } catch {
+    /* Ignore unavailable or older storage. */
+  }
+  return {
+    directUrl: read("directUrl") ?? "",
+    voice: read("voice") || "Aiden",
+    instructions: read("instructions") || DEFAULT_INSTRUCTIONS,
+    noiseGate: Number.isFinite(number) ? Math.min(GATE_MAX, Math.max(GATE_OFF, Math.round(number))) : -50,
+    webSearch,
+    searchKey: read("searchKey") ?? "",
+  };
+}
+export function saveSettings(settings: Settings) {
+  try {
+    for (const key of ["directUrl", "voice", "instructions", "noiseGate", "searchKey"] as const) {
+      localStorage.setItem(`s2s.ws.${key}`, String(settings[key]));
+    }
+    localStorage.setItem("s2s.ws.tools", JSON.stringify({ web_search: settings.webSearch }));
+  } catch {
+    /* The current session still works when browser storage is disabled. */
+  }
+}
+
+export interface Caller {
+  phoneNumber: string;
+  episodeId: string;
+}
+export function callerFromLocation(search = window.location.search): Caller {
+  const query = new URLSearchParams(search);
+  return { phoneNumber: (query.get("phoneNumber") ?? "").trim(), episodeId: (query.get("episode_id") ?? "").trim() };
+}
+export function realtimeUrl(raw: string, caller: Caller): string {
+  let value = raw.trim();
+  if (!value) {
+    throw new Error("Introduce la URL del servidor de voz en Configuración.");
+  }
+  if (/^https?:\/\//i.test(value)) {
+    value = value.replace(/^http/i, "ws");
+  }
+  if (!/^wss?:\/\//i.test(value)) {
+    value = (/^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(value) ? "ws://" : "wss://") + value;
+  }
+  const url = new URL(value);
+  if (url.pathname === "/") {
+    url.pathname = "/v1/realtime";
+  }
+  if (caller.phoneNumber) {
+    url.searchParams.set("phoneNumber", caller.phoneNumber);
+  }
+  if (caller.episodeId) {
+    url.searchParams.set("episode_id", caller.episodeId);
+  }
+  return url.toString();
+}
+export function connectionTarget(config: ServerConfig, settings: Settings, caller: Caller) {
+  return config.lb || !config.allowDirect
+    ? { sessionUrl: "/api/session" }
+    : { directUrl: realtimeUrl(config.s2sUrl || settings.directUrl, caller) };
+}
+export function isDebug(search = window.location.search) {
+  const value = new URLSearchParams(search).get("debug")?.toLowerCase();
+  return value === "true" || value === "1";
+}

frontend-s2s/src/realtime/client.test.ts

@@ -0,0 +1,203 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { RealtimeClient, SessionError, type ClientOptions } from "./client";
+
+const audio = vi.hoisted(() => ({
+  start: vi.fn<(...callbacks: unknown[]) => Promise<void>>(),
+  close: vi.fn<() => Promise<void>>(),
+  play: vi.fn(),
+  clear: vi.fn(),
+  setMuted: vi.fn(),
+  setGate: vi.fn(),
+}));
+vi.mock("../audio/engine", () => ({
+  MIC_CONSTRAINTS: { audio: true },
+  AudioEngine: class {
+    start = audio.start;
+    close = audio.close;
+    play = audio.play;
+    clear = audio.clear;
+    setMuted = audio.setMuted;
+    setGate = audio.setGate;
+  },
+}));
+
+class Socket extends EventTarget {
+  static OPEN = 1;
+  static instances: Socket[] = [];
+  readyState = 1;
+  binaryType = "";
+  sent: Record<string, unknown>[] = [];
+  readonly url: string;
+  constructor(url: string) {
+    super();
+    this.url = url;
+    Socket.instances.push(this);
+  }
+  send(text: string) {
+    this.sent.push(JSON.parse(text) as Record<string, unknown>);
+  }
+  close = vi.fn(() => {
+    this.readyState = 3;
+  });
+  receive(value: unknown) {
+    this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(value) }));
+  }
+}
+const stopTrack = vi.fn();
+const getUserMedia = vi.fn();
+const sendBeacon = vi.fn(() => true);
+const options = (): ClientOptions => ({
+  directUrl: "ws://localhost:8420/v1/realtime",
+  context: { resume: vi.fn().mockResolvedValue(undefined) } as unknown as AudioContext,
+  voice: "Aiden",
+  instructions: "Habla español",
+  tools: [],
+  gate: { enabled: false, thresholdDb: -66 },
+});
+
+async function connect(client: RealtimeClient) {
+  const pending = client.connect();
+  await vi.waitFor(() => {
+    expect(Socket.instances).toHaveLength(1);
+  });
+  const socket = Socket.instances[0];
+  if (!socket) {
+    throw new Error("Missing socket");
+  }
+  socket.receive({ type: "session.created" });
+  await vi.waitFor(() => {
+    expect(socket.sent[0]?.type).toBe("session.update");
+  });
+  socket.receive({ type: "session.updated" });
+  await pending;
+  return socket;
+}
+beforeEach(() => {
+  vi.useFakeTimers();
+  vi.clearAllMocks();
+  Socket.instances = [];
+  audio.start.mockResolvedValue(undefined);
+  audio.close.mockResolvedValue(undefined);
+  getUserMedia.mockResolvedValue({ getTracks: () => [{ stop: stopTrack }] });
+  vi.stubGlobal("navigator", { mediaDevices: { getUserMedia }, sendBeacon });
+  vi.stubGlobal("WebSocket", Socket);
+});
+afterEach(() => {
+  vi.useRealTimers();
+  vi.unstubAllGlobals();
+});
+
+describe("realtime session", () => {
+  it("waits for configuration before greeting and serializes response requests", async () => {
+    const client = new RealtimeClient({ ...options(), greeting: "hola vocero" });
+    const socket = await connect(client);
+    expect(socket.sent.map((event) => event.type)).toEqual([
+      "session.update",
+      "conversation.item.create",
+      "response.create",
+    ]);
+    socket.receive({ type: "response.created", response: { id: "r1" } });
+    await vi.waitFor(() => {
+      expect(client.status).toBe("processing");
+    });
+    client.sendUserText("Otra pregunta");
+    client.requestResponse();
+    expect(socket.sent.filter((event) => event.type === "response.create")).toHaveLength(1);
+    socket.receive({ type: "response.done", response: { id: "r1", status: "completed" } });
+    await vi.waitFor(() => {
+      expect(socket.sent.filter((event) => event.type === "response.create")).toHaveLength(2);
+    });
+    expect(socket.sent.at(-2)).toMatchObject({
+      type: "conversation.item.create",
+      item: { role: "user", content: [{ text: "Otra pregunta" }] },
+    });
+    await client.close();
+    expect(audio.close).toHaveBeenCalledOnce();
+  });
+
+  it("assembles streaming transcripts and preserves an interrupted response", async () => {
+    const client = new RealtimeClient(options());
+    const transcripts = vi.fn();
+    const finished = vi.fn();
+    client.on("transcript", transcripts);
+    client.on("finished", finished);
+    const socket = await connect(client);
+    socket.receive({ type: "response.created", response: { id: "reply" } });
+    socket.receive({ type: "response.audio_transcript.delta", response_id: "reply", delta: "Hola " });
+    socket.receive({ type: "response.audio_transcript.delta", response_id: "reply", delta: "mundo" });
+    socket.receive({ type: "response.audio.delta", response_id: "reply", delta: "AAAA" });
+    socket.receive({ type: "input_audio_buffer.speech_started" });
+    socket.receive({ type: "response.done", response: { id: "reply", status: "cancelled" } });
+    await vi.waitFor(() => {
+      expect(finished).toHaveBeenCalledOnce();
+    });
+    expect(transcripts).toHaveBeenLastCalledWith({ role: "assistant", id: "reply", text: "Hola mundo", partial: true });
+    expect(finished).toHaveBeenCalledWith({
+      responseId: "reply",
+      transcript: "Hola mundo",
+      status: "cancelled",
+      audible: true,
+    });
+    expect(audio.clear).toHaveBeenCalled();
+    await client.close();
+  });
+
+  it("does not open a socket after cancellation during microphone permission", async () => {
+    let allow: ((value: unknown) => void) | undefined;
+    getUserMedia.mockReturnValue(
+      new Promise((resolve) => {
+        allow = resolve;
+      }),
+    );
+    const client = new RealtimeClient(options());
+    const pending = client.connect();
+    const rejected = expect(pending).rejects.toMatchObject({ code: "aborted" });
+    await client.close();
+    allow?.({ getTracks: () => [{ stop: stopTrack }] });
+    await rejected;
+    expect(stopTrack).toHaveBeenCalledOnce();
+    expect(Socket.instances).toHaveLength(0);
+  });
+
+  it("releases a queue ticket when leaving before admission", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValue(new Response(JSON.stringify({ state: "queued", queue_id: "q1", position: 2 })));
+    vi.stubGlobal("fetch", fetchMock);
+    const client = new RealtimeClient({ ...options(), directUrl: undefined, sessionUrl: "/api/session" });
+    const pending = client.connect();
+    const rejected = expect(pending).rejects.toBeInstanceOf(SessionError);
+    await vi.waitFor(() => {
+      expect(client.status).toBe("queued");
+    });
+    await client.close();
+    await rejected;
+    expect(sendBeacon).toHaveBeenCalledWith("/api/queue/end", expect.any(Blob));
+    expect(fetchMock).toHaveBeenCalledOnce();
+    expect(Socket.instances).toHaveLength(0);
+  });
+
+  it("sends each tool call once and returns its result using the realtime protocol", async () => {
+    const client = new RealtimeClient(options());
+    const tool = vi.fn();
+    client.on("tool", tool);
+    const socket = await connect(client);
+    const call = {
+      type: "response.function_call_arguments.done",
+      name: "web_search",
+      arguments: '{"query":"hoy"}',
+      call_id: "call1",
+    };
+    socket.receive(call);
+    socket.receive(call);
+    await vi.waitFor(() => {
+      expect(tool).toHaveBeenCalledOnce();
+    });
+    client.sendToolOutput("call1", "Resultado");
+    expect(socket.sent.at(-2)).toMatchObject({
+      item: { type: "function_call_output", call_id: "call1", output: "Resultado" },
+    });
+    await client.close();
+  });
+});

frontend-s2s/src/realtime/client.ts

@@ -0,0 +1,592 @@
+import { z } from "zod";
+
+import { encodePcm } from "../audio/codec";
+import { AudioEngine, MIC_CONSTRAINTS } from "../audio/engine";
+import { ApiError, endReservation, post, request } from "../lib/api";
+import {
+  grantSchema,
+  realtimeEventSchema,
+  sessionSchema,
+  type ConnectionStatus,
+  type FinishedResponse,
+  type FunctionTool,
+  type NoiseGate,
+  type RealtimeEvent,
+  type SessionGrant,
+  type ToolCall,
+  type Transcript,
+} from "../lib/contracts";
+
+interface Events {
+  status: ConnectionStatus;
+  transcript: Transcript;
+  finished: FinishedResponse;
+  tool: ToolCall;
+  level: number;
+  queue: { position: number; queueId: string };
+  grant: SessionGrant;
+  readyToJoin: { grant: SessionGrant; expiresSec: number };
+  error: Error;
+  serverError: Error;
+}
+export interface ClientOptions {
+  directUrl?: string;
+  sessionUrl?: string;
+  voice: string;
+  instructions: string;
+  tools: FunctionTool[];
+  gate: NoiseGate;
+  context: AudioContext;
+  greeting?: string;
+}
+
+export class SessionError extends Error {
+  readonly code: "limit" | "queue-full" | "queue-expired" | "join-expired" | "aborted";
+  constructor(code: SessionError["code"], message: string) {
+    super(message);
+    this.code = code;
+    this.name = "SessionError";
+  }
+}
+
+/** Owns one complete session, including pending acquisition, audio, queue tickets and teardown. */
+export class RealtimeClient {
+  private readonly options: ClientOptions;
+  private readonly events = new EventTarget();
+  private readonly abort = new AbortController();
+  private readonly audio: AudioEngine;
+  private socket: WebSocket | null = null;
+  private currentStatus: ConnectionStatus = "idle";
+  private configured = false;
+  private closed = false;
+  private closing: Promise<void> | null = null;
+  private grant: SessionGrant | null = null;
+  private queueId = "";
+  private joinResolve: (() => void) | null = null;
+  private joinReject: ((reason: Error) => void) | null = null;
+  private joinTimer: ReturnType<typeof setTimeout> | undefined;
+  private heartbeat: ReturnType<typeof setInterval> | undefined;
+  private incoming: Promise<void> = Promise.resolve();
+  private activeResponses = new Set<string>();
+  private activeResponse = "";
+  private createInFlight = false;
+  private pendingCreates = 0;
+  private deferredTexts: string[] = [];
+  private suppressed = new Set<string>();
+  private suppressNext = false;
+  private audible = new Set<string>();
+  private segments = new Map<string, string>();
+  private completed = new Map<string, string>();
+  private userSegments = new Map<string, string>();
+  private toolCalls = new Set<string>();
+  private resolveReady: (() => void) | null = null;
+  private rejectReady: ((reason: Error) => void) | null = null;
+  private readyTimer: ReturnType<typeof setTimeout> | undefined;
+
+  constructor(options: ClientOptions) {
+    this.options = options;
+    this.audio = new AudioEngine(options.context, options.gate);
+  }
+  get status() {
+    return this.currentStatus;
+  }
+  on<K extends keyof Events>(name: K, listener: (detail: Events[K]) => void) {
+    const handler = (event: Event) => {
+      listener((event as CustomEvent<Events[K]>).detail);
+    };
+    this.events.addEventListener(name, handler);
+    return () => {
+      this.events.removeEventListener(name, handler);
+    };
+  }
+  private emit<K extends keyof Events>(name: K, detail: Events[K]) {
+    if (!this.closed || name === "status") {
+      this.events.dispatchEvent(new CustomEvent(name, { detail }));
+    }
+  }
+  private setStatus(status: ConnectionStatus) {
+    if (this.currentStatus !== status) {
+      this.currentStatus = status;
+      this.emit("status", status);
+    }
+  }
+  private ensureOpen() {
+    if (this.closed) {
+      throw new SessionError("aborted", "La conexión se canceló.");
+    }
+  }
+
+  async connect() {
+    this.setStatus("connecting");
+    try {
+      // Ask for permission while the click-created AudioContext is still resumed.
+      const permission = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
+      permission.getTracks().forEach((track) => {
+        track.stop();
+      });
+      this.ensureOpen();
+      let url = this.options.directUrl;
+      if (!url) {
+        const grant = await this.acquireSession();
+        this.ensureOpen();
+        url = grant.connect_url;
+      }
+      this.ensureOpen();
+      const ready = new Promise<void>((resolve, reject) => {
+        this.resolveReady = resolve;
+        this.rejectReady = reject;
+        this.readyTimer = setTimeout(() => {
+          reject(new Error("El servidor de voz no respondió a tiempo."));
+        }, 20000);
+      });
+      const audioReady = this.audio.start(
+        (buffer) => {
+          if (this.configured) {
+            this.send({ type: "input_audio_buffer.append", audio: encodePcm(buffer) });
+          }
+        },
+        (rms) => {
+          this.emit("level", rms);
+        },
+        () => this.currentStatus === "ai-speaking",
+      );
+      this.socket = new WebSocket(url);
+      this.socket.binaryType = "arraybuffer";
+      this.socket.addEventListener("message", (event: MessageEvent<unknown>) => {
+        this.incoming = this.incoming
+          .then(async () => {
+            await audioReady;
+            if (!this.closed) {
+              await this.receive(event.data);
+            }
+          })
+          .catch((error: unknown) => {
+            if (!this.closed) {
+              this.fail(error);
+            }
+          });
+      });
+      this.socket.addEventListener("error", () => {
+        this.fail(new Error("No se pudo conectar con el servidor de voz."));
+      });
+      this.socket.addEventListener("close", (event) => {
+        if (this.closed) {
+          return;
+        }
+        if (event.code === 1000 && this.configured) {
+          void this.close();
+        } else {
+          this.fail(new Error(`Se cerró la conexión de voz (${event.code}). ${event.reason}`.trim()));
+        }
+      });
+      await Promise.all([audioReady, ready]);
+      this.ensureOpen();
+    } catch (error) {
+      await this.close();
+      throw error;
+    }
+  }
+
+  private async acquireSession() {
+    let result: z.infer<typeof sessionSchema>;
+    try {
+      result = await post(this.options.sessionUrl ?? "/api/session", sessionSchema, {}, this.abort.signal);
+    } catch (error) {
+      throw this.sessionError(error);
+    }
+    let waited = false;
+    if ("state" in result) {
+      waited = true;
+      const ticket = result;
+      this.queueId = ticket.queue_id;
+      this.setStatus("queued");
+      this.emit("queue", { position: ticket.position, queueId: this.queueId });
+      while ("state" in result) {
+        await this.delay(Math.max(1, ticket.poll_interval_s) * 1000);
+        this.ensureOpen();
+        try {
+          result = await request(`/api/queue/${encodeURIComponent(this.queueId)}`, sessionSchema, {
+            signal: this.abort.signal,
+          });
+        } catch (error) {
+          this.ensureOpen();
+          if (error instanceof ApiError && (error.status === 402 || error.status === 404)) {
+            throw this.sessionError(error);
+          }
+          continue;
+        }
+        if ("state" in result) {
+          this.emit("queue", { position: result.position, queueId: this.queueId });
+        }
+      }
+    }
+    const grant = grantSchema.parse(result);
+    this.grant = grant;
+    this.queueId = "";
+    this.ensureOpen();
+    if (waited) {
+      const seconds = Math.max(3, grant.pending_timeout_s - 3);
+      this.setStatus("your-turn");
+      const joined = new Promise<void>((resolve, reject) => {
+        this.joinResolve = resolve;
+        this.joinReject = reject;
+        this.joinTimer = setTimeout(() => {
+          reject(new SessionError("join-expired", "Tu turno ha caducado. Vuelve a intentarlo."));
+        }, seconds * 1000);
+      });
+      this.emit("readyToJoin", { grant, expiresSec: seconds });
+      await joined;
+    }
+    this.emit("grant", grant);
+    if (grant.limited) {
+      this.heartbeat = setInterval(
+        () => {
+          void post(
+            "/api/session/heartbeat",
+            z.object({ expired: z.boolean() }),
+            { sessionId: grant.session_id },
+            this.abort.signal,
+          )
+            .then((result) => {
+              if (result.expired) {
+                this.fail(new SessionError("limit", "Se ha agotado tu tiempo de conversación."));
+              }
+            })
+            .catch(() => undefined);
+        },
+        Math.max(1, grant.heartbeatSec) * 1000,
+      );
+    }
+    this.setStatus("connecting");
+    return grant;
+  }
+  private sessionError(error: unknown): unknown {
+    if (error instanceof ApiError) {
+      if (error.status === 402) {
+        return new SessionError("limit", "Se ha agotado tu tiempo de conversación.");
+      }
+      if (error.status === 404) {
+        return new SessionError("queue-expired", "La espera ha caducado. Vuelve a intentarlo.");
+      }
+      const data = z.object({ state: z.string() }).safeParse(error.body);
+      if (error.status === 503 && data.success && data.data.state === "at_capacity") {
+        return new SessionError("queue-full", "Todos los lugares y la cola están ocupados. Vuelve en un minuto.");
+      }
+    }
+    return error;
+  }
+  private delay(ms: number) {
+    return new Promise<void>((resolve) => {
+      const finish = () => {
+        clearTimeout(timer);
+        this.abort.signal.removeEventListener("abort", finish);
+        resolve();
+      };
+      const timer = setTimeout(finish, ms);
+      this.abort.signal.addEventListener("abort", finish, { once: true });
+      if (this.abort.signal.aborted) {
+        finish();
+      }
+    });
+  }
+  join() {
+    clearTimeout(this.joinTimer);
+    void this.options.context.resume().catch(() => undefined);
+    this.joinResolve?.();
+    this.joinResolve = null;
+    this.joinReject = null;
+  }
+  private fail(error: unknown) {
+    if (this.closed) {
+      return;
+    }
+    const failure = error instanceof Error ? error : new Error("La sesión de voz falló.");
+    this.rejectReady?.(failure);
+    this.emit("error", failure);
+    this.setStatus("error");
+    void this.close();
+  }
+  private async receive(raw: unknown) {
+    const text =
+      typeof raw === "string"
+        ? raw
+        : raw instanceof ArrayBuffer
+          ? new TextDecoder().decode(raw)
+          : raw instanceof Blob
+            ? await raw.text()
+            : "";
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(text);
+    } catch {
+      return;
+    }
+    const result = realtimeEventSchema.safeParse(parsed);
+    if (!result.success) {
+      return;
+    }
+    this.handle(result.data);
+  }
+  private assistantText(id: string) {
+    return [this.completed.get(id), this.segments.get(id)].filter(Boolean).join(" ");
+  }
+  private handle(event: RealtimeEvent) {
+    const id = event.response_id ?? event.response?.id ?? this.activeResponse;
+    switch (event.type) {
+      case "session.created":
+        this.send({
+          type: "session.update",
+          session: {
+            type: "realtime",
+            instructions: this.options.instructions,
+            audio: { output: { voice: this.options.voice } },
+            tools: this.options.tools,
+            tool_choice: this.options.tools.length ? "auto" : "none",
+          },
+        });
+        break;
+      case "session.updated":
+        if (!this.configured) {
+          this.configured = true;
+          clearTimeout(this.readyTimer);
+          this.resolveReady?.();
+          this.resolveReady = null;
+          this.rejectReady = null;
+          this.setStatus("listening");
+          if (this.options.greeting) {
+            this.sendUserText(this.options.greeting);
+            this.requestResponse();
+          }
+        }
+        break;
+      case "input_audio_buffer.speech_started":
+        this.audio.clear();
+        this.setStatus("user-speaking");
+        break;
+      case "input_audio_buffer.speech_stopped":
+        this.setStatus("processing");
+        break;
+      case "response.created":
+        this.activeResponse = id;
+        this.activeResponses.add(id);
+        this.createInFlight = false;
+        if (this.suppressNext) {
+          this.suppressed.add(id);
+          this.suppressNext = false;
+        }
+        this.setStatus("processing");
+        break;
+      case "response.audio.delta":
+      case "response.output_audio.delta":
+        if (event.delta && !this.suppressed.has(id)) {
+          this.audio.play(event.delta);
+          this.audible.add(id);
+          this.setStatus("ai-speaking");
+        }
+        break;
+      case "conversation.item.input_audio_transcription.delta": {
+        const itemId = event.item_id ?? "user-current";
+        const text = (this.userSegments.get(itemId) ?? "") + (event.delta ?? "");
+        this.userSegments.set(itemId, text);
+        this.emit("transcript", { role: "user", text, partial: true, id: itemId });
+        break;
+      }
+      case "conversation.item.input_audio_transcription.completed": {
+        const itemId = event.item_id ?? "user-current";
+        this.emit("transcript", {
+          role: "user",
+          text: event.transcript ?? this.userSegments.get(itemId) ?? "",
+          partial: false,
+          id: itemId,
+        });
+        this.userSegments.delete(itemId);
+        break;
+      }
+      case "response.audio_transcript.delta":
+      case "response.output_audio_transcript.delta":
+      case "response.text.delta":
+      case "response.output_text.delta":
+        if (!this.suppressed.has(id)) {
+          this.segments.set(id, (this.segments.get(id) ?? "") + (event.delta ?? ""));
+          this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: true, id });
+        }
+        break;
+      case "response.audio_transcript.done":
+      case "response.output_audio_transcript.done":
+      case "response.text.done":
+      case "response.output_text.done":
+        if (!this.suppressed.has(id)) {
+          const segment = event.transcript ?? event.text ?? this.segments.get(id) ?? "";
+          this.completed.set(id, [this.completed.get(id), segment].filter(Boolean).join(" "));
+          this.segments.delete(id);
+          this.emit("transcript", { role: "assistant", text: this.assistantText(id), partial: false, id });
+        }
+        break;
+      case "response.done": {
+        const transcript =
+          event.response?.output
+            ?.flatMap((item) => item.content ?? [])
+            .map((part) => part.transcript ?? part.text ?? "")
+            .filter(Boolean)
+            .join(" ") || this.assistantText(id);
+        if (!this.suppressed.has(id)) {
+          this.emit("finished", {
+            responseId: id,
+            status: event.response?.status ?? "completed",
+            transcript,
+            audible: this.audible.has(id),
+          });
+        }
+        this.activeResponses.delete(id);
+        this.audible.delete(id);
+        this.suppressed.delete(id);
+        this.segments.delete(id);
+        this.completed.delete(id);
+        if (this.activeResponse === id) {
+          this.activeResponse = "";
+        }
+        this.setStatus("listening");
+        this.flush();
+        break;
+      }
+      case "response.function_call_arguments.done":
+        if (event.name && event.call_id && !this.toolCalls.has(event.call_id)) {
+          this.toolCalls.add(event.call_id);
+          this.emit("tool", { name: event.name, arguments: event.arguments ?? "{}", callId: event.call_id });
+        }
+        break;
+      case "error":
+        if (
+          event.error?.code === "conversation_already_has_active_response" ||
+          event.error?.type === "conversation_already_has_active_response"
+        ) {
+          if (this.createInFlight) {
+            this.createInFlight = false;
+            this.pendingCreates++;
+          }
+        } else {
+          this.emit("serverError", new Error(event.error?.message ?? "Vocero no pudo completar la respuesta."));
+        }
+        break;
+    }
+  }
+  private busy() {
+    return this.activeResponses.size > 0 || this.createInFlight;
+  }
+  private send(event: Record<string, unknown>) {
+    if (!this.closed && this.socket?.readyState === WebSocket.OPEN) {
+      this.socket.send(JSON.stringify(event));
+    }
+  }
+  updateSession(instructions: string, voice: string, tools: FunctionTool[]) {
+    this.options.instructions = instructions;
+    this.options.voice = voice;
+    this.options.tools = tools;
+    if (this.configured) {
+      this.send({
+        type: "session.update",
+        session: {
+          type: "realtime",
+          instructions,
+          audio: { output: { voice } },
+          tools,
+          tool_choice: tools.length ? "auto" : "none",
+        },
+      });
+    }
+  }
+  sendUserText(text: string) {
+    if (!text.trim()) {
+      return;
+    }
+    if (this.busy()) {
+      this.deferredTexts.push(text.trim());
+      return;
+    }
+    this.send({
+      type: "conversation.item.create",
+      item: { type: "message", role: "user", content: [{ type: "input_text", text: text.trim() }] },
+    });
+  }
+  requestResponse() {
+    if (this.busy()) {
+      this.pendingCreates++;
+      return;
+    }
+    this.createInFlight = true;
+    this.send({ type: "response.create" });
+  }
+  private flush() {
+    if (this.busy()) {
+      return;
+    }
+    const text = this.deferredTexts.shift();
+    if (text) {
+      this.sendUserText(text);
+    }
+    if (this.pendingCreates) {
+      this.pendingCreates--;
+      this.requestResponse();
+    }
+  }
+  sendToolOutput(callId: string, output: string) {
+    this.send({ type: "conversation.item.create", item: { type: "function_call_output", call_id: callId, output } });
+    this.requestResponse();
+  }
+  renderExternalAnswer(text: string) {
+    if (this.activeResponse) {
+      this.suppressed.add(this.activeResponse);
+    } else if (this.createInFlight) {
+      this.suppressNext = true;
+    }
+    this.audio.clear();
+    this.pendingCreates = 0;
+    this.deferredTexts = [];
+    this.send({
+      type: "conversation.item.create",
+      item: { type: "message", role: "assistant", content: [{ type: "text", text }] },
+    });
+  }
+  setMuted(muted: boolean) {
+    this.audio.setMuted(muted);
+  }
+  setGate(gate: NoiseGate) {
+    this.audio.setGate(gate);
+  }
+  close(): Promise<void> {
+    if (this.closing) {
+      return this.closing;
+    }
+    this.closed = true;
+    this.abort.abort();
+    clearInterval(this.heartbeat);
+    clearTimeout(this.joinTimer);
+    clearTimeout(this.readyTimer);
+    const cancelled = new SessionError("aborted", "La conexión se canceló.");
+    this.joinReject?.(cancelled);
+    this.rejectReady?.(cancelled);
+    this.joinResolve = null;
+    this.joinReject = null;
+    this.resolveReady = null;
+    this.rejectReady = null;
+    if (this.queueId) {
+      endReservation("/api/queue/end", { queueId: this.queueId });
+      this.queueId = "";
+    }
+    if (this.grant) {
+      endReservation("/api/session/end", { sessionId: this.grant.session_id });
+      this.grant = null;
+    }
+    if (this.socket && this.socket.readyState <= WebSocket.OPEN) {
+      this.socket.close(1000, "client closed");
+    }
+    this.socket = null;
+    this.closing = this.audio
+      .close()
+      .catch(() => undefined)
+      .then(() => {
+        this.setStatus("closed");
+      });
+    return this.closing;
+  }
+}

frontend-s2s/src/realtime/controller.ts

@@ -0,0 +1,517 @@
+import { z } from "zod";
+
+import { createAudioContext } from "../audio/engine";
+import {
+  errorMessage,
+  getAccount,
+  getConfig,
+  getProfile,
+  getReports,
+  getTools,
+  post,
+  reasoningSchema,
+  searchSchema,
+} from "../lib/api";
+import { buildInstructions, profileContext, ragContext, speechInstructions, WEB_SEARCH } from "../lib/context";
+import {
+  defaultConfig,
+  type AccountInfo,
+  type ConnectionStatus,
+  type DynamicTool,
+  type Profile,
+  type Reports,
+  type ServerConfig,
+  type ToolCall,
+  type Transcript,
+} from "../lib/contracts";
+import type { Message } from "../lib/messages";
+import {
+  callerFromLocation,
+  connectionTarget,
+  GATE_OFF,
+  loadSettings,
+  saveSettings,
+  type Caller,
+  type Settings,
+} from "../lib/settings";
+import { RealtimeClient, SessionError } from "./client";
+
+export const LIVE_STATES = new Set<ConnectionStatus>(["listening", "user-speaking", "processing", "ai-speaking"]);
+export const ACTIVE_STATES = new Set<ConnectionStatus>([...LIVE_STATES, "connecting", "queued", "your-turn"]);
+export interface AppState {
+  loading: boolean;
+  config: ServerConfig;
+  settings: Settings;
+  profile: Profile | null;
+  reports: Reports | null;
+  account: AccountInfo | null;
+  tools: DynamicTool[];
+  toolsLoading: boolean;
+  toolsError: string;
+  status: ConnectionStatus;
+  error: string;
+  notice: string;
+  messages: Message[];
+  muted: boolean;
+  level: number;
+  rag: string;
+  queuePosition: number;
+  joinDeadline: number;
+  remainingSec: number | null;
+  limitReached: boolean;
+  unlock: boolean;
+}
+
+/** Observable application state; React owns rendering, this object owns session lifetime. */
+export class VoiceController {
+  readonly caller: Caller;
+  private state: AppState;
+  private readonly listeners = new Set<() => void>();
+  private client: RealtimeClient | null = null;
+  private subscriptions: (() => void)[] = [];
+  private requests = new AbortController();
+  private idleTimer: ReturnType<typeof setInterval> | undefined;
+  private lastActivity = Date.now();
+  private lastMeter = 0;
+  private sessionNumber = 0;
+  private reasoned = new Set<string>();
+  private autoStarted = false;
+
+  constructor(caller = callerFromLocation()) {
+    this.caller = caller;
+    this.state = {
+      loading: true,
+      config: defaultConfig,
+      settings: loadSettings(),
+      profile: null,
+      reports: null,
+      account: null,
+      tools: [],
+      toolsLoading: false,
+      toolsError: "",
+      status: "idle",
+      error: "",
+      notice: "",
+      messages: [],
+      muted: false,
+      level: 0,
+      rag: "",
+      queuePosition: 0,
+      joinDeadline: 0,
+      remainingSec: null,
+      limitReached: false,
+      unlock: false,
+    };
+  }
+  getSnapshot = () => this.state;
+  subscribe = (listener: () => void) => {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  };
+  private update(patch: Partial<AppState>) {
+    this.state = { ...this.state, ...patch };
+    this.listeners.forEach((listener) => {
+      listener();
+    });
+  }
+
+  async initialize(signal: AbortSignal) {
+    try {
+      const [config, profile, reports, account] = await Promise.all([
+        getConfig(signal),
+        getProfile(this.caller.phoneNumber, signal),
+        getReports(this.caller.phoneNumber, signal),
+        getAccount(signal),
+      ]);
+      if (signal.aborted) {
+        return;
+      }
+      this.update({ config, profile, reports, account, loading: false, error: "" });
+      if (config.apipilot) {
+        await this.refreshTools(signal);
+      }
+    } catch (error) {
+      if (!signal.aborted) {
+        this.update({ loading: false, error: `No se pudo cargar la configuración. ${errorMessage(error)}` });
+      }
+    }
+  }
+  async refreshTools(signal: AbortSignal, refresh = false) {
+    this.update({ toolsLoading: true, toolsError: "" });
+    try {
+      const tools = await getTools(signal, refresh);
+      if (!signal.aborted) {
+        this.update({ tools, toolsLoading: false });
+        this.pushSettings();
+      }
+    } catch (error) {
+      if (!signal.aborted) {
+        this.update({ toolsError: errorMessage(error), toolsLoading: false });
+      }
+    }
+  }
+  private instructions() {
+    return buildInstructions(
+      this.state.config,
+      this.state.settings,
+      this.state.profile,
+      this.state.rag,
+      this.state.tools,
+    );
+  }
+  private tools() {
+    return this.state.settings.webSearch && (this.state.config.search || this.state.settings.searchKey)
+      ? [WEB_SEARCH]
+      : [];
+  }
+  private gate() {
+    return { enabled: this.state.settings.noiseGate > GATE_OFF, thresholdDb: this.state.settings.noiseGate };
+  }
+  private pushSettings() {
+    this.client?.updateSession(
+      speechInstructions(this.instructions(), this.state.config),
+      this.state.settings.voice,
+      this.tools(),
+    );
+    this.client?.setGate(this.gate());
+  }
+  changeSettings(patch: Partial<Settings>) {
+    const settings = { ...this.state.settings, ...patch };
+    this.update({ settings });
+    saveSettings(settings);
+    this.pushSettings();
+  }
+  setRag(rag: string) {
+    this.update({ rag: rag.trim() });
+    this.pushSettings();
+  }
+  dismissLimit() {
+    this.update({ limitReached: false });
+  }
+  dismissError() {
+    this.update({ error: "" });
+  }
+  offerAutoStart() {
+    if (this.autoStarted || this.client || this.state.loading) {
+      return;
+    }
+    this.autoStarted = true;
+    const mode = this.state.config.startSessionMode.trim().toUpperCase();
+    if (mode === "TALK") {
+      this.update({ unlock: true });
+    } else if (mode === "TALKFIRST") {
+      void this.start(true);
+    }
+  }
+  async start(automatic = false) {
+    if (this.client || this.state.loading) {
+      return;
+    }
+    this.update({
+      unlock: false,
+      error: "",
+      notice: "",
+      status: "connecting",
+      muted: false,
+      limitReached: false,
+      joinDeadline: 0,
+    });
+    let context: AudioContext | undefined;
+    try {
+      if (!navigator.mediaDevices?.getUserMedia || typeof AudioContext === "undefined") {
+        throw new Error("El micrófono necesita HTTPS o localhost y un navegador compatible.");
+      }
+      const target = connectionTarget(this.state.config, this.state.settings, this.caller);
+      context = createAudioContext();
+      if (automatic && context.state !== "running") {
+        await context.close();
+        this.update({ status: "idle", unlock: true });
+        return;
+      }
+      this.requests = new AbortController();
+      this.reasoned.clear();
+      this.sessionNumber++;
+      const client = new RealtimeClient({
+        ...target,
+        context,
+        voice: this.state.settings.voice,
+        instructions: speechInstructions(this.instructions(), this.state.config),
+        tools: this.tools(),
+        gate: this.gate(),
+        greeting: ["TALK", "TALKFIRST"].includes(this.state.config.startSessionMode.toUpperCase())
+          ? "hola vocero"
+          : undefined,
+      });
+      this.client = client;
+      this.subscriptions = [
+        client.on("status", (status) => {
+          if (client !== this.client) {
+            return;
+          }
+          if (status === "closed") {
+            void this.stop(this.state.error ? "error" : "closed");
+            return;
+          }
+          if (status === "listening" || status === "user-speaking" || status === "ai-speaking") {
+            this.lastActivity = Date.now();
+          }
+          if (status === "user-speaking") {
+            window.speechSynthesis?.cancel();
+            this.update({
+              messages: this.state.messages.map((message) =>
+                message.role === "assistant" && message.partial
+                  ? { ...message, partial: false, interrupted: true }
+                  : message,
+              ),
+            });
+          }
+          this.update({ status });
+        }),
+        client.on("transcript", (transcript) => {
+          this.lastActivity = Date.now();
+          this.transcript(transcript);
+          if (transcript.role === "user" && !transcript.partial && !this.reasoned.has(transcript.id)) {
+            this.reasoned.add(transcript.id);
+            void this.reason(transcript, client);
+          }
+        }),
+        client.on("finished", (response) => {
+          if (response.transcript) {
+            this.transcript(
+              { id: response.responseId, role: "assistant", text: response.transcript, partial: false },
+              response.status !== "completed",
+            );
+          }
+        }),
+        client.on("level", (level) => {
+          if (!this.state.muted && level > 0.0025) {
+            this.lastActivity = Date.now();
+          }
+          if (Date.now() - this.lastMeter > 150) {
+            this.lastMeter = Date.now();
+            this.update({ level });
+          }
+        }),
+        client.on("queue", ({ position }) => {
+          this.update({ queuePosition: position });
+        }),
+        client.on("readyToJoin", ({ expiresSec }) => {
+          this.update({ joinDeadline: Date.now() + expiresSec * 1000 });
+        }),
+        client.on("grant", ({ remainingSec }) => {
+          this.update({ remainingSec: remainingSec ?? null, joinDeadline: 0 });
+        }),
+        client.on("error", (error) => {
+          this.reportError(error);
+        }),
+        client.on("serverError", (error) => {
+          this.update({ error: error.message });
+        }),
+        client.on("tool", (call) => {
+          void this.runTool(call, client);
+        }),
+      ];
+      this.lastActivity = Date.now();
+      this.idleTimer = setInterval(() => {
+        if (
+          ["listening", "user-speaking", "processing"].includes(this.state.status) &&
+          Date.now() - this.lastActivity >= 30000
+        ) {
+          void this.stop();
+          this.update({ notice: "Sesión finalizada por inactividad. Toca para volver a conversar." });
+        }
+      }, 1000);
+      try {
+        await client.connect();
+      } catch (error) {
+        // A closed event may already have cleared the client; explicit cancellation is silent.
+        if (!(error instanceof SessionError && error.code === "aborted") && (!this.client || this.client === client)) {
+          this.reportError(error);
+          if (automatic) {
+            this.update({ unlock: true });
+          }
+        }
+      }
+    } catch (error) {
+      if (context) {
+        void context.close().catch(() => undefined);
+      }
+      this.reportError(error);
+    }
+  }
+  private reportError(error: unknown) {
+    this.update({
+      error: errorMessage(error),
+      status: "error",
+      limitReached: error instanceof SessionError && error.code === "limit",
+    });
+  }
+  private transcript(transcript: Transcript, interrupted = false) {
+    if (!transcript.text.trim()) {
+      return;
+    }
+    const id = `${String(this.sessionNumber)}:${transcript.role}:${transcript.id}`;
+    const existing = this.state.messages.find((message) => message.id === id);
+    const message: Message = {
+      ...existing,
+      id,
+      role: transcript.role,
+      text: transcript.text,
+      partial: transcript.partial,
+      timestamp: existing?.timestamp ?? new Date().toISOString(),
+      interrupted: interrupted || existing?.interrupted,
+    };
+    this.update({
+      messages: existing
+        ? this.state.messages.map((item) => (item.id === id ? message : item))
+        : [...this.state.messages, message],
+    });
+  }
+  private async reason(transcript: Transcript, client: RealtimeClient) {
+    const { config, profile, rag } = this.state;
+    if (config.reasoningType === "PROXY" || !transcript.text.trim()) {
+      return;
+    }
+    try {
+      const result = await post(
+        "/api/reasoning",
+        reasoningSchema,
+        {
+          transcript: transcript.text.trim(),
+          instructions: this.instructions(),
+          language: "es-ES",
+          user_context: [config.injectProfile ? profileContext(profile) : "", ragContext(rag)]
+            .filter(Boolean)
+            .join("\n\n"),
+          phoneNumber: this.caller.phoneNumber || null,
+          episode_id: this.caller.episodeId || null,
+        },
+        this.requests.signal,
+      );
+      if (this.client !== client || result.log_only) {
+        return;
+      }
+      const answer = result.answer.trim();
+      if (!answer) {
+        throw new Error("El servicio de razonamiento no devolvió una respuesta.");
+      }
+      client.renderExternalAnswer(answer);
+      this.transcript({ role: "assistant", id: `reasoning:${transcript.id}`, text: answer, partial: false });
+      if ("speechSynthesis" in window) {
+        const utterance = new SpeechSynthesisUtterance(answer);
+        utterance.lang = "es-ES";
+        const voice = window.speechSynthesis.getVoices().find((voice) => voice.lang.startsWith("es"));
+        if (voice) {
+          utterance.voice = voice;
+        }
+        utterance.onstart = () => {
+          if (this.client === client) {
+            this.update({ status: "ai-speaking" });
+          }
+        };
+        utterance.onend = utterance.onerror = () => {
+          if (this.client === client) {
+            this.lastActivity = Date.now();
+            this.update({ status: "listening" });
+          }
+        };
+        window.speechSynthesis.cancel();
+        window.speechSynthesis.speak(utterance);
+      }
+    } catch (error) {
+      if (this.client === client && !this.requests.signal.aborted) {
+        this.update({ error: errorMessage(error) });
+      }
+    }
+  }
+  private async runTool(call: ToolCall, client: RealtimeClient) {
+    const id = `${String(this.sessionNumber)}:tool:${call.callId}`;
+    let args = call.arguments;
+    let output: string;
+    try {
+      const parsed: unknown = JSON.parse(call.arguments || "{}");
+      args = JSON.stringify(parsed, null, 2);
+      if (call.name !== "web_search") {
+        throw new Error(`Herramienta desconocida: ${call.name}`);
+      }
+      const query = z.object({ query: z.string().trim().min(1) }).parse(parsed).query;
+      const result = await post(
+        "/api/search",
+        searchSchema,
+        {
+          query,
+          ...(!this.state.config.search && this.state.settings.searchKey ? { key: this.state.settings.searchKey } : {}),
+        },
+        this.requests.signal,
+      );
+      output = [
+        `Google search result from ${new Date().toISOString().slice(0, 10)}:`,
+        result.answer ? `Answer: ${result.answer}` : "",
+        ...result.results.map((item) => `- ${item.title}: ${item.snippet} (${item.url})`),
+      ]
+        .filter(Boolean)
+        .join("\n");
+      if (!result.answer && !result.results.length) {
+        output += "\nNo results found.";
+      }
+    } catch (error) {
+      output = `Tool error: ${errorMessage(error)}`;
+    }
+    if (client !== this.client) {
+      return;
+    }
+    this.update({
+      messages: [
+        ...this.state.messages,
+        {
+          id,
+          role: "tool",
+          text: output,
+          toolName: call.name,
+          toolArguments: args,
+          partial: false,
+          timestamp: new Date().toISOString(),
+        },
+      ],
+    });
+    client.sendToolOutput(call.callId, output);
+  }
+  join() {
+    this.client?.join();
+    this.update({ joinDeadline: 0 });
+  }
+  toggleMuted() {
+    const muted = !this.state.muted;
+    this.client?.setMuted(muted);
+    this.update({ muted });
+  }
+  async restart() {
+    await this.stop();
+    await this.start();
+  }
+  async stop(status: ConnectionStatus = "closed") {
+    const client = this.client;
+    this.client = null;
+    this.subscriptions.forEach((unsubscribe) => {
+      unsubscribe();
+    });
+    this.subscriptions = [];
+    clearInterval(this.idleTimer);
+    this.requests.abort();
+    window.speechSynthesis?.cancel();
+    this.update({
+      status,
+      muted: false,
+      level: 0,
+      joinDeadline: 0,
+      remainingSec: null,
+      unlock: false,
+      messages: this.state.messages.map((message) =>
+        message.partial ? { ...message, partial: false, interrupted: message.role === "assistant" } : message,
+      ),
+    });
+    await client?.close();
+  }
+}