462 lines
21 KiB
Python
462 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from typing import Any, cast
|
|
|
|
from s2s_client.core import logging as app_logging
|
|
from s2s_client.services import context as services_context
|
|
from s2s_client.services import tool_context as services_tool_context
|
|
|
|
|
|
async def _report_server_debug(hypothesis_id: str, location: str, msg: str, data: dict | None = None) -> None:
|
|
return
|
|
|
|
|
|
def _realtime_event_summary(raw: str) -> dict | None:
|
|
try:
|
|
payload = json.loads(raw)
|
|
except Exception:
|
|
return None
|
|
if not isinstance(payload, dict):
|
|
return None
|
|
event_type = payload.get("type")
|
|
if not isinstance(event_type, str):
|
|
return None
|
|
summary: dict[str, Any] = {"type": event_type}
|
|
if event_type == "session.update":
|
|
session = cast(dict[str, Any], payload.get("session")) if isinstance(payload.get("session"), dict) else {}
|
|
tools = cast(list[Any], session.get("tools")) if isinstance(session.get("tools"), list) else []
|
|
summary["toolCount"] = len(tools)
|
|
summary["toolNames"] = [tool.get("name") for tool in tools if isinstance(tool, dict)]
|
|
summary["toolChoice"] = session.get("tool_choice")
|
|
summary["instructionLength"] = len(session.get("instructions") or "")
|
|
elif event_type in {"response.function_call_arguments.done", "error", "session.updated"}:
|
|
for key in ("name", "call_id", "code", "message", "response_id"):
|
|
if key in payload:
|
|
summary[key] = payload.get(key)
|
|
# --- INPUT (user speech → STT transcript) ---
|
|
# Can arrive from upstream (server-side STT) — match either direction.
|
|
elif event_type == "conversation.item.input_audio_transcription.completed":
|
|
summary["transcript"] = payload.get("transcript", "")
|
|
summary["item_id"] = payload.get("item_id")
|
|
elif event_type == "conversation.item.created":
|
|
item = payload.get("item") if isinstance(payload.get("item"), dict) else None
|
|
if isinstance(item, dict):
|
|
summary["item_id"] = item.get("id")
|
|
summary["role"] = item.get("role")
|
|
summary["item_type"] = item.get("type")
|
|
summary["status"] = item.get("status")
|
|
content_parts = []
|
|
for c in item.get("content") or []:
|
|
if isinstance(c, dict):
|
|
if c.get("type") in {"input_text", "text"} and isinstance(c.get("text"), str):
|
|
content_parts.append(c["text"])
|
|
elif c.get("type") == "transcript" and isinstance(c.get("transcript"), str):
|
|
content_parts.append(c["transcript"])
|
|
if content_parts:
|
|
summary["text"] = " ".join(content_parts)
|
|
# --- OUTPUT (agent text/speech generation) ---
|
|
elif event_type == "response.output_item.done":
|
|
item = payload.get("item") if isinstance(payload.get("item"), dict) else None
|
|
if isinstance(item, dict):
|
|
summary["item_id"] = item.get("id")
|
|
summary["role"] = item.get("role")
|
|
summary["status"] = item.get("status")
|
|
content_parts = []
|
|
for c in item.get("content") or []:
|
|
if isinstance(c, dict):
|
|
if c.get("type") == "text" and isinstance(c.get("text"), str) and c["text"]:
|
|
content_parts.append(c["text"])
|
|
elif c.get("type") == "audio" and isinstance(c.get("transcript"), str) and c["transcript"]:
|
|
content_parts.append(c["transcript"])
|
|
if content_parts:
|
|
summary["text"] = " ".join(content_parts)
|
|
elif event_type == "response.content_part.done":
|
|
part = payload.get("part") if isinstance(payload.get("part"), dict) else None
|
|
if isinstance(part, dict):
|
|
summary["response_id"] = payload.get("response_id")
|
|
summary["item_id"] = payload.get("item_id")
|
|
summary["output_index"] = payload.get("output_index")
|
|
summary["content_index"] = payload.get("content_index")
|
|
if part.get("type") == "text":
|
|
summary["text"] = part.get("text", "")
|
|
elif part.get("type") == "audio":
|
|
if isinstance(part.get("transcript"), str) and part["transcript"]:
|
|
summary["text"] = part["transcript"]
|
|
# Standard OpenAI variant:
|
|
elif event_type == "response.audio_transcript.done":
|
|
summary["response_id"] = payload.get("response_id")
|
|
summary["item_id"] = payload.get("item_id")
|
|
summary["text"] = payload.get("transcript", "")
|
|
# Custom upstream variant observed in user logs:
|
|
elif event_type == "response.output_audio_transcript.done":
|
|
summary["response_id"] = payload.get("response_id")
|
|
summary["item_id"] = payload.get("item_id")
|
|
summary["output_index"] = payload.get("output_index")
|
|
# Try field names in order of likelihood
|
|
text = ""
|
|
for key in (
|
|
"transcript",
|
|
"text",
|
|
"output_text",
|
|
"audio_text",
|
|
"output_transcript",
|
|
"final_transcript",
|
|
"output_audio_text",
|
|
"content",
|
|
"message",
|
|
"utterance",
|
|
):
|
|
v = payload.get(key)
|
|
if isinstance(v, str) and v.strip():
|
|
text = v.strip()
|
|
break
|
|
# If text is still empty, search known nested objects.
|
|
if not text:
|
|
for container in ("item", "part", "content", "output", "delta", "audio"):
|
|
sub = payload.get(container)
|
|
if isinstance(sub, dict):
|
|
for key in ("transcript", "text", "output_text"):
|
|
v = sub.get(key)
|
|
if isinstance(v, str) and v.strip():
|
|
text = v.strip()
|
|
break
|
|
if text:
|
|
break
|
|
if not text:
|
|
# Last resort: the content[] array.
|
|
arr = payload.get("content")
|
|
if isinstance(arr, list):
|
|
parts = []
|
|
for c in arr:
|
|
if isinstance(c, dict):
|
|
for key in ("transcript", "text"):
|
|
v = c.get(key)
|
|
if isinstance(v, str) and v.strip():
|
|
parts.append(v.strip())
|
|
if parts:
|
|
text = " ".join(parts)
|
|
summary["text"] = text
|
|
# If no text was found, retain a raw preview to inspect the custom format.
|
|
if not text:
|
|
preview = raw[:800]
|
|
try:
|
|
pretty = json.dumps(payload, ensure_ascii=True)[:800]
|
|
except Exception:
|
|
pretty = preview
|
|
summary["_raw_preview"] = pretty
|
|
elif event_type == "response.function_call_arguments.done":
|
|
summary["arguments"] = payload.get("arguments")
|
|
# This event confirms strategy B: ai-server accepted native tools[].
|
|
_tool_name = payload.get("name") if isinstance(payload.get("name"), str) else ""
|
|
_call_id = payload.get("call_id") if isinstance(payload.get("call_id"), str) else ""
|
|
_args_json = cast(str, payload.get("arguments")) if isinstance(payload.get("arguments"), str) else "{}"
|
|
app_logging.logger.info(
|
|
"[TOOLCALL] ✓ UPSTREAM emitted response.function_call_arguments.done! "
|
|
"tool=%s callId=%s argsLen=%d argsPreview=%s",
|
|
_tool_name or "(empty)",
|
|
_call_id or "-",
|
|
len(_args_json),
|
|
_args_json[:200],
|
|
)
|
|
try:
|
|
_args_parsed = json.loads(_args_json or "{}")
|
|
app_logging.logger.info(
|
|
"[TOOLCALL] ✓ Tool call parsed argsKeys=%s",
|
|
sorted(_args_parsed.keys()) if isinstance(_args_parsed, dict) else "N/A",
|
|
)
|
|
except Exception:
|
|
pass
|
|
return summary
|
|
|
|
|
|
# Suppress noisy high-frequency events in detailed logs. Internal debug
|
|
# reporting still tracks them when enabled.
|
|
_NOISY_REALTIME_EVENT_TYPES = frozenset(
|
|
{
|
|
"response.output_audio.delta",
|
|
"response.audio.delta",
|
|
"conversation.item.input_audio_transcription.delta",
|
|
"response.text.delta",
|
|
"response.output_text.delta",
|
|
"rate_limits.updated",
|
|
}
|
|
)
|
|
|
|
# Suppress client-side high-frequency events such as input_audio_buffer.append.
|
|
_NOISY_CLIENT_EVENT_TYPES = frozenset(
|
|
{
|
|
"input_audio_buffer.append",
|
|
"input_audio_buffer.commit",
|
|
"input_audio_buffer.clear",
|
|
"input_audio_buffer.speech_started",
|
|
"input_audio_buffer.speech_stopped",
|
|
}
|
|
)
|
|
|
|
|
|
def _extract_profile_for_session(session_ctx: dict) -> dict:
|
|
"""Return the profile dict stored in session_ctx (phone auth result)."""
|
|
p = session_ctx.get("profile_obj") or session_ctx.get("user_profile") or {}
|
|
if isinstance(p, dict) and isinstance(p.get("user"), dict):
|
|
return cast(dict[str, Any], p["user"])
|
|
if isinstance(p, dict):
|
|
return p
|
|
return {}
|
|
|
|
|
|
def _sanitize_agent_output(text: Any, session_ctx: dict[str, Any] | None = None) -> str:
|
|
"""Server-side FINAL filter applied to ALL assistant output text.
|
|
|
|
Guarantees (belt-and-suspenders over the 6 prompt layers):
|
|
1. No [...][xxx] bracket markers of any kind.
|
|
2. No inline { ... } JSON / dict literals.
|
|
3. No [-]: <internal action name> pseudo-markers.
|
|
4. No name anglicization: Alessandro → restored to Alessandro.
|
|
5. Trim + collapse multiple spaces.
|
|
"""
|
|
s = str(text or "")
|
|
if not s:
|
|
return s
|
|
# (0) Remove "action line" pseudo-markers FIRST, BEFORE stripping all
|
|
# other brackets, because the pattern uses the literal "[-]" prefix
|
|
# which would get silently swallowed by step (1) otherwise. Match:
|
|
# [-]: checking_podcast_report (bracket literal then colon)
|
|
# []: action_name (empty bracket variant)
|
|
# [ - ] : do_stuff (spaces inside & around)
|
|
# Pattern is intentionally TIGHT: after the colon we only take up to 6
|
|
# underscore/hyphen words so we cannot accidentally eat the following
|
|
# real user speech (bug: greedy class with ,. and 200-char upper bound ate
|
|
# the entire "Claro, revisaré el reporte." clause on 2026-09-11).
|
|
s = re.sub(
|
|
r"\[[\s\-]{0,6}\]\s*:\s*[a-z0-9_\-]+(?:[ \t]+[a-z0-9_\-]+){0,5}",
|
|
" ",
|
|
s,
|
|
)
|
|
# Also catch dangling residual patterns if brackets got normalized to a
|
|
# single char already — handles cases where upstream writes the "colon"
|
|
# action marker without explicit brackets.
|
|
s = re.sub(r"(?<!\w):\s*checking_[A-Za-z0-9_\-]{1,120}", " ", s)
|
|
# (1) Remove ALL bracket-markers — [rightspeak], [endspeak], [thinking],
|
|
# [anything_else01]. We also remove stray [ / ] that don't form a
|
|
# complete pair (e.g. leftover "hello] there").
|
|
s = re.sub(r"\[[^\]\n]{0,80}\]", " ", s)
|
|
# (2) Remove inline dict / JSON literals. Pattern: { ... }, possibly
|
|
# split across a few \n (not greedy). Conservative: matches balanced
|
|
# pairs up to a reasonable length, otherwise fallback.
|
|
_max_iter = 5
|
|
for _ in range(_max_iter):
|
|
new_s = re.sub(r"\{[^{}]{0,1600}\}", " ", s)
|
|
if new_s == s:
|
|
break
|
|
s = new_s
|
|
# (3) Name anglicization. Iterate profile name's known variants →
|
|
# replace with original (case-insensitive match).
|
|
if session_ctx is not None and isinstance(session_ctx, dict):
|
|
profile = _extract_profile_for_session(session_ctx)
|
|
if profile:
|
|
full_name = str(profile.get("name") or "").strip()
|
|
if full_name:
|
|
# Split into parts and check each
|
|
parts = [p for p in re.split(r"[\s\-_]+", full_name) if p]
|
|
for part in parts:
|
|
variants = services_context._common_name_variants(part)
|
|
orig_cap = part[0].upper() + part[1:] if part else part
|
|
for variant in variants:
|
|
# Case-insensitive global replace → original case
|
|
s = re.sub(
|
|
r"\b" + re.escape(variant) + r"\b",
|
|
orig_cap,
|
|
s,
|
|
flags=re.IGNORECASE,
|
|
)
|
|
# Also try full-name variants (combined)
|
|
combined = full_name.replace(" ", "") # noqa: F841
|
|
# (5) Collapse whitespace. Re-punctuation cleanup: avoid normalizing
|
|
# legitimate terminal "?" / "!" into nothing; only collapse doubled
|
|
# or space-separated punctuation that resulted from earlier deletes.
|
|
s = re.sub(r"[ \t\u00a0]+", " ", s)
|
|
s = re.sub(r"\n{3,}", "\n\n", s)
|
|
# Interior punctuation: normalize spaces, but keep the char itself.
|
|
# Example: "hello , world" → "hello, world"
|
|
s = re.sub(r"\s+([,.;:])\s+", r"\1 ", s)
|
|
# Terminal / exclamation / interrogation: only fix spacing, never strip.
|
|
s = re.sub(r"\s+([!?])", r"\1", s)
|
|
s = re.sub(r"([!?])\s{2,}", r"\1 ", s)
|
|
# Strip leftover whitespace. Do NOT strip the legitimate terminal
|
|
# punctuation chars ?, !, ., because they are part of the real speech.
|
|
# Only strip repeated trailing commas / semicolons that appear because
|
|
# we deleted bracket/JSON fragments that separated clauses.
|
|
s = s.strip()
|
|
# Strip 2+ identical terminal punctuation chars (e.g. "hello.." → "hello.")
|
|
s = re.sub(r"([,.;:!?])\1{1,}$", r"\1", s)
|
|
# Strip terminal comma / semicolon / colon that now dangle (clause joins)
|
|
# but preserve ., ?, !.
|
|
s = re.sub(r"[,;:]$", "", s).strip()
|
|
return s
|
|
|
|
|
|
def _sanitize_realtime_event_raw(raw_message: str, session_ctx: dict[str, Any] | None = None) -> tuple[str, bool]:
|
|
"""If the raw upstream event contains assistant OUTPUT text, sanitize it
|
|
in-place and return (new_json_str, True) — otherwise return (raw, False).
|
|
|
|
Mirrors exactly the same traversal used by _realtime_event_summary so we
|
|
touch every field that would become summary["text"].
|
|
"""
|
|
import json as _json
|
|
|
|
if not isinstance(raw_message, str) or not raw_message.strip():
|
|
return raw_message, False
|
|
try:
|
|
payload = _json.loads(raw_message)
|
|
except Exception:
|
|
return raw_message, False
|
|
if not isinstance(payload, dict):
|
|
return raw_message, False
|
|
event_type = payload.get("type")
|
|
if not isinstance(event_type, str):
|
|
return raw_message, False
|
|
|
|
changed = False
|
|
|
|
def _sanitize_value(v: Any) -> Any:
|
|
nonlocal changed
|
|
if isinstance(v, str) and v.strip():
|
|
cleaned = _sanitize_agent_output(v, session_ctx)
|
|
# Layer 8 removes internal ApiPilot CALL markers before UI/TTS.
|
|
# The pass-2 accumulator retains the original transcript so
|
|
# execution can still be triggered by its marker.
|
|
cleaned = services_tool_context._sanitize_text_strip_apipilot_markers(cleaned)
|
|
if cleaned != v:
|
|
changed = True
|
|
return cleaned
|
|
return v
|
|
|
|
# --- Event dispatch (same shape as _realtime_event_summary) -------------
|
|
# Sanitize streaming deltas as well as .done events. Qwen3-TTS reads deltas
|
|
# before generating binary audio and could otherwise pronounce the marker.
|
|
if event_type in {"response.audio_transcript.done", "response.audio_transcript.delta"}:
|
|
if "transcript" in payload:
|
|
payload["transcript"] = _sanitize_value(payload["transcript"])
|
|
if "delta" in payload and isinstance(payload.get("delta"), str):
|
|
payload["delta"] = _sanitize_value(payload["delta"])
|
|
for container in ("item", "part", "content", "output", "audio"):
|
|
sub = payload.get(container)
|
|
if isinstance(sub, dict):
|
|
for key in ("transcript", "text", "delta"):
|
|
if key in sub and isinstance(sub.get(key), str):
|
|
sub[key] = _sanitize_value(sub[key])
|
|
elif event_type in {"response.output_audio_transcript.done", "response.output_audio_transcript.delta"}:
|
|
# Top-level keys
|
|
for key in (
|
|
"transcript",
|
|
"text",
|
|
"output_text",
|
|
"audio_text",
|
|
"output_transcript",
|
|
"final_transcript",
|
|
"output_audio_text",
|
|
"content",
|
|
"message",
|
|
"utterance",
|
|
"delta",
|
|
):
|
|
if key in payload and isinstance(payload.get(key), str):
|
|
payload[key] = _sanitize_value(payload[key])
|
|
# Sub-containers
|
|
for container in ("item", "part", "content", "output", "delta", "audio"):
|
|
sub = payload.get(container)
|
|
if isinstance(sub, dict):
|
|
for key in ("transcript", "text", "output_text", "delta"):
|
|
if key in sub and isinstance(sub.get(key), str):
|
|
sub[key] = _sanitize_value(sub[key])
|
|
# content[] array
|
|
if isinstance(sub.get("content"), list):
|
|
for c in sub["content"]:
|
|
if isinstance(c, dict):
|
|
for k2 in ("transcript", "text", "delta"):
|
|
if k2 in c and isinstance(c.get(k2), str):
|
|
c[k2] = _sanitize_value(c[k2])
|
|
if isinstance(payload.get("content"), list):
|
|
for c in payload["content"]:
|
|
if isinstance(c, dict):
|
|
for k2 in ("transcript", "text", "delta"):
|
|
if k2 in c and isinstance(c.get(k2), str):
|
|
c[k2] = _sanitize_value(c[k2])
|
|
elif event_type in {
|
|
"response.text.delta",
|
|
"response.text.done",
|
|
"response.output_text.delta",
|
|
"response.output_text.done",
|
|
}:
|
|
for key in ("text", "output_text", "delta", "final_text", "content"):
|
|
if key in payload and isinstance(payload.get(key), str):
|
|
payload[key] = _sanitize_value(payload[key])
|
|
for container in ("item", "part", "output", "content", "text"):
|
|
sub = payload.get(container)
|
|
if isinstance(sub, dict):
|
|
for key in ("text", "output_text", "delta"):
|
|
if key in sub and isinstance(sub.get(key), str):
|
|
sub[key] = _sanitize_value(sub[key])
|
|
if isinstance(sub.get("content"), list):
|
|
for c in sub["content"]:
|
|
if isinstance(c, dict):
|
|
for k2 in ("text", "transcript", "delta"):
|
|
if k2 in c and isinstance(c.get(k2), str):
|
|
c[k2] = _sanitize_value(c[k2])
|
|
if isinstance(payload.get("content"), list):
|
|
for c in payload["content"]:
|
|
if isinstance(c, dict):
|
|
for k2 in ("text", "transcript", "delta"):
|
|
if k2 in c and isinstance(c.get(k2), str):
|
|
c[k2] = _sanitize_value(c[k2])
|
|
elif event_type == "response.content_part.done":
|
|
part = payload.get("part")
|
|
if isinstance(part, dict):
|
|
if "text" in part and isinstance(part.get("text"), str):
|
|
part["text"] = _sanitize_value(part["text"])
|
|
if "transcript" in part and isinstance(part.get("transcript"), str):
|
|
part["transcript"] = _sanitize_value(part["transcript"])
|
|
elif event_type == "response.content_part.added":
|
|
part = payload.get("part")
|
|
if isinstance(part, dict):
|
|
if "text" in part and isinstance(part.get("text"), str):
|
|
part["text"] = _sanitize_value(part["text"])
|
|
if "transcript" in part and isinstance(part.get("transcript"), str):
|
|
part["transcript"] = _sanitize_value(part["transcript"])
|
|
elif event_type in {
|
|
"response.content_part.text.delta",
|
|
"response.output_content.text.delta",
|
|
"response.content.text.delta",
|
|
}:
|
|
for key in ("text", "delta"):
|
|
if key in payload and isinstance(payload.get(key), str):
|
|
payload[key] = _sanitize_value(payload[key])
|
|
elif event_type in {"response.output_item.done", "response.output_item.added"}:
|
|
item = payload.get("item")
|
|
if isinstance(item, dict):
|
|
for c in item.get("content") or []:
|
|
if isinstance(c, dict):
|
|
if "text" in c and isinstance(c["text"], str):
|
|
c["text"] = _sanitize_value(c["text"])
|
|
if "transcript" in c and isinstance(c["transcript"], str):
|
|
c["transcript"] = _sanitize_value(c["transcript"])
|
|
elif event_type == "conversation.item.created":
|
|
item = payload.get("item")
|
|
if isinstance(item, dict) and item.get("role") == "assistant":
|
|
for c in item.get("content") or []:
|
|
if isinstance(c, dict):
|
|
if c.get("type") in {"input_text", "text"} and isinstance(c.get("text"), str):
|
|
c["text"] = _sanitize_value(c["text"])
|
|
if c.get("type") == "transcript" and isinstance(c.get("transcript"), str):
|
|
c["transcript"] = _sanitize_value(c["transcript"])
|
|
|
|
if not changed:
|
|
return raw_message, False
|
|
try:
|
|
new_raw = _json.dumps(payload, ensure_ascii=False)
|
|
except Exception:
|
|
return raw_message, False
|
|
return new_raw, True
|