Files
vocero-client/vocero-api/vocero_api/services/context.py
T
alvaro dfed03ac7c chore(vocero): remove the legacy ApiPilot integration (#3)
Vocero now retrieves AquaPortal content through authenticated MCP. This removes the obsolete ApiPilot integration from the API and browser while preserving the separately deployed speech pipeline.

Release notes:

- Removed ApiPilot endpoints, configuration, credential overrides, and HTTP/realtime tool execution.
- Removed browser ApiPilot catalog loading and prompt generation while retaining authenticated MCP discovery.
- Updated regression coverage and operating documentation for the retired routes and settings.

Reviewed-on: AquaPortal/s2s#3
2026-10-08 11:23:41 -04:00

1152 lines
54 KiB
Python

from __future__ import annotations
import datetime as _dt_module
import json
from typing import Any, cast
from vocero_api.core import config as settings
from vocero_api.core import logging as app_logging
VOCERO_USER_CONTEXT_HINT = " Caller profile resolved from the page phoneNumber: "
REPORTS_LIST_HINT = (
" Scheduled reports / podcast scripts available for this caller before the current session. "
"Each entry is a scheduled report; use the podcastScript contents as trusted reference material when the user asks about scheduled topics, reports, or prior content."
" NOTE (2026-09-11 request to remove mass injection of scheduled reports): "
"S2S_INJECT_REPORTS defaults to FALSE. When FALSE the injection below NEVER happens. "
"Operators must explicitly set S2S_INJECT_REPORTS=true to opt back into mass scheduled-report prompt injection. "
)
LATEST_SUMMARY_HINT = " Latest conversation summary for this caller (most recent before the current session): "
PODCAST_EPISODE_HINT = (
" Podcast episode context loaded from the page episode_id query param (current episode the user is referencing). "
"Treat the episode fields (title, description, transcript, script, show notes, guests, audio metadata, etc.) as the PRIMARY trusted reference material for this session. "
)
# ──────────────────────────────────────────────────────────────────────────
# User fix 2026-09-11 Bug 3: "Qwen2.5 has training cutoff mid-2024, when it
# sees 39 reports dated 2026-09-10 it hallucinates meta-comments in Chinese
# like 'today is 2023, why are you simulating 2026 reports?'"
#
# Resolved with S2S_INJECT_TODAY default ON + S2S_REPORTS_INJECT_MODE=minimal:
# • TODAY_BLOCK (marker <<<TODAY_DATE_BEGIN>>>):
# - Fixed reference: "TODAY is YYYY-MM-DD".
# - Earlier reports are historical records; do not question their dates.
# - No internal commentary about prompts, dates, data validity or instructions.
# - Clean output: only words Vocero says aloud; no reasoning or prompt checks.
# - Retrieve report details through RAG rather than inventing from summaries.
# • MINIMAL_REPORTS_BLOCK (marker <<<MINIMAL_REPORTS_TOC>>>): only latest N
# label + date lines. NO podcastScript text, NO full JSON entries. Uses the
# pattern the user explicitly requested:
# Recent scheduled reports (latest 3):
# 1. "Mortality Percentage Report" — 2026-09-10
# 2. ...
# (Retrieve full content through knowledge search.)
# • We register both blocks with the dedupe layer via the new TYPE IDs
# "today_fixed" and "reports_minimal" so they also can never get duplicated.
#
# IMPORTANT MARKER CHANGE 2026-09-11: markers used to be human-readable
# strings like "<<<TODAY_DATE_BEGIN>>>" which got written into the LLM
# prompt verbatim. Qwen2.5 (especially GPTQ quantized variants) would
# sometimes *leak those markers back into the output* alongside meta-comment
# hallucinations (e.g. emitting "[-]: checking_podcast_report" or literal
# "<<<...>>>"). To fix this we now use INVISIBLE markers: a SHA-1 hash of
# stable strings. These hashes are NEVER seen by the LLM as text because
# they are NOT written into the prompt. They exist ONLY as Python-side
# substring sentinels prepended to the *Python string* returned by each
# block builder — stripped out at the very end with .replace() just before
# the string is assembled into the prompt. Result: the dedupe logic still
# works perfectly, but the LLM NEVER sees any marker / guard token.
# ──────────────────────────────────────────────────────────────────────────
def _s2s_marker(label: str) -> str:
"""Return a short, unique, invisible sentinel for dedup classification.
Sentinel is 40 hex chars (SHA1) of a stable string; it is NEVER written
into the prompt — we strip all markers at the end of each builder.
"""
import hashlib
return hashlib.sha1(f"S2S-ctx-marker/{label}/v1".encode()).hexdigest()
TODAY_BLOCK_MARKER = _s2s_marker("today_fixed_block")
MINIMAL_REPORTS_MARKER = _s2s_marker("reports_minimal_toc")
VOICE_RULES_HEAD_MARKER = _s2s_marker("voice_rules_head_lock")
REPORTS_FULL_MARKER = _s2s_marker("reports_full_block")
SUMMARY_FULL_MARKER = _s2s_marker("summary_full_block")
PROFILE_FULL_MARKER = _s2s_marker("profile_full_block")
EPISODE_FULL_MARKER = _s2s_marker("episode_full_block")
ALL_S2S_INTERNAL_MARKERS: tuple = (
TODAY_BLOCK_MARKER,
MINIMAL_REPORTS_MARKER,
VOICE_RULES_HEAD_MARKER,
REPORTS_FULL_MARKER,
SUMMARY_FULL_MARKER,
PROFILE_FULL_MARKER,
EPISODE_FULL_MARKER,
)
def _strip_internal_markers(text: Any) -> str:
"""Strip every internal S2S dedup marker from the final prompt text.
Must be called right before a block is joined to the final instructions /
user_context so the LLM NEVER sees any of our sentinel strings. This is
the final guarantee that prevents marker leakage into spoken output.
"""
s = str(text or "")
if not s:
return s
for m in ALL_S2S_INTERNAL_MARKERS:
if m and m in s:
s = s.replace(m, "")
return s.strip()
def _extract_user_name_parts(profile: Any) -> list[str]:
"""Given a profile dict (possibly wrapped in {"user": ...} or {"result": ...})
return the list of non-empty first/last name parts in order (e.g.
["Alessandro", "Valenti"]). Returns [] when profile has no name.
Used by:
• _build_voice_rules_head_lock_block — prompt rule #5, exact name.
• _resolve_user_context_text — dynamic profile note against name translation.
• _sanitize_agent_output — layer 7 final whole-word regex replacement.
"""
if not profile or not isinstance(profile, dict):
return []
# Unwrap common container keys (server returns both shapes depending on
# the auth source path — direct /phone/ endpoint vs login proxy wrapper).
nested = profile
for wrapper in ("user", "result", "data"):
inner = nested.get(wrapper)
if isinstance(inner, dict):
nested = inner
# Prefer explicit firstName+lastName when available (structured fields)
first = str(nested.get("firstName") or nested.get("first_name") or "").strip()
last = str(nested.get("lastName") or nested.get("last_name") or "").strip()
if first or last:
out: list[str] = []
for token in (first, last):
token_clean = token.strip()
if token_clean:
for sub in token_clean.split():
if sub:
out.append(sub)
return out
# Fallback: split the combined "name" string by whitespace
full_name = str(nested.get("name") or "").strip()
if not full_name:
return []
return [p for p in full_name.split() if p]
def _build_voice_rules_head_lock_block(profile: Any = None) -> str:
"""Return the #1 PRIORITY block prepended to EVERY prompt: voice lock.
Fixes 6 hallucination categories seen in user log 2026-09-11 07:57:
(1) [-]: checking_podcast_report — internal "action thinking" leak
(2) English 'Sure, here's my best focused expression while I look into it'
(3) JSON literal {'reportDetails': '...'} inside output
(4) Unexpected Italian phrases.
(5) leak markers <<<...>>> in TTS output.
(6) Invented content about nonexistent podcasts.
This block contains short English instructions requiring Spanish output. We put
it as the VERY FIRST thing in the prompt (even before the caller profile),
because quantized models obey the first ~400 tokens way more strongly
than rules buried at the end.
2026-09-11: NEW rule #5 — PROPER NAME LOCK. Model keeps writing "Alexander"
for the Italian caller "Alessandro Valenti". We now emit an explicit,
non-ambiguous "Exact name: Alessandro. NEVER Alexander/Alexandre/..."
that the sanitizer's _common_name_variants map can also repair.
INVISIBLE marker: prefixes VOICE_RULES_HEAD_MARKER (stripped before prompt
join, never seen by LLM) to enable dedupe so we never double-prepend.
"""
base = (
"PRIMARY VOICE OUTPUT RULES (always apply; override all other instructions):\n"
"1) EXCLUSIVE LANGUAGE: ALWAYS respond ONLY in Spanish (Castilian). No Italian, English, Portuguese or other language. "
"Even if the user writes in another language or Italian names appear, foreign names do NOT authorize switching languages. "
"No words, phrases or confirmations in another language.\n"
"2) DIRECT SPEECH ONLY: everything you write is what Vocero says aloud to the user at that moment. "
"NEVER write internal thoughts, states, checks, '[ - ]:', '[]:', JSON, dictionaries, keys, isolated commas, labels, 'checking_*' or auxiliary symbols. "
"No '[rightspeak]', '[endspeak]', '[thinking]', '[endthinking]' or bracketed labels. No meta-comments about prompts or data.\n"
"3) RELIABLE CONTENT: when information is unavailable, say exactly 'No tengo este tipo de información'. Do not invent, assume or generalize. "
"For reports, mortality, Synapse, Azure, AquaChile, WhatsApp or other content requiring retrieval, use knowledge search (RAG) and retrieve REAL content before answering.\n"
"4) SHORT NATURAL RESPONSES: speak like a real person on the phone. No lists, bullet points, quotations or long text blocks. "
"Use a couple of short sentences, then ask whether the user wants more details.\n"
)
# Dynamic proper-name lock based on the profile.
name_lines = ""
parts = _extract_user_name_parts(profile)
if parts:
full_name = " ".join(parts)
variants_all: list[str] = []
for part in parts:
vs = _common_name_variants(part)
for v in vs:
if v not in variants_all and v != part:
variants_all.append(v)
cleaned_variants = variants_all[:8]
if cleaned_variants:
name_lines = (
"5) PROPER NAME (mandatory for this session): "
f"the user's exact name, WRITTEN AND PRONOUNCED AS GIVEN, is: {full_name}. "
f"NEVER adapt, translate or normalize it: "
+ ", ".join(f'NEVER "{v}"' for v in cleaned_variants)
+ ". Write and pronounce only the exact profile name.\n"
)
body = base + name_lines
return VOICE_RULES_HEAD_MARKER + "\n" + body
def _build_today_fixed_context_block(
today_override: _dt_module.date | None = None,
) -> str:
"""Return the TODAY + anti-hallucination guard block.
2026-09-11 rewrite:
• English internal instructions; user responses remain Spanish.
Earlier mixed Italian/Spanish prompts caused output language leaks.
• Only 3 bullet sentences, nothing fancy, nothing that triggers meta.
• Invisible dedup marker TODAY_BLOCK_MARKER is prefixed and will be
stripped before prompt join → LLM never sees.
"""
if not settings.S2S_INJECT_TODAY:
return ""
today = today_override if today_override is not None else _dt_module.date.today()
iso_date = today.strftime("%Y-%m-%d")
weekday = [
"Monday",
"Tuesday",
"Wednesday",
"Thursday",
"Friday",
"Saturday",
"Sunday",
][today.weekday()]
body = (
f"Today's date (mandatory reference): {iso_date} ({weekday}).\n"
"Reports dated BEFORE today are HISTORICAL RECORDS, not future dates. Do not comment on this.\n"
"A period '.' separates decimals; a comma ',' separates thousands (the standard international format used by this system).\n"
)
return TODAY_BLOCK_MARKER + "\n" + body
def _build_minimal_reports_toc_block(
filtered_sorted: list[dict[str, Any]] | None,
cap: int | None = None,
) -> str:
"""Return the user-requested MINIMAL scheduled-reports TOC block (NO script).
2026-09-11 rewrite:
• Marker <<<...>>> removed because it was leaking into output. Now the
invisible internal sentinel MINIMAL_REPORTS_MARKER (stripped before
joining) is used for dedupe classification.
• Footer is now the strict RAG rule, not a soft note.
"""
items = list(filtered_sorted or [])
if cap is None:
cap = max(1, int(settings.S2S_REPORTS_QUICK_HINT_MAX or 3))
if len(items) > cap:
items = items[:cap]
if not items:
return ""
lines: list[str] = []
lines.append(f"Recent scheduled reports (latest {len(items)}):")
for idx, r in enumerate(items, start=1):
label = str(r.get("sourceScheduleLabel") or "").strip() or "(untitled)"
raw_d = r.get("date")
date_str = ""
try:
if isinstance(raw_d, (int, float)) and not isinstance(raw_d, bool):
ts_sec = float(raw_d)
if ts_sec > 1e12:
ts_sec = ts_sec / 1000.0
dt = _dt_module.datetime.fromtimestamp(ts_sec, tz=_dt_module.UTC)
date_str = dt.strftime("%Y-%m-%d")
elif raw_d is not None:
s = str(raw_d).strip()
if "T" in s and len(s) >= 10:
date_str = s[:10]
else:
date_str = s
except Exception:
date_str = str(raw_d)[:10] if raw_d else ""
if date_str:
lines.append(f' {idx}. "{label}" — {date_str}')
else:
lines.append(f' {idx}. "{label}"')
lines.append(
"ALWAYS retrieve full report content through knowledge search (RAG). Do not invent details you have not retrieved."
)
return MINIMAL_REPORTS_MARKER + "\n" + "\n".join(lines) + "\n"
# ──────────────────────────────────────────────────────────────────────────
# 2026-09-11 user fix: remove duplicated caller profiles.
# Each context block returned by the _resolve_* helpers above starts with a
# unique, stable, documented HINT prefix (see VOCERO_USER_CONTEXT_HINT /
# PODCAST_EPISODE_HINT / REPORTS_LIST_HINT / LATEST_SUMMARY_HINT). We use
# these prefixes as logical "block type IDs" so a single context type is
# NEVER appended twice to the same prompt / session.instructions / user_context
# — even if two different call paths (phoneNumber fetch + identity /
# client-side supplied user_context + server-side fetch) both carry the same
# block type.
#
# Historical duplicate reported by the user:
# Caller profile resolved from the page phoneNumber: {name:"Alessandro.."}
# Caller profile resolved from the page phoneNumber: {name:"Alessandro.."}
# ↑ IDENTICAL SECOND COPY ↑
# The most likely cause is a client-side reasoning / profile fetch supplying
# the profile as part of `req.user_context` (or two overlapping server
# injection channels) while the server re-fetches it again for the
# instructions augment. Instead of trying to hunt every single call site, we
# add TWO dedupe layers below that are immune to future regressions:
#
# 1) _dedup_context_blocks_by_type([A, B, C, ...]) — given a list of
# candidate context strings, returns a new list where only the FIRST
# occurrence of each logical block TYPE (profile / episode / reports /
# summary) survives. All HINT prefixes are treated as block-type IDs.
# Used inside _augment_instructions_with_profile and the
# effective_user_context join list.
# 2) _strip_already_injected_types_from_base(base_text, [candidates]) —
# before appending new candidates to an existing base prompt string,
# scan the base for each HINT prefix. If a prefix is already present
# somewhere inside base_text, that candidate is DROPPED (skipped).
# Catches the case where session.instructions was ALREADY patched on a
# previous session.update frame, or where the client sent us the
# profile already baked into req.instructions / req.user_context.
#
# Together these two layers guarantee: per prompt / per LLM call, each block
# type appears AT MOST ONCE (zero or one).
# ──────────────────────────────────────────────────────────────────────────
_CONTEXT_DEDUPE_MARKERS: tuple = (
# (logical_type_id, unique_prefix_marker_in_text)
("voice_rules_head", VOICE_RULES_HEAD_MARKER), # NEW: 2026-09-11 #1 PRIORITY voice-language lock
("episode", PODCAST_EPISODE_HINT.strip()),
("profile", VOCERO_USER_CONTEXT_HINT.strip()),
("reports_full", REPORTS_FULL_MARKER),
("reports", "<<<REPORTS_LIST_BEGIN>>>"), # legacy back-compat marker
("summary", "<<<LATEST_SUMMARY_BEGIN>>>"), # legacy back-compat marker
("summary_full", SUMMARY_FULL_MARKER),
("today_fixed", TODAY_BLOCK_MARKER),
("reports_minimal", MINIMAL_REPORTS_MARKER),
)
def _classify_context_block(block: str) -> str | None:
"""Return the logical type of block or None for unrecognised text.
Uses the HINT prefixes / block begin markers above. Uses substring
containment (not startswith) because the augmented prompt may carry the
markers after 1-2 leading spaces. Uses the STRIPPED version of each HINT
to avoid missing matches due to leading whitespace.
"""
if not block:
return None
haystack = (block or "").strip()
if not haystack:
return None
for type_id, marker in _CONTEXT_DEDUPE_MARKERS:
if marker and marker in haystack:
return cast(str, type_id)
return None
def _dedup_context_blocks_by_type(candidates: list[str]) -> list[str]:
"""Keep only the FIRST occurrence of each logical block TYPE in order.
Drops duplicated block types silently and logs + returns a new list with
only the surviving blocks. Also drops empty / whitespace-only candidates
which would otherwise be a no-op append. This is layer 1 of the dedupe.
"""
if not candidates:
return []
seen_types: set[str] = set()
out: list[str] = []
dropped_types: dict[str, int] = {}
for candidate in candidates:
c = (candidate or "").strip()
if not c:
continue
type_id = _classify_context_block(c)
if type_id is None:
# Unknown type (free-form text, plain user_context, ...) is NEVER
# deduped — we only dedupe the 4 well-known S2S injected block
# types so user-supplied arbitrary context is always preserved.
out.append(candidate)
continue
if type_id in seen_types:
dropped_types[type_id] = dropped_types.get(type_id, 0) + 1
continue
seen_types.add(type_id)
out.append(candidate)
if dropped_types and settings.S2S_DEBUG_PROMPT:
# Log only when dedupe actually fired (avoids spamming). Use level
# DEBUG / info level so the admin can see "profile duplicate was
# dropped" when troubleshooting prompt size issues.
app_logging.logger.info(
"[INJECT-DEDUPE] dropped duplicate injected block types=%s kept=%d candidates=%d",
", ".join(f"{k}x{v}" for k, v in dropped_types.items()),
len(out),
len(candidates),
)
return out
def _strip_already_injected_types_from_base(base_text: str, candidates: list[str]) -> list[str]:
"""Layer 2: for each candidate, check if its HINT prefix ALREADY appears
anywhere inside base_text (i.e. was injected by a previous patch / baked
into the instructions sent by the browser). If yes, drop that candidate
entirely (keep only candidates whose types are NOT already present in
base). Works even when base_text contains the profile twice — after this
filter we won't add a third copy; layer 1 handles any further duplicates
among the candidates themselves."""
b = base_text or ""
if not b:
return list(candidates)
out: list[str] = []
dropped_base: list[str] = []
for candidate in candidates:
c = (candidate or "").strip()
if not c:
continue
type_id = _classify_context_block(c)
if type_id is None:
# Unknown context blocks → never dedupe, always append.
out.append(candidate)
continue
# Does base already contain ANY marker for this type? Use _ALL_
# markers for this type, not just the single canonical one, in case
# an older deploy used a different stringification (though prefix
# stability is guaranteed by the module-level constants here).
already = False
for marker_tid, marker in _CONTEXT_DEDUPE_MARKERS:
if marker_tid == type_id and marker and marker in b:
already = True
break
if already:
dropped_base.append(type_id)
continue
out.append(candidate)
if dropped_base and settings.S2S_DEBUG_PROMPT:
app_logging.logger.info(
"[INJECT-DEDUPE] dropped candidate block types already present in base instructions=%s baseLen=%d",
", ".join(dropped_base),
len(b or ""),
)
return out
def _resolve_user_context_text(phone_number: str | None, profile_obj: Any, loaded: bool = True) -> str:
"""Given a phone+profile object -> the short context string appended to
LLM system prompt / reasoning user_context when S2S_INJECT_PROFILE is on.
Generic (deploy-agnostic): profile_obj is the JSON returned by
VOCERO_USER_PHONE_AUTH_URL (optionally nested under {"result": ...}).
2026-09-11 anti-language-leak rule: names like "Alessandro Valenti" are
Italian/Luxembourgish names but our voice assistant uses Castilian Spanish, so
we append an EXPLICIT NOTE that the model sees alongside the caller info.
2026-09-11 anti-anglicization rule (Alexander → Alessandro): when the
profile contains a known given-name (e.g. Alessandro) we emit a DYNAMIC
list of forbidden anglicizations so the model cannot "normalize" it.
"""
if not loaded or not profile_obj or not phone_number:
return ""
try:
serialized = json.dumps(profile_obj, ensure_ascii=False)
except Exception:
serialized = str(profile_obj)
if len(serialized) > 1600:
serialized = serialized[:1600] + "..."
# ─── anti-anglicization dynamic note ─────────────────────────────────
parts = _extract_user_name_parts(profile_obj) or []
forbidden_lines: list[str] = []
for name_part in parts:
variants = _common_name_variants(name_part)
if variants:
# keep the first 6 variants max so the prompt stays tight
short = variants[:6]
quoted = ", ".join('"' + v + '"' for v in short)
forbidden_lines.append(
f"For '{name_part}': NEVER write or pronounce {quoted} or similar English/French/Russian variants. "
f"Use only '{name_part}' exactly as given."
)
if forbidden_lines:
proper_name_note = "PROPER NAME (mandatory for this session): " + " ".join(forbidden_lines) + " "
else:
# fallback to generic example so the rule's existence is still taught
proper_name_note = (
"PROPER NAME: never adapt, translate or normalize proper names "
"(for example, never change 'Alessandro' to 'Alexander'). Write and pronounce only the exact profile name. "
)
# We build with invisible internal PROFILE_FULL_MARKER so dedup + strip
# works consistently with today/voice_rules_head blocks.
body = (
f"{VOCERO_USER_CONTEXT_HINT}{serialized}. Use this as trusted caller context when relevant. "
"IMPORTANT LANGUAGE RULE: even if the user has an Italian name (such as 'Alessandro Valenti') "
"or a name from any other language, respond ONLY in SPANISH (Castilian). "
"Foreign names do NOT authorize switching languages. "
"Never write phrases, words, questions or confirmations in Italian, English or another language. "
"Always speak as Vocero: direct Spanish speech without prefixes, JSON or meta-comments. "
f"{proper_name_note}"
)
return PROFILE_FULL_MARKER + "\n" + body
def _resolve_latest_summary_context(summary_value: Any) -> str:
"""Normalize the payload returned by S2S_LATEST_SUMMARY endpoint into a
single context string to be appended next to the profile block.
Accepts plain string / object (dumped JSON) / None. Capped at 1600 chars.
2026-09-11: uses INVISIBLE SUMMARY_FULL_MARKER (never seen by LLM) so the
dedupe layer can still classify it; human <<<...>>> markers removed to
stop leakage into output.
Returns "" when no summary is available.
"""
if summary_value is None:
return ""
if isinstance(summary_value, dict) or isinstance(summary_value, list):
try:
serialized = json.dumps(summary_value, ensure_ascii=False)
except Exception:
serialized = str(summary_value)
else:
serialized = str(summary_value)
trimmed = serialized.strip()
if not trimmed:
return ""
if len(trimmed) > 1600:
trimmed = trimmed[:1600] + "..."
body = f"{LATEST_SUMMARY_HINT}\n{trimmed}\n. Use this as trusted history when the user references prior calls."
return SUMMARY_FULL_MARKER + "\n" + body
def _resolve_episode_context(raw_value: Any) -> str:
"""Normalize the podcast episode API response into a prompt block injected
at the TOP of the contexts (most specific / current reference).
Accepts a dict (or {"result": {...}} wrapped) with any arbitrary episode
fields (title, description, transcript, script, showNotes, guests,
publishedAt, audioUrl, duration, author, etc.). Capped at 24000 chars
(episodes can have long transcripts).
2026-09-11: uses INVISIBLE EPISODE_FULL_MARKER so dedupe still classifies
it without leaking <<<...>>> wrappers into TTS.
Returns "" if empty / fetch failed / None.
"""
if raw_value is None:
return ""
if isinstance(raw_value, dict) and len(raw_value) == 1 and isinstance(raw_value.get("result"), (dict, list)):
raw_value = raw_value["result"]
if isinstance(raw_value, dict) or isinstance(raw_value, list):
try:
serialized = json.dumps(raw_value, ensure_ascii=False)
except Exception:
serialized = str(raw_value)
else:
serialized = str(raw_value)
trimmed = serialized.strip()
if not trimmed:
return ""
if len(trimmed) > 24000:
trimmed = trimmed[:24000] + "...(episode truncated)"
body = f"{PODCAST_EPISODE_HINT}\n{trimmed}\n. Treat as PRIMARY trusted reference for this session."
return EPISODE_FULL_MARKER + "\n" + body
def _parse_date_key(value: Any) -> tuple[Any, ...]:
"""Return a sortable tuple for an arbitrary date-like value.
Sort order is DESC (newest first). Non-dates / empty sort to the end (lowest priority).
Accepts ISO strings ("YYYY-MM-DD", "YYYY-MM-DDTHH:MM:SS"), Python datetime objects,
int/float unix timestamps, numbers.
"""
import datetime as _dt
if value is None:
return (1, 0)
if isinstance(value, _dt.datetime):
try:
return (0, -int(value.timestamp()))
except Exception:
return (1, 0)
if isinstance(value, _dt.date):
try:
ts = int(_dt.datetime.combine(value, _dt.time.min).timestamp())
return (0, -ts)
except Exception:
return (1, 0)
if isinstance(value, (int, float)):
try:
return (0, -int(value))
except Exception:
return (1, 0)
if isinstance(value, str):
s = value.strip()
if not s:
return (1, 0)
# Try ISO parse first (handles most formats)
try:
dt = _dt.datetime.fromisoformat(s.replace("Z", "+00:00"))
return (0, -int(dt.timestamp()))
except Exception:
pass
# Fallback: date-only YYYY-MM-DD
try:
dt = _dt.datetime.strptime(s[:10], "%Y-%m-%d")
return (0, -int(dt.timestamp()))
except Exception:
pass
# Last resort: lexicographic sort DESC (newest first still works for ISO-ish)
return (0, tuple(-ord(c) for c in s[:32]))
return (1, 0)
def _resolve_reports_context(raw_value: object) -> str:
"""Normalize / filter the S2S_REPORT_LIST response into a prompt block.
Expects an array/list (or dict {"result": [...]} unwrapped by the fetcher).
For each dict item keeps only fields: sourceScheduleLabel, date, podcastScript.
Also supports nested shape: sourceSchedule.label / sourceSchedule.date.
Sorts by date DESC (newest first) via _parse_date_key. Capped at 12000 chars
(podcastScript can be long). Wrapped with <<<REPORTS_LIST_BEGIN/END>>> markers.
Returns "" if no valid items / empty list.
USER FIX 2026-09-11: remove mass injection of scheduled reports.
• If the new env flag S2S_INJECT_REPORTS is FALSE (DEFAULT), this function
ALWAYS returns "" — NO scheduled reports of any kind are ever injected
into the LLM prompt, not even a TOC line with label+date. The function
returns the empty string at the very top before touching the input, so
we also skip JSON processing / sorting / serialization work.
• If S2S_INJECT_REPORTS=TRUE, behaviour is unchanged: full injection of
podcastScript text for every report, just as before.
"""
# ── MASS-INJECTION KILL SWITCH (user request) ────────────────────────
if not settings.S2S_INJECT_REPORTS:
return ""
# ─────────────────────────────────────────────────────────────────────
if raw_value is None:
return ""
items_list: list
if isinstance(raw_value, list):
items_list = raw_value
elif isinstance(raw_value, dict) and isinstance(raw_value.get("result"), list):
items_list = raw_value["result"]
else:
return ""
# Keep only dict items; for each project ONLY the three required fields (in order).
filtered: list[dict] = []
for item in items_list:
if not isinstance(item, dict):
continue
cleaned: dict = {}
# Field extraction with fallback to nested sourceSchedule.* shape.
# 1. Label: prefer flat sourceScheduleLabel, else nested sourceSchedule.label
label_val: object = None
if item.get("sourceScheduleLabel") is not None:
label_val = item["sourceScheduleLabel"]
elif isinstance(item.get("sourceSchedule"), dict):
ss = item["sourceSchedule"]
if ss.get("label") is not None:
label_val = ss["label"]
if label_val is None and ss.get("name") is not None:
label_val = ss["name"]
if label_val is None and ss.get("title") is not None:
label_val = ss["title"]
if label_val is not None:
cleaned["sourceScheduleLabel"] = label_val
# 2. Date: prefer flat "date", else nested sourceSchedule.date
date_val: object = None
if item.get("date") is not None:
date_val = item["date"]
elif isinstance(item.get("sourceSchedule"), dict):
ss = item["sourceSchedule"]
if ss.get("date") is not None:
date_val = ss["date"]
if date_val is None and ss.get("scheduledAt") is not None:
date_val = ss["scheduledAt"]
if date_val is None and ss.get("publishedAt") is not None:
date_val = ss["publishedAt"]
if date_val is not None:
cleaned["date"] = date_val
# 3. Script: prefer flat podcastScript, else nested podcastScript / script / text
script_val: object = None
if item.get("podcastScript") is not None:
script_val = item["podcastScript"]
elif item.get("script") is not None:
script_val = item["script"]
elif item.get("text") is not None:
script_val = item["text"]
elif item.get("content") is not None:
script_val = item["content"]
if script_val is not None:
cleaned["podcastScript"] = script_val
if cleaned:
filtered.append(cleaned)
if not filtered:
return ""
# Sort by date DESC (newest first)
filtered.sort(key=lambda r: _parse_date_key(r.get("date")))
# 2026-09-11 user fix: limit report quick hints through .env (default 3-5).
# after sort by newest → keep ONLY the first S2S_REPORTS_QUICK_HINT_MAX reports.
# Applies cap to BOTH the quick-hint TOC lines AND the serialized JSON entries,
# so the final wrapped block stays small and contains no more than N scripts.
total_raw = len(filtered)
cap = max(1, int(settings.S2S_REPORTS_QUICK_HINT_MAX or 5))
if total_raw > cap:
filtered = filtered[:cap]
truncated_note = ""
if total_raw > cap:
truncated_note = (
f"\n(Showing the {cap} newest scheduled reports of {total_raw} total "
f"returned upstream; older reports omitted. Set S2S_REPORTS_QUICK_HINT_MAX "
f"in .env to raise or lower this window.)\n"
)
# ──────────────────────────────────────────────────────────────────────
# USER FIX BUG 3 2026-09-11: Qwen2.5 cutoff mid-2024 hallucinates meta on 2026 dates.
# Route output through the new tri-state S2S_REPORTS_INJECT_MODE:
# • "off"/"none" → skip entirely (no reports block in prompt, 0 tokens).
# • "minimal" → ONLY last-N label + date TOC. NO podcastScript text, no JSON entries,
# just the requested recent scheduled reports block.
# ~50 tokens instead of ~3200.
# • "full" → legacy mode with hints + full JSON entries + podcastScript.
# Kept only for explicit opt-in because user explicitly asked
# to eliminate massive script injection.
mode = (settings.S2S_REPORTS_INJECT_MODE or "minimal").lower()
if mode in {"off", "none"}:
if settings.S2S_DEBUG_PROMPT:
app_logging.logger.info(
"[INJECT-REPORTS] mode=%s → no scheduled reports injected into prompt (cap=%d totalRaw=%d)",
mode,
cap,
total_raw,
)
return ""
if mode == "minimal":
minimal_block = _build_minimal_reports_toc_block(filtered, cap=cap)
if settings.S2S_DEBUG_PROMPT:
app_logging.logger.info(
"[INJECT-REPORTS] mode=minimal blockLen=%d (no podcastScript injected) cap=%d totalRaw=%d",
len(minimal_block),
cap,
total_raw,
)
return minimal_block
# ───── mode == "full" — legacy behaviour (opt-in only) ───────────────
if settings.S2S_DEBUG_PROMPT:
app_logging.logger.info(
"[INJECT-REPORTS] mode=full (LEGACY, podcastScript included) cap=%d totalRaw=%d",
cap,
total_raw,
)
# Build a quick HINT LINE per report (label + date) so the LLM can see at a
# glance which scheduled topics are available before diving into scripts.
hint_lines = []
import datetime as _dt
for idx, r in enumerate(filtered, start=1):
label_piece = ""
if r.get("sourceScheduleLabel"):
label_piece = str(r["sourceScheduleLabel"]).strip()
date_piece = ""
if r.get("date") is not None:
raw_d = r["date"]
try:
# Integer / float timestamp → render as YYYY-MM-DD HH:MM (local)
if isinstance(raw_d, (int, float)) and not isinstance(raw_d, bool):
ts_sec = float(raw_d)
if ts_sec > 1e12: # ms → seconds
ts_sec = ts_sec / 1000.0
dt = _dt.datetime.fromtimestamp(ts_sec, tz=_dt.UTC)
date_piece = dt.strftime("%Y-%m-%d %H:%M UTC")
else:
s = str(raw_d).strip()
if "T" in s and len(s) >= 16:
date_piece = s.replace("T", " ")[:16]
else:
date_piece = s
except Exception:
date_piece = str(raw_d)[:32]
if label_piece and date_piece:
hint_lines.append(f" • Report #{idx}: '{label_piece}' — scheduled on {date_piece}")
elif label_piece:
hint_lines.append(f" • Report #{idx}: '{label_piece}'")
elif date_piece:
hint_lines.append(f" • Report #{idx}: (no label) — scheduled on {date_piece}")
# Serialize: compact JSON (pretty-ish but one line per report saves space)
entries_lines = []
for idx, r in enumerate(filtered, start=1):
try:
entries_lines.append(f" [{idx}] {json.dumps(r, ensure_ascii=False)}")
except Exception:
entries_lines.append(f" [{idx}] {str(r)[:500]}")
hints_block = ""
if hint_lines:
hints_block = "\n".join(hint_lines) + "\n"
body = truncated_note + hints_block + "\n--- Full JSON entries ---\n" + "\n".join(entries_lines)
wrapped = (
f"{REPORTS_LIST_HINT}\n"
f"Quick hints (available scheduled reports / podcasts by label — cap {cap} newest):\n"
f"{body}\n"
f"<<<REPORTS_LIST_END>>>. Use these scheduled reports as trusted prior "
"content relevant to the caller's company topics. Reference individual "
"reports by their label when answering."
)
# Build wrapped with HINT BLOCK before the JSON. We prepend a summary line per
# report so the LLM has a quick table-of-contents of what's available.
if len(wrapped) > 12000:
# Truncate body preserving markers + hint
budget = (
12000
- len(REPORTS_LIST_HINT)
- len(f"\nQuick hints (available scheduled reports / podcasts by label — cap {cap} newest):\n")
- len("\n--- Full JSON entries ---\n")
- len("\n<<<REPORTS_LIST_END>>>...truncated.")
- 200 # reserve for wrap sentence
)
if budget > 0:
body = body[:budget] + "..."
wrapped = (
f"{REPORTS_LIST_HINT}\n"
f"Quick hints (available scheduled reports / podcasts by label — cap {cap} newest):\n"
f"{body}\n"
f"<<<REPORTS_LIST_END>>>...truncated (cap 12000 chars). Use these "
"scheduled reports as trusted prior content. Reference reports by label when relevant."
)
if total_raw and total_raw > cap and len(truncated_note) and settings.S2S_DEBUG_PROMPT:
app_logging.logger.info(
"[INJECT-REPORTS] quick-hint+entries cap applied: kept=%d upstreamTotal=%d cap=%d",
len(filtered),
total_raw,
cap,
)
return wrapped
def _augment_instructions_with_profile(
base_instructions: str | None,
user_context_text: str | None,
reports_context_text: str | None = None,
summary_context_text: str | None = None,
episode_context_text: str | None = None,
profile_obj: Any = None,
) -> str:
"""When S2S_INJECT_PROFILE is true: prepend TODAY fixed context block (Bug 3 fix)
then append the episode + profile + (optional) reports list + (optional) latest
summary context strings to the LLM system/instructions prompt.
Append order (most specific → most generic / chronologically newest → oldest):
0. VOICE_RULES_HEAD_LOCK (absolute language lock + #1 priority)
1. TODAY_block (date of today + anti-hallucination date rules)
2. episode_ctx (CURRENT podcast episode the user is interacting with)
3. base instructions (primary prompt / user settings / default)
4. profile_ctx (caller identity block)
5. reports_ctx (scheduled reports / podcast scripts list, sorted by date DESC)
6. summary_ctx (latest conversation summary)
Preserves the existing instructions text.
USER FIX 2026-09-11: remove duplicate caller profiles with TWO deduplication layers
so the same block type (profile / episode / reports / summary) NEVER appears twice
in the final augmented instructions, even if the caller passed it both in
base_instructions (e.g. client-supplied profile baked into req.instructions) AND in the
explicit profile_ctx parameter:
1) layer 2 — _strip_already_injected_types_from_base removes from parameters
if their marker already appears in base_instructions.
2) layer 1 — _dedup_context_blocks_by_type then removes any remaining
duplicates among the parameters themselves.
2026-09-11: NEW keyword profile_obj → forwarded to _build_voice_rules_head_lock_block
so the prompt lock can emit rule #5 "Exact name: Alessandro Valenti. NEVER
Alexander" for models that keep anglicizing proper names (Qwen2.5-GPTQ).
"""
base = (base_instructions or "").strip()
# ─── USER FIX BUG 3: TODAY fixed block prepended (HEAD of prompt) ─────
voice_lock_block = _build_voice_rules_head_lock_block(
profile=profile_obj
) # NEW #1 priority: absolute language+output lock
today_block = _build_today_fixed_context_block() # #2: date guard for Spanish responses
episode_ctx = (episode_context_text or "").strip()
profile_ctx = (user_context_text or "").strip()
reports_ctx = (reports_context_text or "").strip()
summary_ctx = (summary_context_text or "").strip()
# raw_candidates is ORDERED by prompt-priority! voice_lock_block (#0) is
# inserted FIRST so it becomes position 0 of the final instructions and
# gets highest LLM attention, then today, then episode, etc.
raw_candidates = [voice_lock_block, today_block, episode_ctx, profile_ctx, reports_ctx, summary_ctx]
# Layer 2: strip candidates whose HINT is ALREADY in base instructions
# (e.g. client sent profile baked in, we won't re-add it).
candidates_against_base = _strip_already_injected_types_from_base(base, raw_candidates)
# Layer 1: dedupe among themselves (e.g. episode was both episode_ctx passed in the same raw_candidates twice via identity + raw).
contexts = _dedup_context_blocks_by_type(candidates_against_base)
if not contexts:
return base_instructions or ""
augmented = base
for ctx in contexts:
augmented = (augmented + " " + ctx).strip() if augmented else ctx
# ─── 2026-09-11 MARKER LEAK GUARD: STRIP INTERNAL SENTINELS ──────────
# All invisible dedup markers are NOW stripped from the final prompt text
# before leaving this function → LLM NEVER sees ANY <<<X>>> / hash string.
augmented = _strip_internal_markers(augmented)
changed = augmented != base
# Compute per-type lengths for log (based on what actually survived dedupe)
final_lock_len = sum(len(c) for c in contexts if _classify_context_block(c) == "voice_rules_head")
final_today_len = sum(len(c) for c in contexts if _classify_context_block(c) == "today_fixed")
final_ep_len = sum(len(c) for c in contexts if _classify_context_block(c) == "episode")
final_pf_len = sum(len(c) for c in contexts if _classify_context_block(c) == "profile")
final_rp_len = sum(
len(c) for c in contexts if _classify_context_block(c) in {"reports", "reports_minimal", "reports_full"}
)
final_sm_len = sum(len(c) for c in contexts if _classify_context_block(c) in {"summary", "summary_full"})
app_logging.logger.info(
"[INJECT-PROFILE] ✏️ prepended VOICE_LOCK + TODAY_block + appended episode/profile/reports/summary to LLM instructions. "
"baseLen=%d voiceLockLen=%d todayCtxLen=%d episodeCtxLen=%d profileCtxLen=%d reportsCtxLen=%d summaryCtxLen=%d newLen=%d changed=%s dedupedCandidates=%d",
len(base),
final_lock_len,
final_today_len,
final_ep_len,
final_pf_len,
final_rp_len,
final_sm_len,
len(augmented),
changed,
len(contexts),
)
if settings.S2S_DEBUG_PROMPT and final_lock_len:
for c in contexts:
if _classify_context_block(c) == "voice_rules_head":
app_logging.logger.info(
"[DEBUG-VOICELOCK][instructions augment] FULL voice lock block (len=%d):\n%s\n",
len(c),
_strip_internal_markers(c),
)
if settings.S2S_DEBUG_PROMPT and final_today_len:
for c in contexts:
if _classify_context_block(c) == "today_fixed":
app_logging.logger.info(
"[DEBUG-TODAY][instructions augment] FULL today+anti-hallucination block (len=%d):\n%s\n",
len(c),
_strip_internal_markers(c),
)
if settings.S2S_DEBUG_PROMPT and final_ep_len:
for c in contexts:
if _classify_context_block(c) == "episode":
app_logging.logger.info(
"[DEBUG-EPISODE][instructions augment] FULL podcast episode block (len=%d):\n%s\n",
len(c),
_strip_internal_markers(c),
)
if settings.S2S_DEBUG_PROMPT and final_rp_len:
for c in contexts:
if _classify_context_block(c) in {"reports", "reports_minimal", "reports_full"}:
app_logging.logger.info(
"[DEBUG-REPORTS][instructions augment] FULL reports block (len=%d mode=%s):\n%s\n",
len(c),
_classify_context_block(c),
_strip_internal_markers(c),
)
if settings.S2S_DEBUG_PROMPT and final_sm_len:
for c in contexts:
if _classify_context_block(c) in {"summary", "summary_full"}:
app_logging.logger.info(
"[DEBUG-SUMMARY][instructions augment] FULL latest summary block (len=%d):\n%s\n",
len(c),
_strip_internal_markers(c),
)
return augmented
def _patch_client_session_update_with_profile(
raw_text: str,
user_context_text: str | None,
reports_context_text: str | None = None,
summary_context_text: str | None = None,
episode_context_text: str | None = None,
profile_obj: Any = None,
) -> str:
"""If raw_text is a client session.update JSON and S2S_INJECT_PROFILE is
true, append episode_context_text + user_context_text + (optional) reports_context_text +
(optional) summary_context_text to session.instructions.
Returns the modified JSON string, or the original string unchanged if:
- it is not session.update JSON (the VAST majority of frames -> silent passthrough, NO LOG)
- episode_context_text AND user_context_text AND reports_context_text AND summary_context_text are all empty
- S2S_INJECT_PROFILE is False
- parse fails
2026-09-11: NEW keyword profile_obj. Forwarded to
_augment_instructions_with_profile so the prompt lock can emit the proper-name
anti-anglicization rule for this caller (e.g. Alessandro never Alexander).
"""
# Step 1: only consider session.update frames. All other frame types
# (response.create, conversation.item.*, audio_* etc.) pass through
# SILENTLY — we never emit logs for them, which avoids spamming the
# console with 6-20 identical "SKIPPED" lines per session.
try:
payload = json.loads(raw_text)
except Exception:
return raw_text
if not isinstance(payload, dict) or payload.get("type") != "session.update":
return raw_text
# From here on: we know this is a real session.update (typically emitted
# just once or twice per realtime session). We can log safely — 1 line,
# not 20 repeated lines.
if not settings.S2S_INJECT_PROFILE:
app_logging.logger.info(
"[INJECT-PROFILE] ⏭️ WS patch SKIPPED on session.update (S2S_INJECT_PROFILE=false). "
"WS stays pure passthrough."
)
return raw_text
has_episode = bool((episode_context_text or "").strip())
has_profile = bool((user_context_text or "").strip())
has_reports = bool((reports_context_text or "").strip())
has_summary = bool((summary_context_text or "").strip())
if not (has_episode or has_profile or has_reports or has_summary):
app_logging.logger.info(
"[INJECT-PROFILE] ⏭️ WS patch SKIPPED on session.update (no episode, no profile context, no reports list, no latest summary). "
"Probably phoneNumber/episode_id query param missing / fetch failed upstream / "
"endpoints or S2S_LATEST_SUMMARY / S2S_REPORT_LIST returned empty result."
)
return raw_text
if has_episode and not has_profile:
app_logging.logger.info(
"[INJECT-EPISODE] profile missing, but PODCAST EPISODE block loaded. "
"Will inject episode alone into session.update."
)
if has_summary and not has_profile and not has_episode:
app_logging.logger.info(
"[INJECT-SUMMARY] profile missing, but LATEST SUMMARY block loaded. "
"Will inject summary alone into session.update."
)
if has_reports and not has_profile and not has_episode:
app_logging.logger.info(
"[INJECT-REPORTS] profile missing, but REPORTS LIST block loaded. "
"Will inject reports alone into session.update."
)
top_session = payload.get("session")
if not isinstance(top_session, dict):
return raw_text
current_instructions = (
cast(str, top_session.get("instructions")) if isinstance(top_session.get("instructions"), str) else ""
)
augmented = _augment_instructions_with_profile(
current_instructions,
user_context_text,
reports_context_text,
summary_context_text,
episode_context_text,
profile_obj=profile_obj,
)
if settings.S2S_DEBUG_PROMPT:
app_logging.logger.info(
"[DEBUG-PROMPT][WS session.update] BEFORE instructions (len=%d):\n%s\n",
len(current_instructions),
current_instructions,
)
if augmented == current_instructions:
app_logging.logger.info(
"[INJECT-PROFILE] ⏭️ WS patch SKIPPED on session.update (augmented == current; "
"episode/profile already injected or context empty)"
)
if settings.S2S_DEBUG_PROMPT and augmented.strip():
app_logging.logger.info(
"[DEBUG-PROMPT][WS session.update] FINAL unchanged (len=%d):\n%s\n",
len(augmented),
augmented,
)
return raw_text
top_session["instructions"] = augmented
app_logging.logger.info(
"[INJECT-PROFILE] ✅ WS session.update PATCHED -> upstream instructions. "
"origLen=%d newLen=%d injectedLen=%d episode=%s profile=%s reports=%s summary=%s",
len(current_instructions),
len(augmented),
(len(augmented) - len(current_instructions)),
has_episode,
has_profile,
has_reports,
has_summary,
)
if settings.S2S_DEBUG_PROMPT:
app_logging.logger.info(
"[DEBUG-PROMPT][WS session.update] AFTER patch (len=%d):\n%s\n",
len(augmented),
augmented,
)
if settings.S2S_DEBUG_PROMPT and episode_context_text and episode_context_text.strip():
app_logging.logger.info(
"[DEBUG-EPISODE][WS session.update] FULL podcast episode block (len=%d):\n%s\n",
len(episode_context_text.strip()),
episode_context_text.strip(),
)
if settings.S2S_DEBUG_PROMPT and reports_context_text and reports_context_text.strip():
app_logging.logger.info(
"[DEBUG-REPORTS][WS session.update] FULL reports list block (len=%d):\n%s\n",
len(reports_context_text.strip()),
reports_context_text.strip(),
)
try:
return json.dumps(payload)
except Exception:
app_logging.logger.info("[INJECT-PROFILE] ⛔ JSON re-serialize failed on session.update. Forwarding unpatched.")
return raw_text
def _common_name_variants(original_name: str) -> list[str]:
"""For a user's first/last name, return known anglicized / adapted variants
that the model may produce (e.g. Alessandro → Alexander). Used by
sanitizer to replace variant → original.
Map is conservative (only common names we have observed in the wild so
far). Any new name can be added here without touching other code.
"""
variants_map: dict[str, tuple[str, ...]] = {
# Western / Romance masculine first names — anglicized / cognates
"Alessandro": (
"Alexander",
"Alexandre",
"Aleksander",
"Alexsander",
"Alessander",
"Aleksandr",
"Alexander",
"Alejandro",
"Alesandro",
"Allessandro",
"Alex",
),
"Alessandra": ("Alexandra", "Aleksandra", "Alejandra", "Alex"),
"Valenti": ("Valentini", "Valentine", "Valentin", "Valente", "Valentín", "Valentí"),
"Giovanni": ("John", "Juan", "Giovani", "Gianni", "Joan", "Ivan"),
"Paolo": ("Paul", "Pablo", "Paulo", "Paul"),
"Marco": ("Marc", "Mark", "Marcos"),
"Luca": ("Lucas", "Luke", "Lukas"),
"Matteo": ("Matthew", "Matías", "Mateo", "Matt"),
"Francesco": ("Francis", "Francisco", "Francesc"),
"Leonardo": ("Leonard", "Leonhard"),
"Giuseppe": ("Joseph", "José", "Giuseppi", "Pepe"),
"Mario": ("Maryo", "Mari", "Marion"),
# Feminine
"Sofia": ("Sophia", "Sophie", "Sofía", "Sofie"),
"Giulia": ("Julia", "Juliana", "Julie", "Giuliana"),
"Anna": ("Ana", "Anne", "Hannah", "Ann"),
}
out: list[str] = []
if not original_name:
return out
o = original_name.strip()
if o in variants_map:
out.extend(variants_map[o])
# Also check case-insensitive match as safety net
lower_map = {k.lower(): v for k, v in variants_map.items()}
if o.lower() in lower_map:
for v in lower_map[o.lower()]:
if v not in out:
out.append(v)
return out