349 lines
15 KiB
Python
349 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from time import perf_counter
|
|
from typing import TYPE_CHECKING, Optional
|
|
|
|
from openai.types.realtime import (
|
|
RealtimeResponse,
|
|
ResponseAudioDoneEvent,
|
|
ResponseAudioTranscriptDoneEvent,
|
|
ResponseCreatedEvent,
|
|
ResponseCreateEvent,
|
|
ResponseDoneEvent,
|
|
ResponseFunctionCallArgumentsDoneEvent,
|
|
ResponseTextDeltaEvent,
|
|
ResponseTextDoneEvent,
|
|
)
|
|
from openai.types.realtime.realtime_response import Audio, AudioOutput
|
|
from openai.types.realtime.realtime_response_status import RealtimeResponseStatus
|
|
from openai.types.realtime.realtime_response_usage import RealtimeResponseUsage
|
|
|
|
from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
|
|
from speech_to_speech.LLM.chat import ChatItemError
|
|
from speech_to_speech.pipeline.events import AssistantTextEvent
|
|
from speech_to_speech.pipeline.messages import GenerateResponseRequest
|
|
from speech_to_speech.utils.utils import _generate_id, is_out_of_band, response_wants_audio
|
|
|
|
if TYPE_CHECKING:
|
|
from speech_to_speech.api.openai_realtime.service import ServerEvent, _ResponseStatus, _StatusReason
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ResponseHandler(RealtimeBaseHandler):
|
|
"""Owns the response lifecycle: create, cancel, finish, and ID management."""
|
|
|
|
# ── ID / state helpers ────────────────────────
|
|
|
|
def _ensure_response(self, conn_id: str) -> tuple[str, str]:
|
|
"""Ensure a response and output item exist, creating them if needed."""
|
|
st = self._state(conn_id)
|
|
if st.current_response_id is None:
|
|
st.current_response_id = _generate_id("resp")
|
|
self._start_item(conn_id)
|
|
st.in_response = True
|
|
st.response_pending = False
|
|
return st.current_response_id, self._current_item_id(conn_id)
|
|
|
|
def _end_response(self, conn_id: str, status: _ResponseStatus = "completed") -> None:
|
|
st = self._state(conn_id)
|
|
if status == "cancelled":
|
|
st.response_usage.responses_cancelled += 1
|
|
else:
|
|
st.response_usage.responses_completed += 1
|
|
self._service.total_usage += st.response_usage
|
|
logger.info(
|
|
"Response done (status=%s) — this response: input_tokens=%d, output_tokens=%d, audio=%.2fs"
|
|
" | cumulative: input_tokens=%d, output_tokens=%d, audio=%.2fs",
|
|
status,
|
|
st.response_usage.input_tokens,
|
|
st.response_usage.output_tokens,
|
|
st.response_usage.audio_duration_s,
|
|
self._service.total_usage.input_tokens,
|
|
self._service.total_usage.output_tokens,
|
|
self._service.total_usage.audio_duration_s,
|
|
)
|
|
st.response_usage.reset()
|
|
st.current_response_id = None
|
|
st.current_item_id = None
|
|
st.content_index = 0
|
|
st.in_response = False
|
|
st.response_pending = False
|
|
st.current_response_params = None
|
|
st.pending_output_text_parts = []
|
|
st.response_audio_wait_started_at_s = None
|
|
st.delay_filler_sent = False
|
|
|
|
def _start_item(self, conn_id: str) -> str:
|
|
"""Generate a new item ID, reset content index, and store it."""
|
|
st = self._state(conn_id)
|
|
item_id = _generate_id("item")
|
|
st.current_item_id = item_id
|
|
st.content_index = 0
|
|
st.input_audio_duration_s = 0.0
|
|
return item_id
|
|
|
|
def _current_item_id(self, conn_id: str) -> str:
|
|
return self._state(conn_id).current_item_id or self._start_item(conn_id)
|
|
|
|
def _next_content_index(self, conn_id: str) -> int:
|
|
"""Return the current content index and advance it."""
|
|
st = self._state(conn_id)
|
|
idx = st.content_index
|
|
st.content_index += 1
|
|
return idx
|
|
|
|
def _build_response(
|
|
self,
|
|
conn_id: str,
|
|
status: _ResponseStatus,
|
|
reason: _StatusReason | None = None,
|
|
) -> RealtimeResponse:
|
|
"""Build a fully-populated RealtimeResponse from the current connection state."""
|
|
st = self._state(conn_id)
|
|
status_details = None
|
|
if reason or status in ("completed", "cancelled", "incomplete", "failed"):
|
|
status_details = RealtimeResponseStatus(type=status, reason=reason) # type: ignore[arg-type]
|
|
|
|
rp = st.current_response_params
|
|
metadata = rp.metadata if rp and rp.metadata else None
|
|
|
|
voice: Optional[str] = None
|
|
if rp and rp.audio and rp.audio.output and rp.audio.output.voice:
|
|
voice = str(rp.audio.output.voice)
|
|
if not voice:
|
|
audio_cfg = st.runtime_config.session.audio
|
|
audio_output = audio_cfg.output if audio_cfg is not None else None
|
|
voice = str(audio_output.voice) if audio_output is not None and audio_output.voice else None
|
|
|
|
# Out-of-band responses are not threaded into any conversation: report a null id.
|
|
conversation_id = None if is_out_of_band(rp) else st.conversation_id
|
|
|
|
return RealtimeResponse(
|
|
id=st.current_response_id,
|
|
object="realtime.response",
|
|
status=status,
|
|
status_details=status_details,
|
|
audio=Audio(output=AudioOutput(voice=str(voice) if voice else None)), # type: ignore[arg-type]
|
|
conversation_id=conversation_id,
|
|
metadata=metadata,
|
|
usage=RealtimeResponseUsage(
|
|
input_tokens=st.response_usage.input_tokens,
|
|
output_tokens=st.response_usage.output_tokens,
|
|
total_tokens=st.response_usage.input_tokens + st.response_usage.output_tokens,
|
|
),
|
|
)
|
|
|
|
# ── Public handlers ───────────────────────────
|
|
|
|
def handle_response_create(self, conn_id: str, event: ResponseCreateEvent) -> ServerEvent | None:
|
|
"""Trigger a response.
|
|
|
|
Returns a ``ResponseCreatedEvent`` on success, a ``RealtimeErrorEvent``
|
|
on failure, or ``None`` if there is no text_prompt_queue.
|
|
"""
|
|
st = self._state(conn_id)
|
|
if event.response:
|
|
if event.response.tool_choice and not isinstance(event.response.tool_choice, str):
|
|
return self.make_error(
|
|
message="Only string tool_choice values are supported for now (auto, required, none).",
|
|
_type="tool_choice_not_supported",
|
|
)
|
|
if st.in_response:
|
|
return self.make_error(
|
|
message="Cannot create response while another response is in progress.",
|
|
_type="conversation_already_has_active_response",
|
|
)
|
|
|
|
out_of_band = is_out_of_band(event.response)
|
|
|
|
# In-band: response.input items are added to the default conversation here so
|
|
# they appear in history. Out-of-band: leave the default conversation untouched —
|
|
# the input rides along on the request and seeds a throwaway chat in the LM.
|
|
if not out_of_band and event.response and event.response.input:
|
|
for input_item in event.response.input:
|
|
try:
|
|
self._service.conversation._append_item(conn_id, input_item)
|
|
except ChatItemError as exc:
|
|
return self.make_error(message=str(exc), _type="invalid_input_item")
|
|
|
|
st.in_response = True
|
|
st.response_pending = False
|
|
|
|
st.current_response_params = event.response
|
|
st.current_response_id = _generate_id("resp")
|
|
self._start_item(conn_id)
|
|
if response_wants_audio(event.response):
|
|
st.response_audio_wait_started_at_s = perf_counter()
|
|
st.delay_filler_sent = False
|
|
else:
|
|
st.response_audio_wait_started_at_s = None
|
|
st.delay_filler_sent = False
|
|
|
|
cfg = st.runtime_config
|
|
queue = self._queue(conn_id)
|
|
if queue:
|
|
# Out-of-band responses carry no turn identity: a null turn_id makes every
|
|
# speculative-turn staleness gate treat them as always-latest, so a new user
|
|
# turn mid-generation can never silently drop their output.
|
|
queue.put(
|
|
GenerateResponseRequest(
|
|
runtime_config=cfg,
|
|
response=event.response,
|
|
turn_id=None if out_of_band else st.speculative_user_turn_id,
|
|
turn_revision=None if out_of_band else st.speculative_user_turn_revision,
|
|
speech_stopped_at_s=None if out_of_band else st.speculative_user_speech_stopped_at_s,
|
|
)
|
|
)
|
|
logger.debug("response.create received, LLM generation triggered")
|
|
return ResponseCreatedEvent(
|
|
type="response.created",
|
|
event_id=self._next_event_id(),
|
|
response=self._build_response(conn_id, "in_progress"),
|
|
)
|
|
|
|
def handle_response_cancel(self, conn_id: str) -> list[ServerEvent]:
|
|
"""Cancel the in-progress response and re-enable listening."""
|
|
events = self.finish_response(conn_id, status="cancelled", reason="client_cancelled")
|
|
should_listen = self._should_listen(conn_id)
|
|
if should_listen:
|
|
should_listen.set()
|
|
logger.info("Response cancelled, listening re-enabled")
|
|
return events
|
|
|
|
def finish_response(
|
|
self,
|
|
conn_id: str,
|
|
status: _ResponseStatus = "completed",
|
|
reason: _StatusReason | None = None,
|
|
) -> list[ServerEvent]:
|
|
"""Close the current response (audio/text done + response done).
|
|
|
|
Audio responses emit ``response.output_audio.done`` for any terminal
|
|
status. Text-only responses emit a single ``response.output_text.done``
|
|
carrying the full streamed text, but only on ``status="completed"`` —
|
|
a cancelled or failed text response sends no audio, so it just closes
|
|
with ``response.done``.
|
|
"""
|
|
st = self._state(conn_id)
|
|
events: list[ServerEvent] = []
|
|
if st.in_response:
|
|
resp_id, item_id = self._ensure_response(conn_id)
|
|
if response_wants_audio(st.current_response_params):
|
|
events.append(
|
|
ResponseAudioDoneEvent(
|
|
type="response.output_audio.done",
|
|
event_id=self._next_event_id(),
|
|
content_index=0,
|
|
item_id=item_id,
|
|
output_index=0,
|
|
response_id=resp_id,
|
|
)
|
|
)
|
|
elif status == "completed" and st.pending_output_text_parts:
|
|
events.append(
|
|
ResponseTextDoneEvent(
|
|
type="response.output_text.done",
|
|
event_id=self._next_event_id(),
|
|
content_index=0,
|
|
item_id=item_id,
|
|
output_index=0,
|
|
response_id=resp_id,
|
|
text="".join(st.pending_output_text_parts),
|
|
)
|
|
)
|
|
events.append(
|
|
ResponseDoneEvent(
|
|
type="response.done",
|
|
event_id=self._next_event_id(),
|
|
response=self._build_response(conn_id, status, reason),
|
|
)
|
|
)
|
|
self._end_response(conn_id, status)
|
|
# Apply any client items that arrived mid-generation now that in_response
|
|
# is cleared and the generation's own write-back has landed. Done outside
|
|
# the in_response guard so a stray terminal call still drains the buffer.
|
|
events.extend(self._service.conversation.flush_deferred_items(conn_id))
|
|
return events
|
|
|
|
# ── Pipeline event handlers ───────────────────
|
|
|
|
def on_assistant_text(
|
|
self,
|
|
conn_id: str,
|
|
event: AssistantTextEvent,
|
|
*,
|
|
wait_for_pending_reopen: bool = True,
|
|
) -> list[ServerEvent] | None:
|
|
"""Handle assistant_text: emit transcript and/or tool-call events."""
|
|
if self._service.speculative_turns:
|
|
commit_result: bool | None
|
|
if wait_for_pending_reopen:
|
|
commit_result = self._service.speculative_turns.commit_if_latest_after_reopen_grace(
|
|
event.turn_id,
|
|
event.turn_revision,
|
|
)
|
|
else:
|
|
commit_result = self._service.speculative_turns.try_commit_if_latest_after_reopen_grace(
|
|
event.turn_id,
|
|
event.turn_revision,
|
|
)
|
|
if commit_result is None:
|
|
return None
|
|
if not commit_result:
|
|
logger.debug("Dropping stale assistant text for turn=%s rev=%s", event.turn_id, event.turn_revision)
|
|
return []
|
|
st = self._state(conn_id)
|
|
events: list[ServerEvent] = []
|
|
resp_id, item_id = self._ensure_response(conn_id)
|
|
st.last_item_id = item_id
|
|
output_idx = 0
|
|
if event.text:
|
|
if response_wants_audio(st.current_response_params):
|
|
events.append(
|
|
ResponseAudioTranscriptDoneEvent(
|
|
type="response.output_audio_transcript.done",
|
|
event_id=self._next_event_id(),
|
|
content_index=0,
|
|
item_id=item_id,
|
|
output_index=output_idx,
|
|
response_id=resp_id,
|
|
transcript=event.text,
|
|
)
|
|
)
|
|
else:
|
|
# Stream the delta now; the matching response.output_text.done is
|
|
# emitted once at close in finish_response, carrying the per-chunk
|
|
# parts collected here, space-joined ("" when there were none).
|
|
st.pending_output_text_parts.append(event.text)
|
|
events.append(
|
|
ResponseTextDeltaEvent(
|
|
type="response.output_text.delta",
|
|
event_id=self._next_event_id(),
|
|
content_index=0,
|
|
item_id=item_id,
|
|
output_index=output_idx,
|
|
response_id=resp_id,
|
|
delta=event.text,
|
|
)
|
|
)
|
|
output_idx += 1
|
|
if event.tools:
|
|
st.response_usage.tool_calls += len(event.tools)
|
|
for tool in event.tools:
|
|
events.append(
|
|
ResponseFunctionCallArgumentsDoneEvent(
|
|
type="response.function_call_arguments.done",
|
|
event_id=self._next_event_id(),
|
|
call_id=tool.call_id,
|
|
name=tool.name,
|
|
arguments=tool.arguments,
|
|
item_id=item_id,
|
|
output_index=output_idx,
|
|
response_id=resp_id,
|
|
)
|
|
)
|
|
output_idx += 1
|
|
return events
|