first
This commit is contained in:
@@ -0,0 +1,348 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from time import perf_counter
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from openai.types.realtime import (
|
||||
RealtimeResponse,
|
||||
ResponseAudioDoneEvent,
|
||||
ResponseAudioTranscriptDoneEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseCreateEvent,
|
||||
ResponseDoneEvent,
|
||||
ResponseFunctionCallArgumentsDoneEvent,
|
||||
ResponseTextDeltaEvent,
|
||||
ResponseTextDoneEvent,
|
||||
)
|
||||
from openai.types.realtime.realtime_response import Audio, AudioOutput
|
||||
from openai.types.realtime.realtime_response_status import RealtimeResponseStatus
|
||||
from openai.types.realtime.realtime_response_usage import RealtimeResponseUsage
|
||||
|
||||
from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
|
||||
from speech_to_speech.LLM.chat import ChatItemError
|
||||
from speech_to_speech.pipeline.events import AssistantTextEvent
|
||||
from speech_to_speech.pipeline.messages import GenerateResponseRequest
|
||||
from speech_to_speech.utils.utils import _generate_id, is_out_of_band, response_wants_audio
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from speech_to_speech.api.openai_realtime.service import ServerEvent, _ResponseStatus, _StatusReason
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ResponseHandler(RealtimeBaseHandler):
|
||||
"""Owns the response lifecycle: create, cancel, finish, and ID management."""
|
||||
|
||||
# ── ID / state helpers ────────────────────────
|
||||
|
||||
def _ensure_response(self, conn_id: str) -> tuple[str, str]:
|
||||
"""Ensure a response and output item exist, creating them if needed."""
|
||||
st = self._state(conn_id)
|
||||
if st.current_response_id is None:
|
||||
st.current_response_id = _generate_id("resp")
|
||||
self._start_item(conn_id)
|
||||
st.in_response = True
|
||||
st.response_pending = False
|
||||
return st.current_response_id, self._current_item_id(conn_id)
|
||||
|
||||
def _end_response(self, conn_id: str, status: _ResponseStatus = "completed") -> None:
|
||||
st = self._state(conn_id)
|
||||
if status == "cancelled":
|
||||
st.response_usage.responses_cancelled += 1
|
||||
else:
|
||||
st.response_usage.responses_completed += 1
|
||||
self._service.total_usage += st.response_usage
|
||||
logger.info(
|
||||
"Response done (status=%s) — this response: input_tokens=%d, output_tokens=%d, audio=%.2fs"
|
||||
" | cumulative: input_tokens=%d, output_tokens=%d, audio=%.2fs",
|
||||
status,
|
||||
st.response_usage.input_tokens,
|
||||
st.response_usage.output_tokens,
|
||||
st.response_usage.audio_duration_s,
|
||||
self._service.total_usage.input_tokens,
|
||||
self._service.total_usage.output_tokens,
|
||||
self._service.total_usage.audio_duration_s,
|
||||
)
|
||||
st.response_usage.reset()
|
||||
st.current_response_id = None
|
||||
st.current_item_id = None
|
||||
st.content_index = 0
|
||||
st.in_response = False
|
||||
st.response_pending = False
|
||||
st.current_response_params = None
|
||||
st.pending_output_text_parts = []
|
||||
st.response_audio_wait_started_at_s = None
|
||||
st.delay_filler_sent = False
|
||||
|
||||
def _start_item(self, conn_id: str) -> str:
|
||||
"""Generate a new item ID, reset content index, and store it."""
|
||||
st = self._state(conn_id)
|
||||
item_id = _generate_id("item")
|
||||
st.current_item_id = item_id
|
||||
st.content_index = 0
|
||||
st.input_audio_duration_s = 0.0
|
||||
return item_id
|
||||
|
||||
def _current_item_id(self, conn_id: str) -> str:
|
||||
return self._state(conn_id).current_item_id or self._start_item(conn_id)
|
||||
|
||||
def _next_content_index(self, conn_id: str) -> int:
|
||||
"""Return the current content index and advance it."""
|
||||
st = self._state(conn_id)
|
||||
idx = st.content_index
|
||||
st.content_index += 1
|
||||
return idx
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
conn_id: str,
|
||||
status: _ResponseStatus,
|
||||
reason: _StatusReason | None = None,
|
||||
) -> RealtimeResponse:
|
||||
"""Build a fully-populated RealtimeResponse from the current connection state."""
|
||||
st = self._state(conn_id)
|
||||
status_details = None
|
||||
if reason or status in ("completed", "cancelled", "incomplete", "failed"):
|
||||
status_details = RealtimeResponseStatus(type=status, reason=reason) # type: ignore[arg-type]
|
||||
|
||||
rp = st.current_response_params
|
||||
metadata = rp.metadata if rp and rp.metadata else None
|
||||
|
||||
voice: Optional[str] = None
|
||||
if rp and rp.audio and rp.audio.output and rp.audio.output.voice:
|
||||
voice = str(rp.audio.output.voice)
|
||||
if not voice:
|
||||
audio_cfg = st.runtime_config.session.audio
|
||||
audio_output = audio_cfg.output if audio_cfg is not None else None
|
||||
voice = str(audio_output.voice) if audio_output is not None and audio_output.voice else None
|
||||
|
||||
# Out-of-band responses are not threaded into any conversation: report a null id.
|
||||
conversation_id = None if is_out_of_band(rp) else st.conversation_id
|
||||
|
||||
return RealtimeResponse(
|
||||
id=st.current_response_id,
|
||||
object="realtime.response",
|
||||
status=status,
|
||||
status_details=status_details,
|
||||
audio=Audio(output=AudioOutput(voice=str(voice) if voice else None)), # type: ignore[arg-type]
|
||||
conversation_id=conversation_id,
|
||||
metadata=metadata,
|
||||
usage=RealtimeResponseUsage(
|
||||
input_tokens=st.response_usage.input_tokens,
|
||||
output_tokens=st.response_usage.output_tokens,
|
||||
total_tokens=st.response_usage.input_tokens + st.response_usage.output_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
# ── Public handlers ───────────────────────────
|
||||
|
||||
def handle_response_create(self, conn_id: str, event: ResponseCreateEvent) -> ServerEvent | None:
|
||||
"""Trigger a response.
|
||||
|
||||
Returns a ``ResponseCreatedEvent`` on success, a ``RealtimeErrorEvent``
|
||||
on failure, or ``None`` if there is no text_prompt_queue.
|
||||
"""
|
||||
st = self._state(conn_id)
|
||||
if event.response:
|
||||
if event.response.tool_choice and not isinstance(event.response.tool_choice, str):
|
||||
return self.make_error(
|
||||
message="Only string tool_choice values are supported for now (auto, required, none).",
|
||||
_type="tool_choice_not_supported",
|
||||
)
|
||||
if st.in_response:
|
||||
return self.make_error(
|
||||
message="Cannot create response while another response is in progress.",
|
||||
_type="conversation_already_has_active_response",
|
||||
)
|
||||
|
||||
out_of_band = is_out_of_band(event.response)
|
||||
|
||||
# In-band: response.input items are added to the default conversation here so
|
||||
# they appear in history. Out-of-band: leave the default conversation untouched —
|
||||
# the input rides along on the request and seeds a throwaway chat in the LM.
|
||||
if not out_of_band and event.response and event.response.input:
|
||||
for input_item in event.response.input:
|
||||
try:
|
||||
self._service.conversation._append_item(conn_id, input_item)
|
||||
except ChatItemError as exc:
|
||||
return self.make_error(message=str(exc), _type="invalid_input_item")
|
||||
|
||||
st.in_response = True
|
||||
st.response_pending = False
|
||||
|
||||
st.current_response_params = event.response
|
||||
st.current_response_id = _generate_id("resp")
|
||||
self._start_item(conn_id)
|
||||
if response_wants_audio(event.response):
|
||||
st.response_audio_wait_started_at_s = perf_counter()
|
||||
st.delay_filler_sent = False
|
||||
else:
|
||||
st.response_audio_wait_started_at_s = None
|
||||
st.delay_filler_sent = False
|
||||
|
||||
cfg = st.runtime_config
|
||||
queue = self._queue(conn_id)
|
||||
if queue:
|
||||
# Out-of-band responses carry no turn identity: a null turn_id makes every
|
||||
# speculative-turn staleness gate treat them as always-latest, so a new user
|
||||
# turn mid-generation can never silently drop their output.
|
||||
queue.put(
|
||||
GenerateResponseRequest(
|
||||
runtime_config=cfg,
|
||||
response=event.response,
|
||||
turn_id=None if out_of_band else st.speculative_user_turn_id,
|
||||
turn_revision=None if out_of_band else st.speculative_user_turn_revision,
|
||||
speech_stopped_at_s=None if out_of_band else st.speculative_user_speech_stopped_at_s,
|
||||
)
|
||||
)
|
||||
logger.debug("response.create received, LLM generation triggered")
|
||||
return ResponseCreatedEvent(
|
||||
type="response.created",
|
||||
event_id=self._next_event_id(),
|
||||
response=self._build_response(conn_id, "in_progress"),
|
||||
)
|
||||
|
||||
def handle_response_cancel(self, conn_id: str) -> list[ServerEvent]:
|
||||
"""Cancel the in-progress response and re-enable listening."""
|
||||
events = self.finish_response(conn_id, status="cancelled", reason="client_cancelled")
|
||||
should_listen = self._should_listen(conn_id)
|
||||
if should_listen:
|
||||
should_listen.set()
|
||||
logger.info("Response cancelled, listening re-enabled")
|
||||
return events
|
||||
|
||||
def finish_response(
|
||||
self,
|
||||
conn_id: str,
|
||||
status: _ResponseStatus = "completed",
|
||||
reason: _StatusReason | None = None,
|
||||
) -> list[ServerEvent]:
|
||||
"""Close the current response (audio/text done + response done).
|
||||
|
||||
Audio responses emit ``response.output_audio.done`` for any terminal
|
||||
status. Text-only responses emit a single ``response.output_text.done``
|
||||
carrying the full streamed text, but only on ``status="completed"`` —
|
||||
a cancelled or failed text response sends no audio, so it just closes
|
||||
with ``response.done``.
|
||||
"""
|
||||
st = self._state(conn_id)
|
||||
events: list[ServerEvent] = []
|
||||
if st.in_response:
|
||||
resp_id, item_id = self._ensure_response(conn_id)
|
||||
if response_wants_audio(st.current_response_params):
|
||||
events.append(
|
||||
ResponseAudioDoneEvent(
|
||||
type="response.output_audio.done",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=0,
|
||||
response_id=resp_id,
|
||||
)
|
||||
)
|
||||
elif status == "completed" and st.pending_output_text_parts:
|
||||
events.append(
|
||||
ResponseTextDoneEvent(
|
||||
type="response.output_text.done",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=0,
|
||||
response_id=resp_id,
|
||||
text="".join(st.pending_output_text_parts),
|
||||
)
|
||||
)
|
||||
events.append(
|
||||
ResponseDoneEvent(
|
||||
type="response.done",
|
||||
event_id=self._next_event_id(),
|
||||
response=self._build_response(conn_id, status, reason),
|
||||
)
|
||||
)
|
||||
self._end_response(conn_id, status)
|
||||
# Apply any client items that arrived mid-generation now that in_response
|
||||
# is cleared and the generation's own write-back has landed. Done outside
|
||||
# the in_response guard so a stray terminal call still drains the buffer.
|
||||
events.extend(self._service.conversation.flush_deferred_items(conn_id))
|
||||
return events
|
||||
|
||||
# ── Pipeline event handlers ───────────────────
|
||||
|
||||
def on_assistant_text(
|
||||
self,
|
||||
conn_id: str,
|
||||
event: AssistantTextEvent,
|
||||
*,
|
||||
wait_for_pending_reopen: bool = True,
|
||||
) -> list[ServerEvent] | None:
|
||||
"""Handle assistant_text: emit transcript and/or tool-call events."""
|
||||
if self._service.speculative_turns:
|
||||
commit_result: bool | None
|
||||
if wait_for_pending_reopen:
|
||||
commit_result = self._service.speculative_turns.commit_if_latest_after_reopen_grace(
|
||||
event.turn_id,
|
||||
event.turn_revision,
|
||||
)
|
||||
else:
|
||||
commit_result = self._service.speculative_turns.try_commit_if_latest_after_reopen_grace(
|
||||
event.turn_id,
|
||||
event.turn_revision,
|
||||
)
|
||||
if commit_result is None:
|
||||
return None
|
||||
if not commit_result:
|
||||
logger.debug("Dropping stale assistant text for turn=%s rev=%s", event.turn_id, event.turn_revision)
|
||||
return []
|
||||
st = self._state(conn_id)
|
||||
events: list[ServerEvent] = []
|
||||
resp_id, item_id = self._ensure_response(conn_id)
|
||||
st.last_item_id = item_id
|
||||
output_idx = 0
|
||||
if event.text:
|
||||
if response_wants_audio(st.current_response_params):
|
||||
events.append(
|
||||
ResponseAudioTranscriptDoneEvent(
|
||||
type="response.output_audio_transcript.done",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=output_idx,
|
||||
response_id=resp_id,
|
||||
transcript=event.text,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Stream the delta now; the matching response.output_text.done is
|
||||
# emitted once at close in finish_response, carrying the per-chunk
|
||||
# parts collected here, space-joined ("" when there were none).
|
||||
st.pending_output_text_parts.append(event.text)
|
||||
events.append(
|
||||
ResponseTextDeltaEvent(
|
||||
type="response.output_text.delta",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=output_idx,
|
||||
response_id=resp_id,
|
||||
delta=event.text,
|
||||
)
|
||||
)
|
||||
output_idx += 1
|
||||
if event.tools:
|
||||
st.response_usage.tool_calls += len(event.tools)
|
||||
for tool in event.tools:
|
||||
events.append(
|
||||
ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
event_id=self._next_event_id(),
|
||||
call_id=tool.call_id,
|
||||
name=tool.name,
|
||||
arguments=tool.arguments,
|
||||
item_id=item_id,
|
||||
output_index=output_idx,
|
||||
response_id=resp_id,
|
||||
)
|
||||
)
|
||||
output_idx += 1
|
||||
return events
|
||||
Reference in New Issue
Block a user