129 lines
5.3 KiB
Python
129 lines
5.3 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import TYPE_CHECKING
|
|
|
|
from openai.types.realtime import (
|
|
ConversationItem,
|
|
ConversationItemCreatedEvent,
|
|
ConversationItemCreateEvent,
|
|
ConversationItemInputAudioTranscriptionCompletedEvent,
|
|
ConversationItemInputAudioTranscriptionDeltaEvent,
|
|
)
|
|
from openai.types.realtime.conversation_item_input_audio_transcription_completed_event import (
|
|
UsageTranscriptTextUsageDuration,
|
|
)
|
|
|
|
from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
|
|
from speech_to_speech.LLM.chat import ChatItemError, add_supported_item
|
|
from speech_to_speech.pipeline.events import PartialTranscriptionEvent, TranscriptionCompletedEvent
|
|
|
|
if TYPE_CHECKING:
|
|
from speech_to_speech.api.openai_realtime.service import ServerEvent
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ConversationHandler(RealtimeBaseHandler):
|
|
"""Owns conversation item injection and pipeline-to-protocol translation."""
|
|
|
|
def handle_conversation_item_create(
|
|
self,
|
|
conn_id: str,
|
|
event: ConversationItemCreateEvent,
|
|
) -> list[ServerEvent]:
|
|
"""Inject a text message or function-call output into the LLM context.
|
|
|
|
Items are added to the LLM chat context but do NOT trigger response
|
|
generation on their own. A subsequent ``response.create`` event is
|
|
required to trigger the model.
|
|
|
|
While a response is generating, the item is *deferred*: applying it now
|
|
would race the LLM handler's end-of-turn chat write-back, which runs on
|
|
the pipeline thread (e.g. a ``function_call_output`` arriving before its
|
|
``function_call`` is recorded, or an image stripped before the next
|
|
turn reads it). Deferred items are flushed, in order, once the response
|
|
completes — see :meth:`flush_deferred_items`.
|
|
"""
|
|
st = self._state(conn_id)
|
|
if st.in_response:
|
|
st.deferred_items.append(event.item)
|
|
logger.debug("Deferred conversation item until the active response completes")
|
|
return []
|
|
return self._apply_item(conn_id, event.item)
|
|
|
|
def _apply_item(self, conn_id: str, item: ConversationItem) -> list[ServerEvent]:
|
|
"""Add one item to the chat and build its ``conversation.item.created``."""
|
|
try:
|
|
self._append_item(conn_id, item)
|
|
except ChatItemError as exc:
|
|
return [self.make_error(str(exc), "invalid_conversation_item")]
|
|
|
|
if not item:
|
|
return []
|
|
st = self._state(conn_id)
|
|
event = ConversationItemCreatedEvent(
|
|
type="conversation.item.created",
|
|
event_id=self._next_event_id(),
|
|
previous_item_id=st.last_item_id,
|
|
item=item,
|
|
)
|
|
st.last_item_id = item.id
|
|
return [event]
|
|
|
|
def flush_deferred_items(self, conn_id: str) -> list[ServerEvent]:
|
|
"""Apply items buffered during a response, in arrival order.
|
|
|
|
Called at response completion (after the generation's own write-back),
|
|
so a ``function_call_output`` pairs with its now-recorded ``function_call``
|
|
and an image survives the just-finished response's ``strip_images``.
|
|
"""
|
|
st = self._state(conn_id)
|
|
if not st.deferred_items:
|
|
return []
|
|
items = st.deferred_items
|
|
st.deferred_items = []
|
|
events: list[ServerEvent] = []
|
|
for item in items:
|
|
events.extend(self._apply_item(conn_id, item))
|
|
return events
|
|
|
|
def _append_item(self, conn_id: str, item: ConversationItem) -> None:
|
|
"""Narrow ``ConversationItem`` to ``SupportedItem`` and delegate to ``Chat.add_item``.
|
|
|
|
Raises :class:`ChatItemError` on validation failure or unsupported type.
|
|
"""
|
|
add_supported_item(self._state(conn_id).runtime_config.chat, item)
|
|
|
|
# ── Pipeline event handlers ────────────────────
|
|
|
|
def on_partial_transcription(self, conn_id: str, event: PartialTranscriptionEvent) -> list[ServerEvent]:
|
|
"""Handle partial_transcription: emit transcription delta event."""
|
|
return [
|
|
ConversationItemInputAudioTranscriptionDeltaEvent(
|
|
type="conversation.item.input_audio_transcription.delta",
|
|
event_id=self._next_event_id(),
|
|
content_index=self._next_input_content_index(conn_id),
|
|
item_id=self._input_item_id(conn_id),
|
|
delta=event.delta,
|
|
)
|
|
]
|
|
|
|
def on_transcription_completed(self, conn_id: str, event: TranscriptionCompletedEvent) -> list[ServerEvent]:
|
|
"""Handle transcription_completed: accumulate duration and emit completed event."""
|
|
st = self._state(conn_id)
|
|
st.response_usage.audio_duration_s += st.input_audio_duration_s
|
|
return [
|
|
ConversationItemInputAudioTranscriptionCompletedEvent(
|
|
type="conversation.item.input_audio_transcription.completed",
|
|
event_id=self._next_event_id(),
|
|
content_index=0,
|
|
item_id=self._input_item_id(conn_id),
|
|
transcript=event.transcript,
|
|
usage=UsageTranscriptTextUsageDuration(
|
|
seconds=st.input_audio_duration_s,
|
|
type="duration",
|
|
),
|
|
)
|
|
]
|