first

speech_to_speech/api/openai_realtime/handlers/conversation.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from openai.types.realtime import (
+    ConversationItem,
+    ConversationItemCreatedEvent,
+    ConversationItemCreateEvent,
+    ConversationItemInputAudioTranscriptionCompletedEvent,
+    ConversationItemInputAudioTranscriptionDeltaEvent,
+)
+from openai.types.realtime.conversation_item_input_audio_transcription_completed_event import (
+    UsageTranscriptTextUsageDuration,
+)
+
+from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
+from speech_to_speech.LLM.chat import ChatItemError, add_supported_item
+from speech_to_speech.pipeline.events import PartialTranscriptionEvent, TranscriptionCompletedEvent
+
+if TYPE_CHECKING:
+    from speech_to_speech.api.openai_realtime.service import ServerEvent
+
+logger = logging.getLogger(__name__)
+
+
+class ConversationHandler(RealtimeBaseHandler):
+    """Owns conversation item injection and pipeline-to-protocol translation."""
+
+    def handle_conversation_item_create(
+        self,
+        conn_id: str,
+        event: ConversationItemCreateEvent,
+    ) -> list[ServerEvent]:
+        """Inject a text message or function-call output into the LLM context.
+
+        Items are added to the LLM chat context but do NOT trigger response
+        generation on their own.  A subsequent ``response.create`` event is
+        required to trigger the model.
+
+        While a response is generating, the item is *deferred*: applying it now
+        would race the LLM handler's end-of-turn chat write-back, which runs on
+        the pipeline thread (e.g. a ``function_call_output`` arriving before its
+        ``function_call`` is recorded, or an image stripped before the next
+        turn reads it). Deferred items are flushed, in order, once the response
+        completes — see :meth:`flush_deferred_items`.
+        """
+        st = self._state(conn_id)
+        if st.in_response:
+            st.deferred_items.append(event.item)
+            logger.debug("Deferred conversation item until the active response completes")
+            return []
+        return self._apply_item(conn_id, event.item)
+
+    def _apply_item(self, conn_id: str, item: ConversationItem) -> list[ServerEvent]:
+        """Add one item to the chat and build its ``conversation.item.created``."""
+        try:
+            self._append_item(conn_id, item)
+        except ChatItemError as exc:
+            return [self.make_error(str(exc), "invalid_conversation_item")]
+
+        if not item:
+            return []
+        st = self._state(conn_id)
+        event = ConversationItemCreatedEvent(
+            type="conversation.item.created",
+            event_id=self._next_event_id(),
+            previous_item_id=st.last_item_id,
+            item=item,
+        )
+        st.last_item_id = item.id
+        return [event]
+
+    def flush_deferred_items(self, conn_id: str) -> list[ServerEvent]:
+        """Apply items buffered during a response, in arrival order.
+
+        Called at response completion (after the generation's own write-back),
+        so a ``function_call_output`` pairs with its now-recorded ``function_call``
+        and an image survives the just-finished response's ``strip_images``.
+        """
+        st = self._state(conn_id)
+        if not st.deferred_items:
+            return []
+        items = st.deferred_items
+        st.deferred_items = []
+        events: list[ServerEvent] = []
+        for item in items:
+            events.extend(self._apply_item(conn_id, item))
+        return events
+
+    def _append_item(self, conn_id: str, item: ConversationItem) -> None:
+        """Narrow ``ConversationItem`` to ``SupportedItem`` and delegate to ``Chat.add_item``.
+
+        Raises :class:`ChatItemError` on validation failure or unsupported type.
+        """
+        add_supported_item(self._state(conn_id).runtime_config.chat, item)
+
+    # ── Pipeline event handlers ────────────────────
+
+    def on_partial_transcription(self, conn_id: str, event: PartialTranscriptionEvent) -> list[ServerEvent]:
+        """Handle partial_transcription: emit transcription delta event."""
+        return [
+            ConversationItemInputAudioTranscriptionDeltaEvent(
+                type="conversation.item.input_audio_transcription.delta",
+                event_id=self._next_event_id(),
+                content_index=self._next_input_content_index(conn_id),
+                item_id=self._input_item_id(conn_id),
+                delta=event.delta,
+            )
+        ]
+
+    def on_transcription_completed(self, conn_id: str, event: TranscriptionCompletedEvent) -> list[ServerEvent]:
+        """Handle transcription_completed: accumulate duration and emit completed event."""
+        st = self._state(conn_id)
+        st.response_usage.audio_duration_s += st.input_audio_duration_s
+        return [
+            ConversationItemInputAudioTranscriptionCompletedEvent(
+                type="conversation.item.input_audio_transcription.completed",
+                event_id=self._next_event_id(),
+                content_index=0,
+                item_id=self._input_item_id(conn_id),
+                transcript=event.transcript,
+                usage=UsageTranscriptTextUsageDuration(
+                    seconds=st.input_audio_duration_s,
+                    type="duration",
+                ),
+            )
+        ]