first
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
from speech_to_speech.api.openai_realtime.handlers.audio import AudioHandler
|
||||
from speech_to_speech.api.openai_realtime.handlers.conversation import ConversationHandler
|
||||
from speech_to_speech.api.openai_realtime.handlers.response import ResponseHandler
|
||||
from speech_to_speech.api.openai_realtime.handlers.session import SessionHandler
|
||||
|
||||
__all__ = [
|
||||
"AudioHandler",
|
||||
"ConversationHandler",
|
||||
"ResponseHandler",
|
||||
"SessionHandler",
|
||||
]
|
||||
@@ -0,0 +1,199 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from openai.types.realtime import (
|
||||
InputAudioBufferAppendEvent,
|
||||
InputAudioBufferSpeechStartedEvent,
|
||||
InputAudioBufferSpeechStoppedEvent,
|
||||
RealtimeErrorEvent,
|
||||
ResponseAudioDeltaEvent,
|
||||
ResponseCreatedEvent,
|
||||
)
|
||||
|
||||
from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
|
||||
from speech_to_speech.api.openai_realtime.utils import resample
|
||||
from speech_to_speech.pipeline.events import SpeechStartedEvent, SpeechStoppedEvent
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from speech_to_speech.api.openai_realtime.service import ServerEvent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PIPELINE_SAMPLE_RATE = 16000
|
||||
CHUNK_SAMPLES = 512
|
||||
BYTES_PER_SAMPLE = 2
|
||||
CHUNK_SIZE_BYTES = CHUNK_SAMPLES * BYTES_PER_SAMPLE
|
||||
|
||||
|
||||
class AudioHandler(RealtimeBaseHandler):
|
||||
"""Owns inbound audio decoding/chunking and outbound audio encoding."""
|
||||
|
||||
def _start_input_item(self, conn_id: str, *, preserve_active_response: bool = False) -> str:
|
||||
response = self._service.response
|
||||
st = self._state(conn_id)
|
||||
if not preserve_active_response:
|
||||
item_id = response._start_item(conn_id)
|
||||
else:
|
||||
response_item_id = st.current_item_id
|
||||
response_content_index = st.content_index
|
||||
item_id = response._start_item(conn_id)
|
||||
st.current_item_id = response_item_id
|
||||
st.content_index = response_content_index
|
||||
st.input_content_index = 0
|
||||
return item_id
|
||||
|
||||
def handle_audio_append(self, conn_id: str, event: InputAudioBufferAppendEvent) -> list[bytes]:
|
||||
"""Decode base64 audio, resample to pipeline rate, and split into 512-sample PCM16 chunks for the VAD."""
|
||||
try:
|
||||
pcm_bytes = base64.b64decode(event.audio)
|
||||
except Exception as e:
|
||||
logger.error(f"Base64 decode error: {e}")
|
||||
return []
|
||||
|
||||
st = self._state(conn_id)
|
||||
|
||||
audio_cfg = st.runtime_config.session.audio
|
||||
if audio_cfg is not None and audio_cfg.input is not None:
|
||||
client_in_rate = getattr(audio_cfg.input.format, "rate", None) or PIPELINE_SAMPLE_RATE
|
||||
else:
|
||||
client_in_rate = PIPELINE_SAMPLE_RATE
|
||||
pcm_bytes = resample(pcm_bytes, client_in_rate, PIPELINE_SAMPLE_RATE)
|
||||
|
||||
pcm_bytes = st.audio_remainder + pcm_bytes
|
||||
|
||||
chunks = []
|
||||
for i in range(0, len(pcm_bytes), CHUNK_SIZE_BYTES):
|
||||
chunk = pcm_bytes[i : i + CHUNK_SIZE_BYTES]
|
||||
if len(chunk) == CHUNK_SIZE_BYTES:
|
||||
chunks.append(chunk)
|
||||
else:
|
||||
st.audio_remainder = chunk
|
||||
break
|
||||
else:
|
||||
st.audio_remainder = b""
|
||||
|
||||
if chunks:
|
||||
st.audio_buffer_has_data = True
|
||||
return chunks
|
||||
|
||||
def handle_audio_commit(self, conn_id: str) -> RealtimeErrorEvent | None:
|
||||
"""Commit the audio buffer. Returns an error if no audio was appended."""
|
||||
st = self._state(conn_id)
|
||||
if not st.audio_buffer_has_data:
|
||||
return self.make_error(
|
||||
message="Input audio buffer is empty, nothing to commit.",
|
||||
_type="input_audio_buffer_commit_empty",
|
||||
)
|
||||
st.audio_buffer_has_data = False
|
||||
logger.debug("Audio buffer committed")
|
||||
return None
|
||||
|
||||
# ── Pipeline event handlers ────────────────────
|
||||
|
||||
def on_speech_started(self, conn_id: str, event: SpeechStartedEvent) -> list[ServerEvent]:
|
||||
"""Handle VAD speech_started: cancel active response if interrupts enabled, start new input item."""
|
||||
response = self._service.response
|
||||
events: list[ServerEvent] = []
|
||||
st = self._state(conn_id)
|
||||
if st.in_response and event.interrupt_response and st.runtime_config.interrupt_response_enabled:
|
||||
events.extend(response.finish_response(conn_id, status="cancelled", reason="turn_detected"))
|
||||
is_reopen = bool(event.reopened and event.turn_id is not None and event.turn_id == st.speculative_turn_id)
|
||||
preserve_active_response = st.in_response
|
||||
if is_reopen:
|
||||
input_item_id = st.speculative_input_item_id
|
||||
if input_item_id is None:
|
||||
input_item_id = self._start_input_item(
|
||||
conn_id,
|
||||
preserve_active_response=preserve_active_response,
|
||||
)
|
||||
st.speculative_input_item_id = input_item_id
|
||||
elif not preserve_active_response:
|
||||
st.current_item_id = input_item_id
|
||||
st.content_index = 0
|
||||
st.input_audio_duration_s = 0.0
|
||||
st.input_content_index = 0
|
||||
else:
|
||||
input_item_id = self._start_input_item(
|
||||
conn_id,
|
||||
preserve_active_response=preserve_active_response,
|
||||
)
|
||||
st.speculative_input_item_id = input_item_id
|
||||
st.response_usage.turns += 1
|
||||
st.speculative_turn_id = event.turn_id
|
||||
st.speculative_turn_revision = event.turn_revision
|
||||
st.last_item_id = input_item_id
|
||||
events.append(
|
||||
InputAudioBufferSpeechStartedEvent(
|
||||
type="input_audio_buffer.speech_started",
|
||||
event_id=self._next_event_id(),
|
||||
audio_start_ms=event.audio_start_ms,
|
||||
item_id=input_item_id,
|
||||
)
|
||||
)
|
||||
return events
|
||||
|
||||
def on_speech_stopped(self, conn_id: str, event: SpeechStoppedEvent) -> list[ServerEvent]:
|
||||
"""Handle VAD speech_stopped: record duration and emit stopped event."""
|
||||
if event.duration_s:
|
||||
self._state(conn_id).input_audio_duration_s = event.duration_s
|
||||
return [
|
||||
InputAudioBufferSpeechStoppedEvent(
|
||||
type="input_audio_buffer.speech_stopped",
|
||||
event_id=self._next_event_id(),
|
||||
audio_end_ms=event.audio_end_ms,
|
||||
item_id=self._input_item_id(conn_id),
|
||||
)
|
||||
]
|
||||
|
||||
# ── Outbound audio encoding ──────────────────
|
||||
|
||||
def encode_audio_chunk(self, conn_id: str, audio: bytes) -> list[ServerEvent]:
|
||||
"""Encode a raw PCM audio chunk, emitting ResponseCreated on the first chunk.
|
||||
|
||||
When ``handle_response_create`` already allocated the response,
|
||||
``current_response_id`` is set and no duplicate event is emitted.
|
||||
For the implicit-response path (VAD -> STT -> LLM -> TTS, no
|
||||
``response.create``), ``current_response_id`` is still ``None``
|
||||
and the event is emitted here on the first audio chunk.
|
||||
"""
|
||||
response = self._service.response
|
||||
st = self._state(conn_id)
|
||||
|
||||
events: list[ServerEvent] = []
|
||||
need_created = st.current_response_id is None
|
||||
resp_id, item_id = response._ensure_response(conn_id)
|
||||
if need_created:
|
||||
events.append(
|
||||
ResponseCreatedEvent(
|
||||
type="response.created",
|
||||
event_id=self._next_event_id(),
|
||||
response=response._build_response(conn_id, "in_progress"),
|
||||
)
|
||||
)
|
||||
rp = st.current_response_params
|
||||
client_out_rate = None
|
||||
if rp and rp.audio and rp.audio.output and rp.audio.output.format:
|
||||
client_out_rate = getattr(rp.audio.output.format, "rate", None)
|
||||
if client_out_rate is None:
|
||||
audio_cfg = st.runtime_config.session.audio
|
||||
if audio_cfg is not None and audio_cfg.output is not None:
|
||||
client_out_rate = getattr(audio_cfg.output.format, "rate", None) or PIPELINE_SAMPLE_RATE
|
||||
else:
|
||||
client_out_rate = PIPELINE_SAMPLE_RATE
|
||||
audio = resample(audio, PIPELINE_SAMPLE_RATE, client_out_rate)
|
||||
b64 = base64.b64encode(audio).decode("ascii")
|
||||
events.append(
|
||||
ResponseAudioDeltaEvent(
|
||||
type="response.output_audio.delta",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=response._next_content_index(conn_id),
|
||||
delta=b64,
|
||||
item_id=item_id,
|
||||
output_index=0,
|
||||
response_id=resp_id,
|
||||
)
|
||||
)
|
||||
return events
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from queue import Queue
|
||||
from threading import Event as ThreadingEvent
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from openai.types.realtime import RealtimeErrorEvent
|
||||
|
||||
from speech_to_speech.utils.utils import _generate_id
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from speech_to_speech.api.openai_realtime.service import ConnState, RealtimeService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RealtimeBaseHandler:
|
||||
"""Shared base for domain handlers.
|
||||
|
||||
Provides conn_id-keyed accessors for per-connection state, config,
|
||||
and queues. Each connection owns its own ``RuntimeConfig`` via
|
||||
``ConnState``.
|
||||
"""
|
||||
|
||||
def __init__(self, service: RealtimeService) -> None:
|
||||
self._service = service
|
||||
|
||||
# ── conn_id-keyed accessors ──
|
||||
|
||||
def _queue(self, conn_id: str) -> Queue | None:
|
||||
return self._service.text_prompt_queue
|
||||
|
||||
def _should_listen(self, conn_id: str) -> ThreadingEvent | None:
|
||||
return self._service.should_listen
|
||||
|
||||
# ── shared helpers ────────────────────────────────
|
||||
|
||||
def _state(self, conn_id: str) -> ConnState:
|
||||
return self._service._state(conn_id)
|
||||
|
||||
def _input_item_id(self, conn_id: str) -> str:
|
||||
st = self._state(conn_id)
|
||||
return st.speculative_input_item_id or self._service.response._current_item_id(conn_id)
|
||||
|
||||
def _next_input_content_index(self, conn_id: str) -> int:
|
||||
st = self._state(conn_id)
|
||||
idx = st.input_content_index
|
||||
st.input_content_index += 1
|
||||
return idx
|
||||
|
||||
@staticmethod
|
||||
def _next_event_id() -> str:
|
||||
return _generate_id("event")
|
||||
|
||||
def make_error(self, message: str, _type: str) -> RealtimeErrorEvent:
|
||||
return self._service.make_error(message, _type)
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from openai.types.realtime import (
|
||||
ConversationItem,
|
||||
ConversationItemCreatedEvent,
|
||||
ConversationItemCreateEvent,
|
||||
ConversationItemInputAudioTranscriptionCompletedEvent,
|
||||
ConversationItemInputAudioTranscriptionDeltaEvent,
|
||||
)
|
||||
from openai.types.realtime.conversation_item_input_audio_transcription_completed_event import (
|
||||
UsageTranscriptTextUsageDuration,
|
||||
)
|
||||
|
||||
from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
|
||||
from speech_to_speech.LLM.chat import ChatItemError, add_supported_item
|
||||
from speech_to_speech.pipeline.events import PartialTranscriptionEvent, TranscriptionCompletedEvent
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from speech_to_speech.api.openai_realtime.service import ServerEvent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConversationHandler(RealtimeBaseHandler):
|
||||
"""Owns conversation item injection and pipeline-to-protocol translation."""
|
||||
|
||||
def handle_conversation_item_create(
|
||||
self,
|
||||
conn_id: str,
|
||||
event: ConversationItemCreateEvent,
|
||||
) -> list[ServerEvent]:
|
||||
"""Inject a text message or function-call output into the LLM context.
|
||||
|
||||
Items are added to the LLM chat context but do NOT trigger response
|
||||
generation on their own. A subsequent ``response.create`` event is
|
||||
required to trigger the model.
|
||||
|
||||
While a response is generating, the item is *deferred*: applying it now
|
||||
would race the LLM handler's end-of-turn chat write-back, which runs on
|
||||
the pipeline thread (e.g. a ``function_call_output`` arriving before its
|
||||
``function_call`` is recorded, or an image stripped before the next
|
||||
turn reads it). Deferred items are flushed, in order, once the response
|
||||
completes — see :meth:`flush_deferred_items`.
|
||||
"""
|
||||
st = self._state(conn_id)
|
||||
if st.in_response:
|
||||
st.deferred_items.append(event.item)
|
||||
logger.debug("Deferred conversation item until the active response completes")
|
||||
return []
|
||||
return self._apply_item(conn_id, event.item)
|
||||
|
||||
def _apply_item(self, conn_id: str, item: ConversationItem) -> list[ServerEvent]:
|
||||
"""Add one item to the chat and build its ``conversation.item.created``."""
|
||||
try:
|
||||
self._append_item(conn_id, item)
|
||||
except ChatItemError as exc:
|
||||
return [self.make_error(str(exc), "invalid_conversation_item")]
|
||||
|
||||
if not item:
|
||||
return []
|
||||
st = self._state(conn_id)
|
||||
event = ConversationItemCreatedEvent(
|
||||
type="conversation.item.created",
|
||||
event_id=self._next_event_id(),
|
||||
previous_item_id=st.last_item_id,
|
||||
item=item,
|
||||
)
|
||||
st.last_item_id = item.id
|
||||
return [event]
|
||||
|
||||
def flush_deferred_items(self, conn_id: str) -> list[ServerEvent]:
|
||||
"""Apply items buffered during a response, in arrival order.
|
||||
|
||||
Called at response completion (after the generation's own write-back),
|
||||
so a ``function_call_output`` pairs with its now-recorded ``function_call``
|
||||
and an image survives the just-finished response's ``strip_images``.
|
||||
"""
|
||||
st = self._state(conn_id)
|
||||
if not st.deferred_items:
|
||||
return []
|
||||
items = st.deferred_items
|
||||
st.deferred_items = []
|
||||
events: list[ServerEvent] = []
|
||||
for item in items:
|
||||
events.extend(self._apply_item(conn_id, item))
|
||||
return events
|
||||
|
||||
def _append_item(self, conn_id: str, item: ConversationItem) -> None:
|
||||
"""Narrow ``ConversationItem`` to ``SupportedItem`` and delegate to ``Chat.add_item``.
|
||||
|
||||
Raises :class:`ChatItemError` on validation failure or unsupported type.
|
||||
"""
|
||||
add_supported_item(self._state(conn_id).runtime_config.chat, item)
|
||||
|
||||
# ── Pipeline event handlers ────────────────────
|
||||
|
||||
def on_partial_transcription(self, conn_id: str, event: PartialTranscriptionEvent) -> list[ServerEvent]:
|
||||
"""Handle partial_transcription: emit transcription delta event."""
|
||||
return [
|
||||
ConversationItemInputAudioTranscriptionDeltaEvent(
|
||||
type="conversation.item.input_audio_transcription.delta",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=self._next_input_content_index(conn_id),
|
||||
item_id=self._input_item_id(conn_id),
|
||||
delta=event.delta,
|
||||
)
|
||||
]
|
||||
|
||||
def on_transcription_completed(self, conn_id: str, event: TranscriptionCompletedEvent) -> list[ServerEvent]:
|
||||
"""Handle transcription_completed: accumulate duration and emit completed event."""
|
||||
st = self._state(conn_id)
|
||||
st.response_usage.audio_duration_s += st.input_audio_duration_s
|
||||
return [
|
||||
ConversationItemInputAudioTranscriptionCompletedEvent(
|
||||
type="conversation.item.input_audio_transcription.completed",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=self._input_item_id(conn_id),
|
||||
transcript=event.transcript,
|
||||
usage=UsageTranscriptTextUsageDuration(
|
||||
seconds=st.input_audio_duration_s,
|
||||
type="duration",
|
||||
),
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,348 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from time import perf_counter
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from openai.types.realtime import (
|
||||
RealtimeResponse,
|
||||
ResponseAudioDoneEvent,
|
||||
ResponseAudioTranscriptDoneEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseCreateEvent,
|
||||
ResponseDoneEvent,
|
||||
ResponseFunctionCallArgumentsDoneEvent,
|
||||
ResponseTextDeltaEvent,
|
||||
ResponseTextDoneEvent,
|
||||
)
|
||||
from openai.types.realtime.realtime_response import Audio, AudioOutput
|
||||
from openai.types.realtime.realtime_response_status import RealtimeResponseStatus
|
||||
from openai.types.realtime.realtime_response_usage import RealtimeResponseUsage
|
||||
|
||||
from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
|
||||
from speech_to_speech.LLM.chat import ChatItemError
|
||||
from speech_to_speech.pipeline.events import AssistantTextEvent
|
||||
from speech_to_speech.pipeline.messages import GenerateResponseRequest
|
||||
from speech_to_speech.utils.utils import _generate_id, is_out_of_band, response_wants_audio
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from speech_to_speech.api.openai_realtime.service import ServerEvent, _ResponseStatus, _StatusReason
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ResponseHandler(RealtimeBaseHandler):
|
||||
"""Owns the response lifecycle: create, cancel, finish, and ID management."""
|
||||
|
||||
# ── ID / state helpers ────────────────────────
|
||||
|
||||
def _ensure_response(self, conn_id: str) -> tuple[str, str]:
|
||||
"""Ensure a response and output item exist, creating them if needed."""
|
||||
st = self._state(conn_id)
|
||||
if st.current_response_id is None:
|
||||
st.current_response_id = _generate_id("resp")
|
||||
self._start_item(conn_id)
|
||||
st.in_response = True
|
||||
st.response_pending = False
|
||||
return st.current_response_id, self._current_item_id(conn_id)
|
||||
|
||||
def _end_response(self, conn_id: str, status: _ResponseStatus = "completed") -> None:
|
||||
st = self._state(conn_id)
|
||||
if status == "cancelled":
|
||||
st.response_usage.responses_cancelled += 1
|
||||
else:
|
||||
st.response_usage.responses_completed += 1
|
||||
self._service.total_usage += st.response_usage
|
||||
logger.info(
|
||||
"Response done (status=%s) — this response: input_tokens=%d, output_tokens=%d, audio=%.2fs"
|
||||
" | cumulative: input_tokens=%d, output_tokens=%d, audio=%.2fs",
|
||||
status,
|
||||
st.response_usage.input_tokens,
|
||||
st.response_usage.output_tokens,
|
||||
st.response_usage.audio_duration_s,
|
||||
self._service.total_usage.input_tokens,
|
||||
self._service.total_usage.output_tokens,
|
||||
self._service.total_usage.audio_duration_s,
|
||||
)
|
||||
st.response_usage.reset()
|
||||
st.current_response_id = None
|
||||
st.current_item_id = None
|
||||
st.content_index = 0
|
||||
st.in_response = False
|
||||
st.response_pending = False
|
||||
st.current_response_params = None
|
||||
st.pending_output_text_parts = []
|
||||
st.response_audio_wait_started_at_s = None
|
||||
st.delay_filler_sent = False
|
||||
|
||||
def _start_item(self, conn_id: str) -> str:
|
||||
"""Generate a new item ID, reset content index, and store it."""
|
||||
st = self._state(conn_id)
|
||||
item_id = _generate_id("item")
|
||||
st.current_item_id = item_id
|
||||
st.content_index = 0
|
||||
st.input_audio_duration_s = 0.0
|
||||
return item_id
|
||||
|
||||
def _current_item_id(self, conn_id: str) -> str:
|
||||
return self._state(conn_id).current_item_id or self._start_item(conn_id)
|
||||
|
||||
def _next_content_index(self, conn_id: str) -> int:
|
||||
"""Return the current content index and advance it."""
|
||||
st = self._state(conn_id)
|
||||
idx = st.content_index
|
||||
st.content_index += 1
|
||||
return idx
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
conn_id: str,
|
||||
status: _ResponseStatus,
|
||||
reason: _StatusReason | None = None,
|
||||
) -> RealtimeResponse:
|
||||
"""Build a fully-populated RealtimeResponse from the current connection state."""
|
||||
st = self._state(conn_id)
|
||||
status_details = None
|
||||
if reason or status in ("completed", "cancelled", "incomplete", "failed"):
|
||||
status_details = RealtimeResponseStatus(type=status, reason=reason) # type: ignore[arg-type]
|
||||
|
||||
rp = st.current_response_params
|
||||
metadata = rp.metadata if rp and rp.metadata else None
|
||||
|
||||
voice: Optional[str] = None
|
||||
if rp and rp.audio and rp.audio.output and rp.audio.output.voice:
|
||||
voice = str(rp.audio.output.voice)
|
||||
if not voice:
|
||||
audio_cfg = st.runtime_config.session.audio
|
||||
audio_output = audio_cfg.output if audio_cfg is not None else None
|
||||
voice = str(audio_output.voice) if audio_output is not None and audio_output.voice else None
|
||||
|
||||
# Out-of-band responses are not threaded into any conversation: report a null id.
|
||||
conversation_id = None if is_out_of_band(rp) else st.conversation_id
|
||||
|
||||
return RealtimeResponse(
|
||||
id=st.current_response_id,
|
||||
object="realtime.response",
|
||||
status=status,
|
||||
status_details=status_details,
|
||||
audio=Audio(output=AudioOutput(voice=str(voice) if voice else None)), # type: ignore[arg-type]
|
||||
conversation_id=conversation_id,
|
||||
metadata=metadata,
|
||||
usage=RealtimeResponseUsage(
|
||||
input_tokens=st.response_usage.input_tokens,
|
||||
output_tokens=st.response_usage.output_tokens,
|
||||
total_tokens=st.response_usage.input_tokens + st.response_usage.output_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
# ── Public handlers ───────────────────────────
|
||||
|
||||
def handle_response_create(self, conn_id: str, event: ResponseCreateEvent) -> ServerEvent | None:
|
||||
"""Trigger a response.
|
||||
|
||||
Returns a ``ResponseCreatedEvent`` on success, a ``RealtimeErrorEvent``
|
||||
on failure, or ``None`` if there is no text_prompt_queue.
|
||||
"""
|
||||
st = self._state(conn_id)
|
||||
if event.response:
|
||||
if event.response.tool_choice and not isinstance(event.response.tool_choice, str):
|
||||
return self.make_error(
|
||||
message="Only string tool_choice values are supported for now (auto, required, none).",
|
||||
_type="tool_choice_not_supported",
|
||||
)
|
||||
if st.in_response:
|
||||
return self.make_error(
|
||||
message="Cannot create response while another response is in progress.",
|
||||
_type="conversation_already_has_active_response",
|
||||
)
|
||||
|
||||
out_of_band = is_out_of_band(event.response)
|
||||
|
||||
# In-band: response.input items are added to the default conversation here so
|
||||
# they appear in history. Out-of-band: leave the default conversation untouched —
|
||||
# the input rides along on the request and seeds a throwaway chat in the LM.
|
||||
if not out_of_band and event.response and event.response.input:
|
||||
for input_item in event.response.input:
|
||||
try:
|
||||
self._service.conversation._append_item(conn_id, input_item)
|
||||
except ChatItemError as exc:
|
||||
return self.make_error(message=str(exc), _type="invalid_input_item")
|
||||
|
||||
st.in_response = True
|
||||
st.response_pending = False
|
||||
|
||||
st.current_response_params = event.response
|
||||
st.current_response_id = _generate_id("resp")
|
||||
self._start_item(conn_id)
|
||||
if response_wants_audio(event.response):
|
||||
st.response_audio_wait_started_at_s = perf_counter()
|
||||
st.delay_filler_sent = False
|
||||
else:
|
||||
st.response_audio_wait_started_at_s = None
|
||||
st.delay_filler_sent = False
|
||||
|
||||
cfg = st.runtime_config
|
||||
queue = self._queue(conn_id)
|
||||
if queue:
|
||||
# Out-of-band responses carry no turn identity: a null turn_id makes every
|
||||
# speculative-turn staleness gate treat them as always-latest, so a new user
|
||||
# turn mid-generation can never silently drop their output.
|
||||
queue.put(
|
||||
GenerateResponseRequest(
|
||||
runtime_config=cfg,
|
||||
response=event.response,
|
||||
turn_id=None if out_of_band else st.speculative_user_turn_id,
|
||||
turn_revision=None if out_of_band else st.speculative_user_turn_revision,
|
||||
speech_stopped_at_s=None if out_of_band else st.speculative_user_speech_stopped_at_s,
|
||||
)
|
||||
)
|
||||
logger.debug("response.create received, LLM generation triggered")
|
||||
return ResponseCreatedEvent(
|
||||
type="response.created",
|
||||
event_id=self._next_event_id(),
|
||||
response=self._build_response(conn_id, "in_progress"),
|
||||
)
|
||||
|
||||
def handle_response_cancel(self, conn_id: str) -> list[ServerEvent]:
|
||||
"""Cancel the in-progress response and re-enable listening."""
|
||||
events = self.finish_response(conn_id, status="cancelled", reason="client_cancelled")
|
||||
should_listen = self._should_listen(conn_id)
|
||||
if should_listen:
|
||||
should_listen.set()
|
||||
logger.info("Response cancelled, listening re-enabled")
|
||||
return events
|
||||
|
||||
def finish_response(
|
||||
self,
|
||||
conn_id: str,
|
||||
status: _ResponseStatus = "completed",
|
||||
reason: _StatusReason | None = None,
|
||||
) -> list[ServerEvent]:
|
||||
"""Close the current response (audio/text done + response done).
|
||||
|
||||
Audio responses emit ``response.output_audio.done`` for any terminal
|
||||
status. Text-only responses emit a single ``response.output_text.done``
|
||||
carrying the full streamed text, but only on ``status="completed"`` —
|
||||
a cancelled or failed text response sends no audio, so it just closes
|
||||
with ``response.done``.
|
||||
"""
|
||||
st = self._state(conn_id)
|
||||
events: list[ServerEvent] = []
|
||||
if st.in_response:
|
||||
resp_id, item_id = self._ensure_response(conn_id)
|
||||
if response_wants_audio(st.current_response_params):
|
||||
events.append(
|
||||
ResponseAudioDoneEvent(
|
||||
type="response.output_audio.done",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=0,
|
||||
response_id=resp_id,
|
||||
)
|
||||
)
|
||||
elif status == "completed" and st.pending_output_text_parts:
|
||||
events.append(
|
||||
ResponseTextDoneEvent(
|
||||
type="response.output_text.done",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=0,
|
||||
response_id=resp_id,
|
||||
text="".join(st.pending_output_text_parts),
|
||||
)
|
||||
)
|
||||
events.append(
|
||||
ResponseDoneEvent(
|
||||
type="response.done",
|
||||
event_id=self._next_event_id(),
|
||||
response=self._build_response(conn_id, status, reason),
|
||||
)
|
||||
)
|
||||
self._end_response(conn_id, status)
|
||||
# Apply any client items that arrived mid-generation now that in_response
|
||||
# is cleared and the generation's own write-back has landed. Done outside
|
||||
# the in_response guard so a stray terminal call still drains the buffer.
|
||||
events.extend(self._service.conversation.flush_deferred_items(conn_id))
|
||||
return events
|
||||
|
||||
# ── Pipeline event handlers ───────────────────
|
||||
|
||||
def on_assistant_text(
|
||||
self,
|
||||
conn_id: str,
|
||||
event: AssistantTextEvent,
|
||||
*,
|
||||
wait_for_pending_reopen: bool = True,
|
||||
) -> list[ServerEvent] | None:
|
||||
"""Handle assistant_text: emit transcript and/or tool-call events."""
|
||||
if self._service.speculative_turns:
|
||||
commit_result: bool | None
|
||||
if wait_for_pending_reopen:
|
||||
commit_result = self._service.speculative_turns.commit_if_latest_after_reopen_grace(
|
||||
event.turn_id,
|
||||
event.turn_revision,
|
||||
)
|
||||
else:
|
||||
commit_result = self._service.speculative_turns.try_commit_if_latest_after_reopen_grace(
|
||||
event.turn_id,
|
||||
event.turn_revision,
|
||||
)
|
||||
if commit_result is None:
|
||||
return None
|
||||
if not commit_result:
|
||||
logger.debug("Dropping stale assistant text for turn=%s rev=%s", event.turn_id, event.turn_revision)
|
||||
return []
|
||||
st = self._state(conn_id)
|
||||
events: list[ServerEvent] = []
|
||||
resp_id, item_id = self._ensure_response(conn_id)
|
||||
st.last_item_id = item_id
|
||||
output_idx = 0
|
||||
if event.text:
|
||||
if response_wants_audio(st.current_response_params):
|
||||
events.append(
|
||||
ResponseAudioTranscriptDoneEvent(
|
||||
type="response.output_audio_transcript.done",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=output_idx,
|
||||
response_id=resp_id,
|
||||
transcript=event.text,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Stream the delta now; the matching response.output_text.done is
|
||||
# emitted once at close in finish_response, carrying the per-chunk
|
||||
# parts collected here, space-joined ("" when there were none).
|
||||
st.pending_output_text_parts.append(event.text)
|
||||
events.append(
|
||||
ResponseTextDeltaEvent(
|
||||
type="response.output_text.delta",
|
||||
event_id=self._next_event_id(),
|
||||
content_index=0,
|
||||
item_id=item_id,
|
||||
output_index=output_idx,
|
||||
response_id=resp_id,
|
||||
delta=event.text,
|
||||
)
|
||||
)
|
||||
output_idx += 1
|
||||
if event.tools:
|
||||
st.response_usage.tool_calls += len(event.tools)
|
||||
for tool in event.tools:
|
||||
events.append(
|
||||
ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
event_id=self._next_event_id(),
|
||||
call_id=tool.call_id,
|
||||
name=tool.name,
|
||||
arguments=tool.arguments,
|
||||
item_id=item_id,
|
||||
output_index=output_idx,
|
||||
response_id=resp_id,
|
||||
)
|
||||
)
|
||||
output_idx += 1
|
||||
return events
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from openai.types.realtime import (
|
||||
RealtimeErrorEvent,
|
||||
SessionCreatedEvent,
|
||||
SessionUpdateEvent,
|
||||
)
|
||||
from openai.types.realtime.realtime_transcription_session_create_request import (
|
||||
RealtimeTranscriptionSessionCreateRequest,
|
||||
)
|
||||
|
||||
from speech_to_speech.api.openai_realtime.handlers.base import RealtimeBaseHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SessionHandler(RealtimeBaseHandler):
|
||||
"""Owns session lifecycle: config updates and session.created events."""
|
||||
|
||||
def handle_session_update(self, conn_id: str, event: SessionUpdateEvent) -> Optional[RealtimeErrorEvent]:
|
||||
"""Apply session config changes.
|
||||
|
||||
Only ``RealtimeSessionCreateRequest`` sessions are accepted;
|
||||
``RealtimeTranscriptionSessionCreateRequest`` sessions not yet supported.
|
||||
Incoming fields are deep-merged into the existing session so that
|
||||
partial updates preserve previously-set values.
|
||||
"""
|
||||
s = event.session
|
||||
|
||||
# Forza il bypass della voce inviata dal browser
|
||||
if hasattr(s, "voice") and s.voice is not None:
|
||||
s.voice = None
|
||||
elif isinstance(s, dict) and "voice" in s:
|
||||
s["voice"] = None
|
||||
|
||||
if s is None:
|
||||
return None
|
||||
|
||||
if isinstance(s, RealtimeTranscriptionSessionCreateRequest):
|
||||
return self.make_error(
|
||||
message="Only 'realtime' session type is supported; transcription sessions are not.",
|
||||
_type="invalid_session_type",
|
||||
)
|
||||
|
||||
model = getattr(s, "model", None)
|
||||
if model is not None:
|
||||
logger.info(f"Session model set to: {model}")
|
||||
|
||||
cfg = self._state(conn_id).runtime_config
|
||||
current = cfg.session
|
||||
if current is None:
|
||||
cfg.session = s
|
||||
else:
|
||||
cfg.apply_session_update(s)
|
||||
logger.info("Session configuration updated")
|
||||
return None
|
||||
|
||||
def build_session_created(self, conn_id: str) -> SessionCreatedEvent:
|
||||
"""Build a SessionCreatedEvent populated with the current config."""
|
||||
cfg = self._state(conn_id).runtime_config
|
||||
session = cfg.session
|
||||
return SessionCreatedEvent(
|
||||
type="session.created",
|
||||
event_id=self._next_event_id(),
|
||||
session=session,
|
||||
)
|
||||
Reference in New Issue
Block a user