312 lines
14 KiB
Python
312 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import time
|
|
from collections.abc import Iterator
|
|
from typing import Any, cast
|
|
|
|
from openai import Stream
|
|
from openai.types.chat import (
|
|
ChatCompletionChunk,
|
|
ChatCompletionContentPartImageParam,
|
|
ChatCompletionContentPartParam,
|
|
ChatCompletionContentPartTextParam,
|
|
ChatCompletionNamedToolChoiceParam,
|
|
ChatCompletionToolChoiceOptionParam,
|
|
ChatCompletionToolParam,
|
|
)
|
|
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
|
|
from openai.types.chat.chat_completion_named_tool_choice_param import Function as NamedToolChoiceFunction
|
|
from openai.types.realtime.realtime_conversation_item_assistant_message import (
|
|
Content as AssistantContent,
|
|
)
|
|
from openai.types.responses import ResponseFunctionToolCall
|
|
from openai.types.shared_params import FunctionDefinition
|
|
|
|
from speech_to_speech.LLM.base_openai_compatible_language_model import (
|
|
WARMUP_MAX_RETRIES,
|
|
AssistantMessage,
|
|
BaseOpenAICompatibleHandler,
|
|
ProviderEvent,
|
|
TextDelta,
|
|
ToolCall,
|
|
Usage,
|
|
)
|
|
from speech_to_speech.LLM.chat import Chat
|
|
from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn
|
|
from speech_to_speech.utils.utils import _generate_id
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _to_chat_tools(req_tools: Any) -> list[ChatCompletionToolParam] | None:
|
|
"""Convert Responses-API function tools to Chat-Completions tool format.
|
|
|
|
Responses tools are flat ``{type:"function", name, description, parameters}``;
|
|
Chat Completions nests them under a ``function`` key. Items already in the
|
|
nested form (or non-function tools) are passed through untouched.
|
|
"""
|
|
if not req_tools:
|
|
return None
|
|
chat_tools: list[ChatCompletionToolParam] = []
|
|
for t in req_tools:
|
|
d = t if isinstance(t, dict) else t.model_dump(exclude_none=True)
|
|
if d.get("type") == "function" and "function" not in d:
|
|
fn = FunctionDefinition(name=d["name"])
|
|
if d.get("description") is not None:
|
|
fn["description"] = d["description"]
|
|
if d.get("parameters") is not None:
|
|
fn["parameters"] = d["parameters"]
|
|
chat_tools.append(ChatCompletionToolParam(type="function", function=fn))
|
|
else:
|
|
chat_tools.append(cast("ChatCompletionToolParam", d))
|
|
return chat_tools
|
|
|
|
|
|
def _to_chat_tool_choice(tool_choice: Any) -> ChatCompletionToolChoiceOptionParam:
|
|
"""Convert a Responses-API tool_choice to Chat-Completions form.
|
|
|
|
The string forms ("auto"/"required"/"none") are identical across both APIs;
|
|
only the forced-function object differs (flat ``name`` vs nested ``function``).
|
|
"""
|
|
if isinstance(tool_choice, dict) and tool_choice.get("type") == "function" and "name" in tool_choice:
|
|
return ChatCompletionNamedToolChoiceParam(
|
|
type="function", function=NamedToolChoiceFunction(name=tool_choice["name"])
|
|
)
|
|
if tool_choice is not None and not isinstance(tool_choice, (str, dict)):
|
|
d = tool_choice.model_dump(exclude_none=True)
|
|
if d.get("type") == "function" and "name" in d:
|
|
return ChatCompletionNamedToolChoiceParam(type="function", function=NamedToolChoiceFunction(name=d["name"]))
|
|
return cast("ChatCompletionToolChoiceOptionParam", d)
|
|
return cast("ChatCompletionToolChoiceOptionParam", tool_choice)
|
|
|
|
|
|
class ChatCompletionsApiModelHandler(BaseOpenAICompatibleHandler):
|
|
"""LLM handler that talks to an OpenAI-compatible ``/v1/chat/completions`` server.
|
|
|
|
Functionally mirrors :class:`ResponsesApiModelHandler` but uses the mature
|
|
Chat Completions streaming tool-call protocol (``choices[].delta.tool_calls``)
|
|
instead of ``/v1/responses``. This is the robust path for vLLM + Qwen tool
|
|
calling. The conversation is serialised with :meth:`Chat.to_transformers_chat`,
|
|
which already emits OpenAI chat messages including ``tool_calls``/``tool`` roles.
|
|
"""
|
|
|
|
def warmup(self) -> None:
|
|
logger.info(f"Warming up {self.__class__.__name__}")
|
|
start = time.time()
|
|
self.client.with_options(max_retries=WARMUP_MAX_RETRIES).chat.completions.create(
|
|
model=self.model_name,
|
|
messages=[
|
|
{"role": "system", "content": "You are a helpful assistant"},
|
|
{"role": "user", "content": "Hello"},
|
|
],
|
|
extra_body=self._extra_body,
|
|
timeout=self.request_timeout,
|
|
)
|
|
end = time.time()
|
|
logger.info(f"{self.__class__.__name__}: warmed up! time: {(end - start):.3f} s")
|
|
|
|
def _build_compaction_generate_fn(self) -> CompactGenerateFn:
|
|
"""Return a generate fn that calls Chat Completions for compaction."""
|
|
client = self.client
|
|
model_name = self.model_name
|
|
timeout = self.request_timeout
|
|
extra_body = self._extra_body
|
|
|
|
def generate(system: str, user: str) -> str:
|
|
response = client.chat.completions.create(
|
|
model=model_name,
|
|
messages=[
|
|
{"role": "system", "content": system},
|
|
{"role": "user", "content": user},
|
|
],
|
|
extra_body=extra_body,
|
|
timeout=timeout,
|
|
)
|
|
return response.choices[0].message.content or ""
|
|
|
|
return generate
|
|
|
|
@staticmethod
|
|
def _to_chat_content_part(part: dict[str, Any]) -> ChatCompletionContentPartParam:
|
|
"""Convert one transformers content part to Chat-Completions shape.
|
|
|
|
``to_transformers_chat`` keeps Realtime-style parts (``input_text`` /
|
|
``input_image`` with a bare-string ``image_url``). The Chat Completions
|
|
HTTP API instead wants ``{type:"text", text}`` and
|
|
``{type:"image_url", image_url:{url, detail}}``. Unknown parts pass through.
|
|
"""
|
|
ptype = part.get("type")
|
|
if ptype == "input_text":
|
|
return ChatCompletionContentPartTextParam(type="text", text=part.get("text") or "")
|
|
if ptype == "input_image":
|
|
raw_url: Any = part.get("image_url")
|
|
if isinstance(raw_url, dict):
|
|
image_url = cast("ImageURL", raw_url)
|
|
else:
|
|
image_url = ImageURL(url=raw_url)
|
|
detail = part.get("detail")
|
|
if detail is not None:
|
|
image_url["detail"] = detail
|
|
return ChatCompletionContentPartImageParam(type="image_url", image_url=image_url)
|
|
return cast("ChatCompletionContentPartParam", part)
|
|
|
|
@classmethod
|
|
def _chat_messages(cls, chat: Chat) -> list[dict[str, Any]]:
|
|
"""Serialise the chat for the Chat Completions API.
|
|
|
|
``Chat.to_transformers_chat`` targets HuggingFace ``apply_chat_template``,
|
|
so two shapes need fixing up for the OpenAI Chat Completions HTTP API:
|
|
tool-call ``arguments`` must be a JSON *string* (not a parsed object), and
|
|
multimodal ``content`` parts must use the Chat Completions ``text`` /
|
|
``image_url`` shape rather than the Realtime ``input_text`` /
|
|
``input_image`` shape.
|
|
"""
|
|
messages = chat.to_transformers_chat()
|
|
for message in messages:
|
|
for tool_call in message.get("tool_calls") or []:
|
|
fn = tool_call.get("function")
|
|
if fn is not None and not isinstance(fn.get("arguments"), str):
|
|
fn["arguments"] = json.dumps(fn.get("arguments") or {}, ensure_ascii=False)
|
|
content = message.get("content")
|
|
if isinstance(content, list):
|
|
message["content"] = [cls._to_chat_content_part(p) for p in content]
|
|
if message.get("role") == "tool":
|
|
message.pop("name", None)
|
|
return messages
|
|
|
|
# ── base hooks ──────────────────────────────────────────────────────────--
|
|
|
|
def _serialize(self, active_chat: Chat) -> list[dict[str, Any]]:
|
|
return self._chat_messages(active_chat)
|
|
|
|
def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
|
|
optional_kwargs: dict[str, Any] = {}
|
|
chat_tools = _to_chat_tools(req_tools)
|
|
if chat_tools is not None:
|
|
optional_kwargs["tools"] = chat_tools
|
|
if req_tool_choice is not None:
|
|
optional_kwargs["tool_choice"] = _to_chat_tool_choice(req_tool_choice)
|
|
return optional_kwargs
|
|
|
|
|
|
def _request(self, api_input: list[dict[str, Any]], optional_kwargs: dict[str, Any]) -> Any:
|
|
create_kwargs: dict[str, Any] = dict(optional_kwargs)
|
|
if self.stream:
|
|
create_kwargs["stream_options"] = {"include_usage": True}
|
|
|
|
# Remove tool parameters to avoid NVIDIA NIM's HTTP 400 response.
|
|
create_kwargs.pop("tools", None)
|
|
create_kwargs.pop("tool_choice", None)
|
|
|
|
return self.client.chat.completions.create(
|
|
model=self.model_name,
|
|
messages=api_input, # type: ignore[arg-type] # runtime dicts match the Chat Completions message shape
|
|
stream=self.stream,
|
|
extra_body=self._extra_body,
|
|
timeout=self.request_timeout,
|
|
**create_kwargs,
|
|
)
|
|
|
|
# def _request(self, api_input: list[dict[str, Any]], optional_kwargs: dict[str, Any]) -> Any:
|
|
# create_kwargs: dict[str, Any] = dict(optional_kwargs)
|
|
# if self.stream:
|
|
# create_kwargs["stream_options"] = {"include_usage": True}
|
|
# return self.client.chat.completions.create(
|
|
# model=self.model_name,
|
|
# messages=api_input, # type: ignore[arg-type] # runtime dicts match the Chat Completions message shape
|
|
# stream=self.stream,
|
|
# extra_body=self._extra_body,
|
|
# timeout=self.request_timeout,
|
|
# **create_kwargs,
|
|
# )
|
|
|
|
def _iter_stream_events(self, api_response: Stream[ChatCompletionChunk]) -> Iterator[ProviderEvent]:
|
|
# Accumulate streamed tool-call deltas, keyed by their stream index, and the
|
|
# raw assistant text, then emit assistant message + tool calls + usage once
|
|
# the stream is exhausted.
|
|
tool_accum: dict[int, dict[str, str]] = {}
|
|
usage: Usage | None = None
|
|
raw_text = ""
|
|
for chunk in api_response:
|
|
# Usage-only trailing chunk (choices == []) when include_usage is set.
|
|
if chunk.usage is not None:
|
|
usage = Usage(
|
|
input_tokens=chunk.usage.prompt_tokens or 0, output_tokens=chunk.usage.completion_tokens or 0
|
|
)
|
|
if not chunk.choices:
|
|
continue
|
|
delta = chunk.choices[0].delta
|
|
if delta.tool_calls:
|
|
for tc in delta.tool_calls:
|
|
entry = tool_accum.setdefault(tc.index, {"name": "", "args": "", "id": ""})
|
|
if tc.id:
|
|
entry["id"] = tc.id
|
|
if tc.function is not None:
|
|
if tc.function.name:
|
|
entry["name"] = tc.function.name
|
|
if tc.function.arguments:
|
|
entry["args"] += tc.function.arguments
|
|
# A refusal streams as `delta.refusal` with `delta.content` None;
|
|
# surface it as assistant text so it is spoken and stored.
|
|
text_piece = delta.content or getattr(delta, "refusal", None)
|
|
if text_piece:
|
|
raw_text += text_piece
|
|
yield TextDelta(text=text_piece)
|
|
|
|
if raw_text.strip():
|
|
yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_text)])
|
|
yield from self._tool_calls_from_accum(tool_accum)
|
|
if usage is not None:
|
|
yield usage
|
|
|
|
def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
|
|
usage = api_response.usage
|
|
if usage:
|
|
yield Usage(input_tokens=usage.prompt_tokens or 0, output_tokens=usage.completion_tokens or 0)
|
|
# A valid-but-empty response (e.g. content filter) returns no choices;
|
|
# complete cleanly with no assistant text rather than raising IndexError.
|
|
message = api_response.choices[0].message if api_response.choices else None
|
|
if message is None:
|
|
return
|
|
# A refusal arrives as `message.refusal` with `message.content` None; treat
|
|
# it as assistant text so it is spoken and stored.
|
|
raw_content = message.content or getattr(message, "refusal", None)
|
|
if raw_content:
|
|
yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_content)])
|
|
yield TextDelta(text=raw_content)
|
|
tool_accum: dict[int, dict[str, str]] = {}
|
|
for tc in message.tool_calls or []:
|
|
tool_accum[len(tool_accum)] = {
|
|
"name": tc.function.name or "",
|
|
"args": tc.function.arguments or "",
|
|
"id": tc.id or "",
|
|
}
|
|
yield from self._tool_calls_from_accum(tool_accum)
|
|
|
|
@staticmethod
|
|
def _tool_calls_from_accum(tool_accum: dict[int, dict[str, str]]) -> Iterator[ToolCall]:
|
|
"""Turn accumulated tool-call deltas into ToolCall events.
|
|
|
|
IDs are regenerated (mirroring the Responses handler) so the rest of the
|
|
pipeline pairs each call_id with its function_call_output consistently.
|
|
"""
|
|
for index in sorted(tool_accum):
|
|
entry = tool_accum[index]
|
|
if not entry["name"]:
|
|
continue
|
|
yield ToolCall(
|
|
item=ResponseFunctionToolCall(
|
|
type="function_call",
|
|
name=entry["name"],
|
|
arguments=entry["args"] or "{}",
|
|
call_id=_generate_id("call"),
|
|
id=_generate_id("fc"),
|
|
status="completed",
|
|
)
|
|
)
|
|
|
|
def on_session_end(self) -> None:
|
|
logger.debug("Chat Completions API language model session state reset")
|