first
This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from typing import Any, cast
|
||||
|
||||
from openai import Stream
|
||||
from openai.types.chat import (
|
||||
ChatCompletionChunk,
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionToolChoiceOptionParam,
|
||||
ChatCompletionToolParam,
|
||||
)
|
||||
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
|
||||
from openai.types.chat.chat_completion_named_tool_choice_param import Function as NamedToolChoiceFunction
|
||||
from openai.types.realtime.realtime_conversation_item_assistant_message import (
|
||||
Content as AssistantContent,
|
||||
)
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
from openai.types.shared_params import FunctionDefinition
|
||||
|
||||
from speech_to_speech.LLM.base_openai_compatible_language_model import (
|
||||
WARMUP_MAX_RETRIES,
|
||||
AssistantMessage,
|
||||
BaseOpenAICompatibleHandler,
|
||||
ProviderEvent,
|
||||
TextDelta,
|
||||
ToolCall,
|
||||
Usage,
|
||||
)
|
||||
from speech_to_speech.LLM.chat import Chat
|
||||
from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn
|
||||
from speech_to_speech.utils.utils import _generate_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _to_chat_tools(req_tools: Any) -> list[ChatCompletionToolParam] | None:
|
||||
"""Convert Responses-API function tools to Chat-Completions tool format.
|
||||
|
||||
Responses tools are flat ``{type:"function", name, description, parameters}``;
|
||||
Chat Completions nests them under a ``function`` key. Items already in the
|
||||
nested form (or non-function tools) are passed through untouched.
|
||||
"""
|
||||
if not req_tools:
|
||||
return None
|
||||
chat_tools: list[ChatCompletionToolParam] = []
|
||||
for t in req_tools:
|
||||
d = t if isinstance(t, dict) else t.model_dump(exclude_none=True)
|
||||
if d.get("type") == "function" and "function" not in d:
|
||||
fn = FunctionDefinition(name=d["name"])
|
||||
if d.get("description") is not None:
|
||||
fn["description"] = d["description"]
|
||||
if d.get("parameters") is not None:
|
||||
fn["parameters"] = d["parameters"]
|
||||
chat_tools.append(ChatCompletionToolParam(type="function", function=fn))
|
||||
else:
|
||||
chat_tools.append(cast("ChatCompletionToolParam", d))
|
||||
return chat_tools
|
||||
|
||||
|
||||
def _to_chat_tool_choice(tool_choice: Any) -> ChatCompletionToolChoiceOptionParam:
|
||||
"""Convert a Responses-API tool_choice to Chat-Completions form.
|
||||
|
||||
The string forms ("auto"/"required"/"none") are identical across both APIs;
|
||||
only the forced-function object differs (flat ``name`` vs nested ``function``).
|
||||
"""
|
||||
if isinstance(tool_choice, dict) and tool_choice.get("type") == "function" and "name" in tool_choice:
|
||||
return ChatCompletionNamedToolChoiceParam(
|
||||
type="function", function=NamedToolChoiceFunction(name=tool_choice["name"])
|
||||
)
|
||||
if tool_choice is not None and not isinstance(tool_choice, (str, dict)):
|
||||
d = tool_choice.model_dump(exclude_none=True)
|
||||
if d.get("type") == "function" and "name" in d:
|
||||
return ChatCompletionNamedToolChoiceParam(type="function", function=NamedToolChoiceFunction(name=d["name"]))
|
||||
return cast("ChatCompletionToolChoiceOptionParam", d)
|
||||
return cast("ChatCompletionToolChoiceOptionParam", tool_choice)
|
||||
|
||||
|
||||
class ChatCompletionsApiModelHandler(BaseOpenAICompatibleHandler):
|
||||
"""LLM handler that talks to an OpenAI-compatible ``/v1/chat/completions`` server.
|
||||
|
||||
Functionally mirrors :class:`ResponsesApiModelHandler` but uses the mature
|
||||
Chat Completions streaming tool-call protocol (``choices[].delta.tool_calls``)
|
||||
instead of ``/v1/responses``. This is the robust path for vLLM + Qwen tool
|
||||
calling. The conversation is serialised with :meth:`Chat.to_transformers_chat`,
|
||||
which already emits OpenAI chat messages including ``tool_calls``/``tool`` roles.
|
||||
"""
|
||||
|
||||
def warmup(self) -> None:
|
||||
logger.info(f"Warming up {self.__class__.__name__}")
|
||||
start = time.time()
|
||||
self.client.with_options(max_retries=WARMUP_MAX_RETRIES).chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant"},
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
extra_body=self._extra_body,
|
||||
timeout=self.request_timeout,
|
||||
)
|
||||
end = time.time()
|
||||
logger.info(f"{self.__class__.__name__}: warmed up! time: {(end - start):.3f} s")
|
||||
|
||||
def _build_compaction_generate_fn(self) -> CompactGenerateFn:
|
||||
"""Return a generate fn that calls Chat Completions for compaction."""
|
||||
client = self.client
|
||||
model_name = self.model_name
|
||||
timeout = self.request_timeout
|
||||
extra_body = self._extra_body
|
||||
|
||||
def generate(system: str, user: str) -> str:
|
||||
response = client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
)
|
||||
return response.choices[0].message.content or ""
|
||||
|
||||
return generate
|
||||
|
||||
@staticmethod
|
||||
def _to_chat_content_part(part: dict[str, Any]) -> ChatCompletionContentPartParam:
|
||||
"""Convert one transformers content part to Chat-Completions shape.
|
||||
|
||||
``to_transformers_chat`` keeps Realtime-style parts (``input_text`` /
|
||||
``input_image`` with a bare-string ``image_url``). The Chat Completions
|
||||
HTTP API instead wants ``{type:"text", text}`` and
|
||||
``{type:"image_url", image_url:{url, detail}}``. Unknown parts pass through.
|
||||
"""
|
||||
ptype = part.get("type")
|
||||
if ptype == "input_text":
|
||||
return ChatCompletionContentPartTextParam(type="text", text=part.get("text") or "")
|
||||
if ptype == "input_image":
|
||||
raw_url: Any = part.get("image_url")
|
||||
if isinstance(raw_url, dict):
|
||||
image_url = cast("ImageURL", raw_url)
|
||||
else:
|
||||
image_url = ImageURL(url=raw_url)
|
||||
detail = part.get("detail")
|
||||
if detail is not None:
|
||||
image_url["detail"] = detail
|
||||
return ChatCompletionContentPartImageParam(type="image_url", image_url=image_url)
|
||||
return cast("ChatCompletionContentPartParam", part)
|
||||
|
||||
@classmethod
|
||||
def _chat_messages(cls, chat: Chat) -> list[dict[str, Any]]:
|
||||
"""Serialise the chat for the Chat Completions API.
|
||||
|
||||
``Chat.to_transformers_chat`` targets HuggingFace ``apply_chat_template``,
|
||||
so two shapes need fixing up for the OpenAI Chat Completions HTTP API:
|
||||
tool-call ``arguments`` must be a JSON *string* (not a parsed object), and
|
||||
multimodal ``content`` parts must use the Chat Completions ``text`` /
|
||||
``image_url`` shape rather than the Realtime ``input_text`` /
|
||||
``input_image`` shape.
|
||||
"""
|
||||
messages = chat.to_transformers_chat()
|
||||
for message in messages:
|
||||
for tool_call in message.get("tool_calls") or []:
|
||||
fn = tool_call.get("function")
|
||||
if fn is not None and not isinstance(fn.get("arguments"), str):
|
||||
fn["arguments"] = json.dumps(fn.get("arguments") or {}, ensure_ascii=False)
|
||||
content = message.get("content")
|
||||
if isinstance(content, list):
|
||||
message["content"] = [cls._to_chat_content_part(p) for p in content]
|
||||
if message.get("role") == "tool":
|
||||
message.pop("name", None)
|
||||
return messages
|
||||
|
||||
# ── base hooks ──────────────────────────────────────────────────────────--
|
||||
|
||||
def _serialize(self, active_chat: Chat) -> list[dict[str, Any]]:
|
||||
return self._chat_messages(active_chat)
|
||||
|
||||
def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
|
||||
optional_kwargs: dict[str, Any] = {}
|
||||
chat_tools = _to_chat_tools(req_tools)
|
||||
if chat_tools is not None:
|
||||
optional_kwargs["tools"] = chat_tools
|
||||
if req_tool_choice is not None:
|
||||
optional_kwargs["tool_choice"] = _to_chat_tool_choice(req_tool_choice)
|
||||
return optional_kwargs
|
||||
|
||||
|
||||
def _request(self, api_input: list[dict[str, Any]], optional_kwargs: dict[str, Any]) -> Any:
|
||||
create_kwargs: dict[str, Any] = dict(optional_kwargs)
|
||||
if self.stream:
|
||||
create_kwargs["stream_options"] = {"include_usage": True}
|
||||
|
||||
# Svuota i parametri dei tool per evitare il blocco 400 di NVIDIA NIM
|
||||
create_kwargs.pop("tools", None)
|
||||
create_kwargs.pop("tool_choice", None)
|
||||
|
||||
return self.client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=api_input, # type: ignore[arg-type] # runtime dicts match the Chat Completions message shape
|
||||
stream=self.stream,
|
||||
extra_body=self._extra_body,
|
||||
timeout=self.request_timeout,
|
||||
**create_kwargs,
|
||||
)
|
||||
|
||||
# def _request(self, api_input: list[dict[str, Any]], optional_kwargs: dict[str, Any]) -> Any:
|
||||
# create_kwargs: dict[str, Any] = dict(optional_kwargs)
|
||||
# if self.stream:
|
||||
# create_kwargs["stream_options"] = {"include_usage": True}
|
||||
# return self.client.chat.completions.create(
|
||||
# model=self.model_name,
|
||||
# messages=api_input, # type: ignore[arg-type] # runtime dicts match the Chat Completions message shape
|
||||
# stream=self.stream,
|
||||
# extra_body=self._extra_body,
|
||||
# timeout=self.request_timeout,
|
||||
# **create_kwargs,
|
||||
# )
|
||||
|
||||
def _iter_stream_events(self, api_response: Stream[ChatCompletionChunk]) -> Iterator[ProviderEvent]:
|
||||
# Accumulate streamed tool-call deltas, keyed by their stream index, and the
|
||||
# raw assistant text, then emit assistant message + tool calls + usage once
|
||||
# the stream is exhausted.
|
||||
tool_accum: dict[int, dict[str, str]] = {}
|
||||
usage: Usage | None = None
|
||||
raw_text = ""
|
||||
for chunk in api_response:
|
||||
# Usage-only trailing chunk (choices == []) when include_usage is set.
|
||||
if chunk.usage is not None:
|
||||
usage = Usage(
|
||||
input_tokens=chunk.usage.prompt_tokens or 0, output_tokens=chunk.usage.completion_tokens or 0
|
||||
)
|
||||
if not chunk.choices:
|
||||
continue
|
||||
delta = chunk.choices[0].delta
|
||||
if delta.tool_calls:
|
||||
for tc in delta.tool_calls:
|
||||
entry = tool_accum.setdefault(tc.index, {"name": "", "args": "", "id": ""})
|
||||
if tc.id:
|
||||
entry["id"] = tc.id
|
||||
if tc.function is not None:
|
||||
if tc.function.name:
|
||||
entry["name"] = tc.function.name
|
||||
if tc.function.arguments:
|
||||
entry["args"] += tc.function.arguments
|
||||
# A refusal streams as `delta.refusal` with `delta.content` None;
|
||||
# surface it as assistant text so it is spoken and stored.
|
||||
text_piece = delta.content or getattr(delta, "refusal", None)
|
||||
if text_piece:
|
||||
raw_text += text_piece
|
||||
yield TextDelta(text=text_piece)
|
||||
|
||||
if raw_text.strip():
|
||||
yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_text)])
|
||||
yield from self._tool_calls_from_accum(tool_accum)
|
||||
if usage is not None:
|
||||
yield usage
|
||||
|
||||
def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
|
||||
usage = api_response.usage
|
||||
if usage:
|
||||
yield Usage(input_tokens=usage.prompt_tokens or 0, output_tokens=usage.completion_tokens or 0)
|
||||
# A valid-but-empty response (e.g. content filter) returns no choices;
|
||||
# complete cleanly with no assistant text rather than raising IndexError.
|
||||
message = api_response.choices[0].message if api_response.choices else None
|
||||
if message is None:
|
||||
return
|
||||
# A refusal arrives as `message.refusal` with `message.content` None; treat
|
||||
# it as assistant text so it is spoken and stored.
|
||||
raw_content = message.content or getattr(message, "refusal", None)
|
||||
if raw_content:
|
||||
yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_content)])
|
||||
yield TextDelta(text=raw_content)
|
||||
tool_accum: dict[int, dict[str, str]] = {}
|
||||
for tc in message.tool_calls or []:
|
||||
tool_accum[len(tool_accum)] = {
|
||||
"name": tc.function.name or "",
|
||||
"args": tc.function.arguments or "",
|
||||
"id": tc.id or "",
|
||||
}
|
||||
yield from self._tool_calls_from_accum(tool_accum)
|
||||
|
||||
@staticmethod
|
||||
def _tool_calls_from_accum(tool_accum: dict[int, dict[str, str]]) -> Iterator[ToolCall]:
|
||||
"""Turn accumulated tool-call deltas into ToolCall events.
|
||||
|
||||
IDs are regenerated (mirroring the Responses handler) so the rest of the
|
||||
pipeline pairs each call_id with its function_call_output consistently.
|
||||
"""
|
||||
for index in sorted(tool_accum):
|
||||
entry = tool_accum[index]
|
||||
if not entry["name"]:
|
||||
continue
|
||||
yield ToolCall(
|
||||
item=ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=entry["name"],
|
||||
arguments=entry["args"] or "{}",
|
||||
call_id=_generate_id("call"),
|
||||
id=_generate_id("fc"),
|
||||
status="completed",
|
||||
)
|
||||
)
|
||||
|
||||
def on_session_end(self) -> None:
|
||||
logger.debug("Chat Completions API language model session state reset")
|
||||
Reference in New Issue
Block a user