valenti/vocero-s2s
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

first git
Some checks are pending
CI / ruff (push) Waiting to run
Details
CI / mypy (push) Waiting to run
Details
CI / pytest (push) Waiting to run
Details
CI / package (push) Waiting to run
Details
CI / Install smoke (${{ matrix.label }}) (linux, ubuntu-latest) (push) Blocked by required conditions
Details
CI / Install smoke (${{ matrix.label }}) (macos-arm64, macos-14) (push) Blocked by required conditions
Details

Browse Source
This commit is contained in:
valenti 2026-08-26 11:30:14 +00:00
commit b5f82fb48c
207 changed files with 42776 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
.DS_Store vendored Normal file
View File

Binary file not shown.

17
.dockerignore Normal file
View File

@ -0,0 +1,17 @@
tmp
cache
.venv
.git
.mypy_cache
.pytest_cache
.ruff_cache
__pycache__
**/__pycache__
*.py[cod]
*.egg-info
dist
build
Dockerfile
docker-compose.yml
.dockerignore
.gitignore

11
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,11 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
cooldown:
default-days: 7
groups:
actions:
patterns: ["*"]

194
.github/scripts/star_history.py vendored Normal file
View File

@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""Fetch stargazer history for a repo and render it as a static SVG.
Runs in CI with the automatic GITHUB_TOKEN. No third-party dependencies.
Usage:
GITHUB_TOKEN=... python star_history.py owner/repo output.svg
"""
import json
import math
import os
import sys
import urllib.request
from datetime import datetime, timezone
API = "https://api.github.com/graphql"
PER_PAGE = 100
MAX_PAGES = 400 # preserve the REST implementation's 40k-star cap
MAX_POINTS = 120 # downsample the curve to at most this many points
def gh_post(query: str, variables: dict, token: str) -> dict:
payload = json.dumps({"query": query, "variables": variables}).encode()
req = urllib.request.Request(API, data=payload)
req.add_header("Accept", "application/vnd.github+json")
req.add_header("Content-Type", "application/json")
req.add_header("X-GitHub-Api-Version", "2022-11-28")
if token:
req.add_header("Authorization", f"Bearer {token}")
with urllib.request.urlopen(req, timeout=30) as resp:
result = json.loads(resp.read())
if result.get("errors"):
raise RuntimeError(json.dumps(result["errors"]))
return result
def fetch_star_dates(repo: str, token: str) -> list[datetime]:
owner, name = repo.split("/", 1)
query = """
query($owner: String!, $name: String!, $cursor: String, $perPage: Int!) {
repository(owner: $owner, name: $name) {
stargazers(first: $perPage, after: $cursor) {
edges { starredAt }
pageInfo { hasNextPage endCursor }
}
}
}
"""
dates = []
page = 1
cursor = None
while True:
result = gh_post(
query,
{
"owner": owner,
"name": name,
"cursor": cursor,
"perPage": PER_PAGE,
},
token,
)
stargazers = result["data"]["repository"]["stargazers"]
for edge in stargazers["edges"]:
ts = edge.get("starredAt")
if ts:
dates.append(
datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace(
tzinfo=timezone.utc
)
)
page_info = stargazers["pageInfo"]
if not page_info["hasNextPage"]:
break
page += 1
if page > MAX_PAGES:
break
cursor = page_info["endCursor"]
dates.sort()
return dates
def downsample(dates: list[datetime]) -> list[tuple[datetime, int]]:
n = len(dates)
points = [(dates[0], 1)]
if n > 1:
step = max(1, n // MAX_POINTS)
for i in range(step, n, step):
points.append((dates[i], i + 1))
points.append((dates[-1], n))
now = datetime.now(timezone.utc)
points.append((now, n))
return points
def nice_ceil(v: float) -> int:
if v <= 0:
return 1
mag = 10 ** int(math.floor(math.log10(v)))
for mult in (1, 1.2, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10):
if v <= mag * mult:
return int(mag * mult)
return int(mag * 10)
def render_svg(points: list[tuple[datetime, int]], repo: str) -> str:
width, height = 800, 420
ml, mr, mt, mb = 70, 30, 50, 60
pw, ph = width - ml - mr, height - mt - mb
t0 = points[0][0].timestamp()
t1 = points[-1][0].timestamp()
tspan = max(t1 - t0, 1)
ymax = nice_ceil(points[-1][1] * 1.08)
def x(ts: float) -> float:
return ml + (ts - t0) / tspan * pw
def y(v: float) -> float:
return mt + ph - v / ymax * ph
line = " ".join(
f"{'M' if i == 0 else 'L'}{x(d.timestamp()):.1f},{y(c):.1f}"
for i, (d, c) in enumerate(points)
)
area = (
line
+ f" L{x(points[-1][0].timestamp()):.1f},{y(0):.1f}"
+ f" L{x(points[0][0].timestamp()):.1f},{y(0):.1f} Z"
)
grid, ylabels = [], []
for i in range(6):
v = ymax * i / 5
yy = y(v)
grid.append(
f'<line x1="{ml}" y1="{yy:.1f}" x2="{ml + pw}" y2="{yy:.1f}" '
f'stroke="#8b949e" stroke-opacity="0.25" stroke-width="1"/>'
)
label = f"{v / 1000:.1f}k".replace(".0k", "k") if v >= 1000 else f"{int(v)}"
ylabels.append(
f'<text x="{ml - 10}" y="{yy + 4:.1f}" text-anchor="end" '
f'class="lbl">{label}</text>'
)
xlabels = []
for i in range(6):
ts = t0 + tspan * i / 5
d = datetime.fromtimestamp(ts, tz=timezone.utc)
xlabels.append(
f'<text x="{x(ts):.1f}" y="{mt + ph + 22}" text-anchor="middle" '
f'class="lbl">{d.strftime("%b %Y")}</text>'
)
total = points[-1][1]
return f"""<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {width} {height}" font-family="-apple-system,'Segoe UI',Helvetica,Arial,sans-serif">
<style>
.lbl {{ font-size: 12px; fill: #8b949e; }}
.title {{ font-size: 16px; font-weight: 600; fill: #8b949e; }}
.total {{ font-size: 13px; fill: #8b949e; }}
</style>
<text x="{ml}" y="28" class="title">{repo} star history</text>
<text x="{ml + pw}" y="28" text-anchor="end" class="total">{total:,} stars</text>
{"".join(grid)}
{"".join(ylabels)}
{"".join(xlabels)}
<path d="{area}" fill="#f4b400" fill-opacity="0.12"/>
<path d="{line}" fill="none" stroke="#f4b400" stroke-width="2.5" stroke-linejoin="round"/>
<circle cx="{x(points[-1][0].timestamp()):.1f}" cy="{y(total):.1f}" r="4" fill="#f4b400"/>
</svg>
"""
def main() -> int:
if len(sys.argv) != 3:
print(__doc__)
return 2
repo, out = sys.argv[1], sys.argv[2]
token = os.environ.get("GITHUB_TOKEN", "")
dates = fetch_star_dates(repo, token)
if not dates:
print("no stargazers found", file=sys.stderr)
return 1
os.makedirs(os.path.dirname(out) or ".", exist_ok=True)
svg = render_svg(downsample(dates), repo)
with open(out, "w") as f:
f.write(svg)
print(f"wrote {out} ({len(dates):,} stars)")
return 0
if __name__ == "__main__":
sys.exit(main())

98
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,98 @@
name: CI
on:
push:
branches: [main]
pull_request:
jobs:
ruff:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.0
- uses: astral-sh/setup-uv@v8.2.0
with:
version: "latest"
- run: uv sync --group dev
- name: Ruff check
run: uv run ruff check src/ tests/
- name: Ruff format
run: uv run ruff format --check src/ tests/
mypy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.0
- uses: astral-sh/setup-uv@v8.2.0
with:
version: "latest"
- run: uv sync --group dev
- name: Mypy
run: uv run mypy src/
pytest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.0
- uses: astral-sh/setup-uv@v8.2.0
with:
version: "latest"
- run: uv sync --group dev
- name: NLTK data (punkt_tab for sent_tokenize)
run: uv run python -c "import nltk; nltk.download('punkt_tab')"
- name: Pytest
run: uv run pytest tests/ -x -q
package:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.0
- uses: astral-sh/setup-uv@v8.2.0
with:
version: "latest"
- name: Build distributions
run: uv build
- name: Check package metadata
run: uvx twine check --strict dist/*
- uses: actions/upload-artifact@v7
with:
name: dist
path: dist/*
install-smoke:
name: Install smoke (${{ matrix.label }})
needs: package
runs-on: ${{ matrix.os }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- label: linux
os: ubuntu-latest
- label: macos-arm64
os: macos-14
steps:
- uses: actions/checkout@v7.0.0
- uses: actions/setup-python@v6.3.0
with:
python-version: "3.11"
- uses: actions/download-artifact@v8
with:
name: dist
path: dist
- name: Install wheel with pip
shell: bash
run: |
python -m venv .venv-smoke
source .venv-smoke/bin/activate
python -m pip install --upgrade pip
python -m pip install dist/speech_to_speech-*.whl
python -m pip check
- name: Run installed CLI smoke test
shell: bash
run: |
source .venv-smoke/bin/activate
unset OPENAI_API_KEY
# Validate the installed entry point without loading models or calling OpenAI.
python tests/install_smoke.py

48
.github/workflows/publish.yml vendored Normal file
View File

@ -0,0 +1,48 @@
name: Publish
on:
push:
tags:
- "v*"
workflow_dispatch:
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
with:
version: "latest"
enable-cache: false
- name: Build distributions
run: uv build
- name: Check package metadata
run: uvx twine check --strict dist/*
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dist
path: dist/*
publish:
needs: build
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/project/speech-to-speech/
permissions:
contents: read
id-token: write
steps:
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: dist
path: dist
- uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1

38
.github/workflows/star-history.yml vendored Normal file
View File

@ -0,0 +1,38 @@
name: Update star history chart
on:
workflow_dispatch:
permissions:
contents: write
jobs:
update-chart:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.repository.default_branch }}
- name: Generate chart
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: python3 .github/scripts/star_history.py "${{ github.repository }}" assets/star-history.svg
- name: Commit to update branch if changed
env:
BASE_BRANCH: ${{ github.event.repository.default_branch }}
UPDATE_BRANCH: star-history-update-${{ github.run_id }}-${{ github.run_attempt }}
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add assets/star-history.svg
if ! git diff --cached --quiet; then
git commit -m "chore: update star history chart [skip ci]"
git push origin "HEAD:refs/heads/${UPDATE_BRANCH}"
pr_url="https://github.com/${{ github.repository }}/compare/${BASE_BRANCH}...${UPDATE_BRANCH}?expand=1"
echo "Open a PR: ${pr_url}"
echo "Open a PR: [${UPDATE_BRANCH}](${pr_url})" >> "$GITHUB_STEP_SUMMARY"
else
echo "No changes"
fi

231
.gitignore vendored Normal file
View File

@ -0,0 +1,231 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[codz]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py.cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
# Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
# poetry.lock
# poetry.toml
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
# pdm.lock
# pdm.toml
.pdm-python
.pdm-build/
# pixi
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
# pixi.lock
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
# in the .venv directory. It is recommended not to include this directory in version control.
.pixi
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# Redis
*.rdb
*.aof
*.pid
# RabbitMQ
mnesia/
rabbitmq/
rabbitmq-data/
# ActiveMQ
activemq-data/
# SageMath parsed files
*.sage.py
# Environments
.env
.envrc
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
# .idea/
# Abstra
# Abstra is an AI-powered process automation framework.
# Ignore directories containing user credentials, local state, and settings.
# Learn more at https://abstra.io/docs
.abstra/
# Visual Studio Code
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
# and can be added to the global gitignore or merged into this file. However, if you prefer,
# you could uncomment the following to ignore the entire vscode folder
# .vscode/
# Temporary file for partial code execution
tempCodeRunnerFile.py
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
# Marimo
marimo/_static/
marimo/_lsp/
__marimo__/
# Streamlit
.streamlit/secrets.toml
.mypy_cache/
.pytype/
.pyre/
.dmypy.json
dmypy.json
.pytest_cache/
.hypothesis/
.coverage*
.coverage
cover/
.tox/
uv.lock

26
AGENTS.md Normal file
View File

@ -0,0 +1,26 @@
# Repository Instructions
- Never include `codex` in branch names or pull request titles.
- Keep release pull requests focused on version metadata and release documentation.
- Do not commit local build artifacts such as `dist/`, `build/`, or generated wheel/sdist files.
## Publishing to PyPI
PyPI publishing is handled by GitHub Actions in `.github/workflows/publish.yml`. The workflow runs on pushed tags that match `v*`, builds the package with `uv build`, checks the artifacts with `twine check --strict`, and publishes through the configured `pypi` environment.
To prepare a release:
1. Confirm the intended version is not already published on PyPI.
2. Bump `version` in `pyproject.toml`.
3. Bump `__version__` in `src/speech_to_speech/__init__.py`.
4. Open and merge a pull request with only the release preparation changes.
To publish after the release PR is merged:
1. Update `main` locally: `git checkout main && git pull origin main`.
2. Create an annotated tag for the version: `git tag -a vX.Y.Z -m "Release vX.Y.Z"`.
3. Push the tag: `git push origin vX.Y.Z`.
4. Watch the `Publish` GitHub Actions workflow complete successfully.
5. Verify the new version appears at `https://pypi.org/project/speech-to-speech/`.
Only upload manually if the GitHub Actions workflow is unavailable and the maintainers have explicitly chosen that fallback.

27
Dockerfile Normal file
View File

@ -0,0 +1,27 @@
FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04
ENV DEBIAN_FRONTEND=noninteractive
ENV PYTHONUNBUFFERED=1
ENV PATH="/usr/src/app/.venv/bin:${PATH}"
WORKDIR /usr/src/app
# Install packages
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
git \
libportaudio2 \
libsndfile1 \
python3 \
python3-pip \
python3-venv \
&& rm -rf /var/lib/apt/lists/*
RUN python3 -m pip install --no-cache-dir --break-system-packages uv
COPY pyproject.toml README.md LICENSE MANIFEST.in ./
RUN uv sync --python /usr/bin/python3 --no-install-project --no-dev
COPY . .
RUN uv sync --python /usr/bin/python3 --no-dev
RUN python -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"

28
Dockerfile.arm64 Normal file
View File

@ -0,0 +1,28 @@
ARG BASE_PLATFORM=linux/arm64
FROM --platform=$BASE_PLATFORM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04
ENV DEBIAN_FRONTEND=noninteractive
ENV PYTHONUNBUFFERED=1
ENV PATH="/usr/src/app/.venv/bin:${PATH}"
WORKDIR /usr/src/app
# Install packages
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
git \
libportaudio2 \
libsndfile1 \
python3 \
python3-pip \
python3-venv \
&& rm -rf /var/lib/apt/lists/*
RUN python3 -m pip install --no-cache-dir --break-system-packages uv
COPY pyproject.toml README.md LICENSE MANIFEST.in ./
RUN uv sync --python /usr/bin/python3 --no-install-project --no-dev
COPY . .
RUN uv sync --python /usr/bin/python3 --no-dev
RUN python -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"

201
LICENSE Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [2024] [The HuggingFace Inc. team]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

4
MANIFEST.in Normal file
View File

@ -0,0 +1,4 @@
prune tests
global-exclude __pycache__
global-exclude *.py[cod]
global-exclude .DS_Store

606
README.md Normal file
View File

@ -0,0 +1,606 @@
<div align="center">
<div>&nbsp;</div>
<img src="https://raw.githubusercontent.com/huggingface/speech-to-speech/main/logo.png" width="600"/>
# Speech To Speech: Build voice agents with open-source models
[![PyPI](https://img.shields.io/pypi/v/speech-to-speech)](https://pypi.org/project/speech-to-speech/)
[![Python](https://img.shields.io/pypi/pyversions/speech-to-speech)](https://pypi.org/project/speech-to-speech/)
[![License](https://img.shields.io/badge/license-Apache%202.0-blue)](./LICENSE)
</div>
A low-latency, fully modular voice-agent pipeline: **VAD -> STT -> LLM -> TTS**, exposed through an **OpenAI Realtime-compatible WebSocket API**. Every component is swappable. The LLM slot speaks OpenAI-compatible protocols, so you can point it at a hosted provider, at [HF Inference Providers](https://huggingface.co/inference-providers), or at a vLLM or llama.cpp server on your own hardware for a fully local, fully open stack.
This pipeline runs in production as the conversation backend for thousands of [Reachy Mini](https://huggingface.co/blog/reachy-mini) robots.
<p align="center">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="./docs/assets/endpoint-swap-dark.gif">
<source media="(prefers-color-scheme: light)" srcset="./docs/assets/endpoint-swap-light.gif">
<img src="./docs/assets/endpoint-swap-light.gif" alt="Switching an OpenAI Realtime client endpoint from hosted OpenAI to a self-hosted speech-to-speech server" width="640">
</picture>
</p>
## Quickstart
```bash
pip install speech-to-speech
export OPENAI_API_KEY=...
speech-to-speech
```
This starts an OpenAI Realtime-compatible server at `ws://localhost:8765/v1/realtime` using Parakeet TDT for local STT, an OpenAI-compatible LLM, and Qwen3-TTS for local speech output.
From a source checkout, talk to it from a second terminal:
```bash
python scripts/listen_and_play_realtime.py --host 127.0.0.1 --port 8765
```
Prefer to keep the LLM on your own machine? Serve Gemma 4 with llama.cpp:
```bash
llama-server -hf ggml-org/gemma-4-E4B-it-GGUF -np 2 -c 65536 -fa on --swa-full
```
Then point the OpenAI-compatible LLM backend at it:
```bash
speech-to-speech \
--model_name "ggml-org/gemma-4-E4B-it-GGUF" \
--responses_api_base_url "http://127.0.0.1:8080/v1" \
--responses_api_api_key ""
```
Any OpenAI Realtime-compatible client can connect. See [Realtime API](#realtime-api) for the protocol and [LLM backends](#llm-backends) for provider and local-server options.
## Index
* [How it works](#how-it-works)
* [Installation](#installation)
* [Supported components](#supported-components)
* [Run modes](#run-modes)
* [Realtime API](#realtime-api)
* [LLM backends](#llm-backends)
* [Multi-language support](#multi-language-support)
* [Pocket TTS](#pocket-tts)
* [CLI reference](#cli-reference)
* [Contributing](#contributing)
* [Star history](#star-history)
* [Citations](#citations)
## How it works
The pipeline is a cascade of four components, each running in its own thread and connected by queues:
1. **Voice Activity Detection (VAD)**: [Silero VAD v5](https://github.com/snakers4/silero-vad) detects speech boundaries and turn-taking.
2. **Speech to Text (STT)**: transcribes the user's turn, with optional live partial transcripts.
3. **Language Model (LLM)**: generates the response, streaming text and tool calls.
4. **Text to Speech (TTS)**: synthesizes audio and streams it back to the client.
Every stage has multiple interchangeable backends, selected via CLI flags. The code is designed for easy modification, with a focus on models available through Transformers and the Hugging Face Hub.
## Installation
Requires Python 3.10+.
```bash
pip install speech-to-speech
```
The default install covers the standard realtime path:
- Parakeet TDT for STT
- OpenAI-compatible API for the language model
- Qwen3-TTS for speech output, using the GGML backend by default on non-macOS platforms and `mlx-audio` on Apple Silicon
- local audio and realtime server modes
macOS and non-macOS dependencies are resolved automatically via platform markers in `pyproject.toml`.
### CUDA Note for Qwen3-TTS
On Linux, the Qwen3-TTS GGML backend comes from `faster-qwen3-tts[ggml]`. Its default `qwentts-cpp-python` wheel on PyPI targets CUDA 12.8. If your machine does not have the CUDA 12 runtime that wheel expects, install the matching wheel from the Hugging Face wheelhouse before installing `speech-to-speech`:
```bash
# CUDA 13.x
pip install "qwentts-cpp-python==0.3.1+cu130" \
-f https://huggingface.co/datasets/andito/qwentts-cpp-python-wheels/tree/main/whl/cu130
# CUDA 12.4
pip install "qwentts-cpp-python==0.3.1+cu124" \
-f https://huggingface.co/datasets/andito/qwentts-cpp-python-wheels/tree/main/whl/cu124
# CPU-only fallback
pip install "qwentts-cpp-python==0.3.1+cpu" \
-f https://huggingface.co/datasets/andito/qwentts-cpp-python-wheels/tree/main/whl/cpu
pip install speech-to-speech
```
To use the previous CUDA-graphs implementation instead of GGML, pass `--qwen3_tts_backend torch`.
### Optional Backends
Extra backends are installed with pip extras:
```bash
pip install "speech-to-speech[kokoro]" # Kokoro-82M TTS on non-macOS
pip install "speech-to-speech[pocket]" # Pocket TTS
pip install "speech-to-speech[chattts]" # ChatTTS
pip install "speech-to-speech[facebook-mms]" # MMS TTS
pip install "speech-to-speech[faster-whisper]" # Faster Whisper STT
pip install "speech-to-speech[whisper-mlx]" # Lightning Whisper MLX STT on macOS
pip install "speech-to-speech[paraformer]" # Paraformer STT through FunASR
pip install "speech-to-speech[mlx-lm]" # mlx-vlm support for vision models on macOS
```
Deprecated implementations, including MeloTTS, live in [`archive/`](./archive) and are no longer wired into the CLI.
**Note on DeepFilterNet:** DeepFilterNet, used for optional audio enhancement in VAD, requires `numpy<2` and conflicts with Pocket TTS, which requires `numpy>=2`. Install it manually only in environments where you are not using Pocket TTS.
### From Source
```bash
git clone https://github.com/huggingface/speech-to-speech.git
cd speech-to-speech
uv sync
```
This installs the package in editable mode and makes the `speech-to-speech` CLI available.
## Supported Components
| Component | Backend | Platforms | Install |
|---|---|---|---|
| VAD | [Silero VAD v5](https://github.com/snakers4/silero-vad) | all | built-in |
| STT | [Parakeet TDT](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) (default) | CUDA / CPU through nano-parakeet, Apple Silicon through MLX | built-in |
| STT | [Whisper](https://huggingface.co/docs/transformers/en/model_doc/whisper) through Transformers | CUDA / CPU | built-in |
| STT | [Faster Whisper](https://github.com/SYSTRAN/faster-whisper) | CUDA / CPU | `faster-whisper` |
| STT | [Lightning Whisper MLX](https://github.com/mustafaaljadery/lightning-whisper-mlx) | Apple Silicon | `whisper-mlx` |
| STT | [MLX Audio Whisper](https://github.com/huggingface/mlx-audio) | Apple Silicon | built-in on macOS |
| STT | [Paraformer](https://github.com/modelscope/FunASR) | CUDA / CPU | `paraformer` |
| LLM | OpenAI-compatible API (`responses-api`, `chat-completions`) | hosted providers or self-hosted servers | built-in |
| LLM | [Transformers](https://huggingface.co/models?pipeline_tag=text-generation&sort=trending) | CUDA / CPU | built-in |
| LLM | [mlx-lm](https://github.com/ml-explore/mlx-lm) | Apple Silicon | built-in on macOS |
| TTS | [Qwen3-TTS](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) (default) | GGML / CUDA on Linux, mlx-audio on macOS | built-in |
| TTS | [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M) | CUDA / CPU, Apple Silicon | `kokoro` on non-macOS; built-in on macOS |
| TTS | [Pocket TTS](https://github.com/kyutai-labs/pocket-tts) | CPU / CUDA | `pocket` |
| TTS | [ChatTTS](https://github.com/2noise/ChatTTS) | CUDA / CPU | `chattts` |
| TTS | [MMS TTS](https://huggingface.co/docs/transformers/model_doc/mms) | CUDA / CPU | `facebook-mms` |
Select implementations with `--stt`, `--llm_backend`, and `--tts`. Run `speech-to-speech -h` for exact values and backend-specific flags.
## Run Modes
| Mode | Transport | Use it when |
|---|---|---|
| `realtime` (default) | WebSocket, OpenAI Realtime protocol at `/v1/realtime` | You are building an app or device against a standard voice API. |
| `local` | Your machine's microphone and speakers | You want to talk to the pipeline directly, no client needed. |
| `websocket` | Raw PCM over WebSocket | You want a minimal custom client without the Realtime protocol. |
| `socket` | Raw PCM over TCP | Models run on a remote server, with a simple microphone/playback client. |
### Realtime Server
```bash
export OPENAI_API_KEY=...
speech-to-speech
```
This is equivalent to:
```bash
speech-to-speech \
--thresh 0.6 \
--stt parakeet-tdt \
--llm_backend responses-api \
--tts qwen3 \
--qwen3_tts_model_name Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
--qwen3_tts_speaker Aiden \
--qwen3_tts_language auto \
--qwen3_tts_backend ggml \
--qwen3_tts_non_streaming_mode True \
--qwen3_tts_mlx_quantization 6bit \
--model_name gpt-5.4-mini \
--chat_size 30 \
--responses_api_stream \
--enable_live_transcription \
--mode realtime
```
The default model is `gpt-5.4-mini` through the OpenAI Responses API. Override it with `--model_name`, and set `--responses_api_base_url` for another OpenAI-compatible provider or server.
### Local Mac
```bash
speech-to-speech --local_mac_optimal_settings
```
Optionally with a specific LLM:
```bash
speech-to-speech \
--local_mac_optimal_settings \
--model_name mlx-community/Qwen3-4B-Instruct-2507-bf16
```
This setting:
- Adds `--device mps` to use MPS for all models.
- Sets Parakeet TDT for STT.
- Sets MLX LM as the LLM backend.
- Sets Qwen3-TTS for TTS, using `mlx-audio` with the `6bit` MLX variant by default.
- Sets `--mode local`.
`--tts pocket` and `--tts kokoro` are also valid on macOS.
To compare the MLX quantization variants locally:
```bash
python scripts/benchmark_tts.py \
--handlers qwen3 \
--iterations 3 \
--qwen3_mlx_quantizations bf16 4bit 6bit 8bit
```
### WebSocket
1. Run the pipeline in WebSocket mode:
```bash
speech-to-speech --mode websocket --ws_host 0.0.0.0 --ws_port 8765
```
2. Connect from your client at `ws://<server-ip>:8765`. Send raw audio bytes as 16 kHz, int16, mono PCM and receive generated audio bytes back.
### TCP Socket
TCP socket mode is intentionally minimal. It streams raw PCM audio, but does not provide the full Realtime API feature set, including interruption handling, live transcript events, or tool-call events.
1. Run the pipeline on the server:
```bash
speech-to-speech --mode socket --recv_host 0.0.0.0 --send_host 0.0.0.0
```
2. Run the client locally to handle microphone input and playback:
```bash
python scripts/listen_and_play.py --host <IP address of your server>
```
### Docker
Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html), then:
```bash
docker compose up
```
The compose file starts a llama.cpp server with Gemma 4, starts the TCP socket server, and exposes ports `8080`, `12345`, and `12346`.
## Realtime API
Realtime mode streams audio over a WebSocket using the OpenAI Realtime protocol, with live transcription and low-latency turn-taking. The server exposes `/v1/realtime`, and any OpenAI Realtime-compatible client can connect:
```python
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:8765/v1",
websocket_base_url="ws://localhost:8765/v1",
api_key="not-needed",
)
with client.realtime.connect(model="local") as conn:
conn.send(
{
"type": "session.update",
"session": {
"type": "realtime",
"instructions": "You are a helpful assistant.",
"audio": {
"input": {
"turn_detection": {
"type": "server_vad",
"interrupt_response": True,
}
}
},
},
}
)
for event in conn:
print(event.type)
```
The server implements the core Realtime event set: `input_audio_buffer.append`, `session.update`, `conversation.item.create`, `response.create`, and `response.cancel` inbound; speech start/stop, streaming transcription, audio deltas, tool calls, and `response.done` outbound. The full event reference, architecture, and design details live in the [Realtime Engine README](./src/speech_to_speech/api/openai_realtime/README.md).
## LLM Backends
The LLM is the most compute-intensive and highest-latency component in the pipeline. A single forward pass through a large model can dominate end-to-end response time, so choosing the right backend for your hardware and latency budget matters. The pipeline supports:
- **Local inference**: `transformers` on CUDA / CPU and `mlx-lm` on Apple Silicon.
- **Self-hosted servers**: `responses-api` and `chat-completions` can point at a local [vLLM](https://github.com/vllm-project/vllm) or [llama.cpp](https://github.com/ggerganov/llama.cpp) server.
- **Provider APIs**: the same backends work with OpenAI, [HF Inference Providers](https://huggingface.co/inference-providers), [OpenRouter](https://openrouter.ai), and other OpenAI-compatible providers.
Two API backends are available, sharing the same `--responses_api_*` connection flags:
- `--llm_backend responses-api` (default) targets `/v1/responses`.
- `--llm_backend chat-completions` targets `/v1/chat/completions`.
The examples below pair Parakeet TDT for local STT and Qwen3-TTS for local TTS with different LLM backends.
### Responses API Backend
Works with any provider or server that implements the OpenAI Responses API. Point `--responses_api_base_url` at the endpoint and set `--model_name` accordingly:
| Provider / server | `--responses_api_base_url` | `--responses_api_api_key` |
|---|---|---|
| OpenAI | omit, uses OpenAI default | `$OPENAI_API_KEY` |
| HF Inference Providers | `https://router.huggingface.co/v1` | `$HF_TOKEN` |
| OpenRouter | `https://openrouter.ai/api/v1` | `$OPENROUTER_API_KEY` |
| vLLM | `http://localhost:8000/v1` | omit or any string |
| llama.cpp | `http://127.0.0.1:8080/v1` | empty string |
```bash
# OpenAI
speech-to-speech \
--mode local \
--stt parakeet-tdt \
--llm_backend responses-api \
--tts qwen3 \
--qwen3_tts_mlx_quantization 6bit \
--model_name "gpt-4o-mini" \
--responses_api_api_key "$OPENAI_API_KEY" \
--responses_api_stream \
--enable_live_transcription
```
```bash
# HF Inference Providers: Qwen3.5-9B via Together
speech-to-speech \
--mode local \
--stt parakeet-tdt \
--llm_backend responses-api \
--tts qwen3 \
--qwen3_tts_mlx_quantization 6bit \
--model_name "Qwen/Qwen3.5-9B:together" \
--responses_api_base_url "https://router.huggingface.co/v1" \
--responses_api_api_key "$HF_TOKEN" \
--responses_api_stream \
--enable_live_transcription
```
```bash
# HF Inference Providers: GPT-oss-20B via Groq
speech-to-speech \
--stt parakeet-tdt \
--llm_backend responses-api \
--tts qwen3 \
--qwen3_tts_mlx_quantization 6bit \
--model_name "openai/gpt-oss-20b:groq" \
--responses_api_base_url "https://router.huggingface.co/v1" \
--responses_api_api_key "$HF_TOKEN" \
--responses_api_stream \
--enable_live_transcription
```
### Chat Completions Backend
Identical configuration to `responses-api`, reusing the same `--responses_api_*` connection flags, but talks to `/v1/chat/completions` instead of `/v1/responses`. Prefer it when:
- the provider ignores `chat_template_kwargs.enable_thinking` on the Responses path and needs a `reasoning_effort` knob to suppress reasoning, or
- the server's Responses streaming tool-call path is unreliable, while its Chat Completions tool-call streaming is solid. This is useful for some vLLM builds; see [#312](https://github.com/huggingface/speech-to-speech/issues/312).
Add `--responses_api_reasoning_effort none` to disable reasoning on providers where the chat-template flag has no effect:
```bash
# vLLM serving a Qwen model with tool calling
speech-to-speech \
--mode realtime \
--stt parakeet-tdt \
--llm_backend chat-completions \
--tts qwen3 \
--model_name "Qwen/Qwen3-4B-Instruct-2507" \
--responses_api_base_url "http://localhost:8000/v1" \
--responses_api_stream
```
```bash
# Gemma 4 31B via the HF router on Cerebras, with reasoning disabled for low voice latency
speech-to-speech \
--mode realtime \
--stt parakeet-tdt \
--llm_backend chat-completions \
--tts qwen3 \
--model_name "google/gemma-4-31B-it:cerebras" \
--responses_api_base_url "https://router.huggingface.co/v1" \
--responses_api_api_key "$HF_TOKEN" \
--responses_api_reasoning_effort none \
--responses_api_stream
```
### Fully Local
Run the LLM in a separate llama.cpp process for the lowest-friction fully local setup, as shown in the [Reachy Mini local conversation guide](https://huggingface.co/blog/local-reachy-mini-conversation):
```bash
# Terminal 1: llama.cpp serving Gemma 4
llama-server -hf ggml-org/gemma-4-E4B-it-GGUF -np 2 -c 65536 -fa on --swa-full
```
```bash
# Terminal 2: speech-to-speech using that local LLM server
speech-to-speech \
--mode realtime \
--stt parakeet-tdt \
--llm_backend responses-api \
--tts qwen3 \
--model_name "ggml-org/gemma-4-E4B-it-GGUF" \
--responses_api_base_url "http://127.0.0.1:8080/v1" \
--responses_api_api_key "" \
--responses_api_stream \
--enable_live_transcription
```
You can use `--mode local` instead of `--mode realtime` when you want to talk through the machine running the server directly. In-process local backends are still available with `--llm_backend mlx-lm` on Apple Silicon or `--llm_backend transformers` on CUDA / CPU.
## Multi-Language Support
Language coverage depends on the STT and TTS backends you pick, not on the pipeline itself:
| Component | Backend | Languages |
|---|---|---|
| STT | Parakeet TDT (default) | 25 European languages |
| STT | Whisper / Whisper MLX / Faster Whisper | Broad multilingual coverage, depending on the selected Whisper checkpoint |
| STT | Paraformer | Depends on the selected FunASR checkpoint; the default is Chinese-oriented |
| TTS | Qwen3-TTS (default) | Multilingual, with `--qwen3_tts_language auto` by default |
| TTS | Kokoro | Multiple language/voice mappings, depending on backend availability |
| TTS | ChatTTS | English and Chinese |
| TTS | MMS TTS | Broad multilingual coverage through MMS checkpoints |
Make sure the STT, LLM, and TTS you pair all cover your target language(s). Two usage patterns:
- **Single language**: set `--language` to the target language code. The default is `en`.
- **Language switching**: set `--language auto`. The STT detects the language of each spoken prompt and forwards it to the LLM. Optionally add `--enable_lang_prompt` to append a "Please reply to my message in ..." instruction. It defaults to `False`; large LLMs usually infer the language from context, but the explicit instruction can help smaller models.
Automatic language detection:
```bash
speech-to-speech \
--stt parakeet-tdt \
--language auto \
--llm_backend mlx-lm \
--model_name "mlx-community/Qwen3-4B-Instruct-2507-bf16"
```
A single non-English language, Chinese in this example:
```bash
speech-to-speech \
--stt whisper-mlx \
--stt_model_name large-v3 \
--language zh \
--llm_backend mlx-lm \
--model_name mlx-community/Qwen3-4B-Instruct-2507-bf16
```
Both commands also work on top of `--local_mac_optimal_settings`; explicit `--stt` flags override the defaults it sets.
## Pocket TTS
Pocket TTS from Kyutai Labs provides streaming TTS with voice cloning:
```bash
speech-to-speech \
--tts pocket \
--pocket_tts_voice jean \
--pocket_tts_device cpu
```
Available voice presets: `alba`, `marius`, `javert`, `jean`, `fantine`, `cosette`, `eponine`, `azelma`. Custom voice files and Hugging Face paths also work.
## CLI Reference
References for all CLI arguments live in the [arguments classes](./src/speech_to_speech/arguments_classes) and in `speech-to-speech -h`.
### Module-Level Parameters
See [ModuleArguments](./src/speech_to_speech/arguments_classes/module_arguments.py). It allows setting:
- a common `--device`, if every part should run on the same device
- `--mode`: `realtime` (default), `local`, `socket`, or `websocket`
- STT implementation (`--stt`)
- LLM backend (`--llm_backend`: `transformers`, `mlx-lm`, `responses-api`, or `chat-completions`)
- TTS implementation (`--tts`)
- logging level
- realtime pipeline pool size (`--num_pipelines`)
### VAD Parameters
See [VADHandlerArguments](./src/speech_to_speech/arguments_classes/vad_arguments.py). Notable options:
- `--thresh`: threshold value to trigger voice activity detection.
- `--min_speech_ms`: minimum duration of detected voice activity to be considered speech.
- `--min_speech_continuation_ms`: sustain-bar hysteresis threshold for speech that continues a reopenable soft-ended, uncommitted turn within the reopen window. The default and recommended pairing is `--min_speech_ms 384 --min_speech_continuation_ms 192`.
- `--min_silence_ms`: minimum length of silence intervals for segmenting speech. Default is 64 ms.
- `--short_segment_merge_ms`: optional merge window for stitching adjacent VAD segments that are each shorter than `--min_speech_ms`.
- `--unanswered_reopen_ms`: sanity cap on how long a soft-ended speculative turn that has not yet received any assistant output stays reopenable.
### STT, LLM, and TTS Parameters
`model_name`, `torch_dtype`, and `device` are exposed for each STT, LLM, and TTS implementation. STT and TTS parameters use the handler prefix, for example `--stt_model_name` or `--qwen3_tts_device`. LLM model selection and chat settings are shared across backends via unprefixed flags, for example `--model_name` and `--chat_size`; backend-specific flags use the `responses_api_` prefix for the `responses-api` and `chat-completions` backends and the `llm_` prefix for local backends.
For example:
```bash
# Local transformers/mlx-lm backend
--model_name google/gemma-2b-it
# OpenAI-compatible backend
--llm_backend responses-api --model_name deepseek-chat --responses_api_base_url https://api.deepseek.com
```
### Generation Parameters
Other generation parameters can be set using the handler prefix plus `_gen_`, for example `--stt_gen_max_new_tokens 128` or `--llm_gen_temperature 0.7`. Parameters not yet exposed can be added to the relevant arguments class.
## Contributing
Issues and PRs are welcome. Good starting points are the [open issues](https://github.com/huggingface/speech-to-speech/issues). For larger changes, open an issue first to discuss the approach.
For local development:
```bash
uv sync
pytest
ruff check
```
## Star History
[![Star History Chart](assets/star-history.svg)](https://github.com/huggingface/speech-to-speech/stargazers)
## Citations
If you use this pipeline, please also cite the component models you run. The defaults are:
### Silero VAD
```bibtex
@misc{SileroVAD,
author = {Silero Team},
title = {Silero VAD: pre-trained enterprise-grade Voice Activity Detector (VAD), Number Detector and Language Classifier},
year = {2021},
publisher = {GitHub},
journal = {GitHub repository},
howpublished = {\url{https://github.com/snakers4/silero-vad}},
email = {hello@silero.ai}
}
```
### Parakeet TDT
```bibtex
@misc{parakeet-tdt,
author = {NVIDIA},
title = {Parakeet TDT 0.6B v3},
publisher = {Hugging Face},
howpublished = {\url{https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3}}
}
```
### Qwen3-TTS
```bibtex
@misc{qwen3-tts,
author = {Qwen Team},
title = {Qwen3-TTS},
publisher = {Hugging Face},
howpublished = {\url{https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}}
}
```
Citations for optional backends such as Kokoro, Pocket TTS, ChatTTS, Whisper variants, Paraformer, and MMS live in the respective [component READMEs](./src/speech_to_speech).

11
archive/README.md Normal file
View File

@ -0,0 +1,11 @@
# Archived Models
This directory stores sunset model implementations that are kept in-repo but are no longer wired into `s2s_pipeline.py`.
- STT: `moonshine` -> `archive/STT/moonshine_handler.py`
- TTS: `parler` -> `archive/TTS/parler_handler.py`
- TTS: `melo` -> `archive/TTS/melo_handler.py`
- Legacy args: `archive/arguments_classes/parler_tts_arguments.py`
- Legacy args: `archive/arguments_classes/melo_tts_arguments.py`
These models are also removed from default requirements. If you want to run them manually, install their dependencies separately.

0
archive/STT/__init__.py Normal file
View File

72
archive/STT/moonshine_handler.py Normal file
View File

@ -0,0 +1,72 @@
import os
os.environ['KERAS_BACKEND'] = 'torch'
import logging
import moonshine
import torch
from rich.console import Console
from speech_to_speech.baseHandler import BaseHandler
from speech_to_speech.pipeline.messages import VADAudio
logger = logging.getLogger(__name__)
console = Console()
class MoonshineSTTHandler(BaseHandler[VADAudio]):
"""
Handles the Speech To Text generation using a Moonshine model.
"""
def setup(
self,
model_name="moonshine/base",
torch_dtype="float16",
gen_kwargs={},
):
self.torch_dtype = getattr(torch, torch_dtype)
self.gen_kwargs = gen_kwargs
self.tokenizer = moonshine.load_tokenizer()
self.model = moonshine.load_model(model_name)
self.warmup()
def warmup(self):
logger.info(f"Warming up {self.__class__.__name__}")
n_steps = 2
dummy_input = torch.randn(
(1, 16000),
dtype=self.torch_dtype,
)
if torch.cuda.is_available():
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
torch.cuda.synchronize()
start_event.record()
for _ in range(n_steps):
_ = self.model.generate(dummy_input)
if torch.cuda.is_available():
end_event.record()
torch.cuda.synchronize()
logger.info(
f"{self.__class__.__name__}: warmed up! time: {start_event.elapsed_time(end_event) * 1e-3:.3f} s"
)
def process(self, vad_audio: VADAudio):
logger.debug("infering moonshine...")
pred_ids = self.model.generate(vad_audio.audio[None, :])
pred_text = self.tokenizer.decode_batch(pred_ids)[0]
logger.debug("finished whisper inference")
console.print(f"[yellow]USER: {pred_text}")
yield (pred_text, "en")

0
archive/TTS/__init__.py Normal file
View File

130
archive/TTS/melo_handler.py Normal file
View File

@ -0,0 +1,130 @@
from __future__ import annotations
import logging
from threading import Event
from typing import Any, Iterator
import librosa
import numpy as np
import torch
from melo.api import TTS
from rich.console import Console
from speech_to_speech.baseHandler import BaseHandler
from speech_to_speech.pipeline.cancel_scope import CancelScope
from speech_to_speech.pipeline.handler_types import TTSIn, TTSOut
from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse
from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
logger = logging.getLogger(__name__)
console = Console()
WHISPER_LANGUAGE_TO_MELO_LANGUAGE = {
"en": "EN",
"fr": "FR",
"es": "ES",
"zh": "ZH",
"ja": "JP",
"ko": "KR",
}
WHISPER_LANGUAGE_TO_MELO_SPEAKER = {
"en": "EN-BR",
"fr": "FR",
"es": "ES",
"zh": "ZH",
"ja": "JP",
"ko": "KR",
}
class MeloTTSHandler(BaseHandler[TTSIn, TTSOut]):
def setup(
self,
should_listen: Event,
device: str = "mps",
language: str = "en",
speaker_to_id: str = "en",
gen_kwargs: dict[str, Any] = {}, # Unused
blocksize: int = 512,
cancel_scope: CancelScope | None = None,
speculative_turns: SpeculativeTurnTracker | None = None,
) -> None:
self.should_listen = should_listen
self.cancel_scope = cancel_scope
self.speculative_turns = speculative_turns
self.device = device
self.language = language
self.model = TTS(language=WHISPER_LANGUAGE_TO_MELO_LANGUAGE[self.language], device=device)
self.speaker_id = self.model.hps.data.spk2id[WHISPER_LANGUAGE_TO_MELO_SPEAKER[speaker_to_id]]
self.blocksize = blocksize
self._initial_language = self.language
self.warmup()
def warmup(self) -> None:
logger.info(f"Warming up {self.__class__.__name__}")
_ = self.model.tts_to_file("text", self.speaker_id, quiet=True)
def process(self, tts_input: TTSIn) -> Iterator[TTSOut]:
if isinstance(tts_input, EndOfResponse):
yield AUDIO_RESPONSE_DONE
return
speculative_turns = getattr(self, "speculative_turns", None)
if speculative_turns and not speculative_turns.is_latest(
tts_input.turn_id,
tts_input.turn_revision,
):
logger.debug("Dropping stale TTS input for turn=%s rev=%s", tts_input.turn_id, tts_input.turn_revision)
return
gen = self.cancel_scope.generation if self.cancel_scope else None
language_code = tts_input.language_code
text = tts_input.text
console.print(f"[green]ASSISTANT: {text}")
if language_code is not None and self.language != language_code:
try:
self.model = TTS(
language=WHISPER_LANGUAGE_TO_MELO_LANGUAGE[language_code],
device=self.device,
)
self.speaker_id = self.model.hps.data.spk2id[WHISPER_LANGUAGE_TO_MELO_SPEAKER[language_code]]
self.language = language_code
except KeyError:
console.print(f"[red]Language {language_code} not supported by Melo. Using {self.language} instead.")
if self.device == "mps":
import time
start = time.time()
torch.mps.synchronize() # Waits for all kernels in all streams on the MPS device to complete.
torch.mps.empty_cache() # Frees all memory allocated by the MPS device.
_ = time.time() - start # Removing this line makes it fail more often. I'm looking into it.
try:
audio_chunk = self.model.tts_to_file(text, self.speaker_id, quiet=True)
except (AssertionError, RuntimeError) as e:
logger.error(f"Error in MeloTTSHandler: {e}")
audio_chunk = np.array([])
if len(audio_chunk) == 0:
return
audio_chunk = librosa.resample(audio_chunk, orig_sr=44100, target_sr=16000)
audio_chunk = (audio_chunk * 32768).astype(np.int16)
for i in range(0, len(audio_chunk), self.blocksize):
if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen):
logger.info("TTS generation cancelled (interruption)")
return
yield np.pad(
audio_chunk[i : i + self.blocksize],
(0, self.blocksize - len(audio_chunk[i : i + self.blocksize])),
)
def on_session_end(self) -> None:
if self.language != self._initial_language:
self.language = self._initial_language
self.model = TTS(language=WHISPER_LANGUAGE_TO_MELO_LANGUAGE[self.language], device=self.device)
self.speaker_id = self.model.hps.data.spk2id[WHISPER_LANGUAGE_TO_MELO_SPEAKER[self.language]]
logger.debug("Melo TTS session state reset")

244
archive/TTS/parler_handler.py Normal file
View File

@ -0,0 +1,244 @@
from __future__ import annotations
import logging
from threading import Event, Thread
from time import perf_counter
from typing import Any, Optional
import librosa
import numpy as np
import torch
from parler_tts import ParlerTTSForConditionalGeneration, ParlerTTSStreamer
from rich.console import Console
from transformers import (
AutoTokenizer,
)
from transformers.utils.import_utils import (
is_flash_attn_2_available,
)
from speech_to_speech.baseHandler import BaseHandler
from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse, TTSInput
from speech_to_speech.utils.utils import next_power_of_2
torch._inductor.config.fx_graph_cache = True
# mind about this parameter ! should be >= 2 * number of padded prompt sizes for TTS
torch._dynamo.config.cache_size_limit = 15
logger = logging.getLogger(__name__)
console = Console()
if not is_flash_attn_2_available() and torch.cuda.is_available():
logger.warn(
"""Parler TTS works best with flash attention 2, but is not installed
Given that CUDA is available in this system, you can install flash attention 2 with `uv pip install flash-attn --no-build-isolation`"""
)
WHISPER_LANGUAGE_TO_PARLER_SPEAKER = {
"en": "Jason",
"fr": "Christine",
"es": "Steven",
"de": "Nicole",
"pt": "Sophia",
"pl": "Alex",
"it": "Richard",
"nl": "Mark",
}
class ParlerTTSHandler(BaseHandler[TTSInput | EndOfResponse]):
def setup(
self,
should_listen,
model_name="parler-tts/parler-mini-v1-jenny",
device="cuda",
torch_dtype="float16",
compile_mode=None,
gen_kwargs={},
max_prompt_pad_length=8,
description=(
"Jenny speaks at a slightly slow pace with an animated delivery with clear audio quality."
),
play_steps_s=1,
blocksize=512,
use_default_speakers_list=True,
cancel_response: Event | None = None,
):
self.should_listen = should_listen
self.cancel_response = cancel_response
self.device = device
self.torch_dtype = getattr(torch, torch_dtype)
self.gen_kwargs = gen_kwargs
self.compile_mode = compile_mode
self.max_prompt_pad_length = max_prompt_pad_length
self.use_default_speakers_list = use_default_speakers_list
if self.use_default_speakers_list:
description = description.replace("Jenny", "")
self.speaker = "Jason"
self.description = description
self.model = ParlerTTSForConditionalGeneration.from_pretrained(
model_name, torch_dtype=self.torch_dtype
).to(device)
self.description_tokenizer = AutoTokenizer.from_pretrained(self.model.config.text_encoder._name_or_path)
self.prompt_tokenizer = AutoTokenizer.from_pretrained(model_name)
framerate = self.model.audio_encoder.config.frame_rate
self.play_steps = int(framerate * play_steps_s)
self.blocksize = blocksize
if self.compile_mode not in (None, "default"):
logger.warning(
"Torch compilation modes that captures CUDA graphs are not yet compatible with the TTS part. Reverting to 'default'"
)
self.compile_mode = "default"
if self.compile_mode:
self.model.generation_config.cache_implementation = "static"
self.model.forward = torch.compile(
self.model.forward, mode=self.compile_mode, fullgraph=True
)
self.warmup()
def prepare_model_inputs(
self,
prompt,
max_length_prompt=50,
pad=False,
):
pad_args_prompt = (
{"padding": "max_length", "max_length": max_length_prompt} if pad else {}
)
description = self.description
if self.use_default_speakers_list:
description = self.speaker + " " + self.description
tokenized_description = self.description_tokenizer(
description, return_tensors="pt"
).to(self.device)
input_ids = tokenized_description.input_ids
attention_mask = tokenized_description.attention_mask
tokenized_prompt = self.prompt_tokenizer(
prompt, return_tensors="pt", **pad_args_prompt
).to(self.device)
prompt_input_ids = tokenized_prompt.input_ids
prompt_attention_mask = tokenized_prompt.attention_mask
gen_kwargs = {
"input_ids": input_ids,
"attention_mask": attention_mask,
"prompt_input_ids": prompt_input_ids,
"prompt_attention_mask": prompt_attention_mask,
**self.gen_kwargs,
}
return gen_kwargs
def warmup(self):
logger.info(f"Warming up {self.__class__.__name__}")
if self.device == "cuda":
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
# 2 warmup steps for no compile or compile mode with CUDA graphs capture
n_steps = 1 if self.compile_mode == "default" else 2
if self.device == "cuda":
torch.cuda.synchronize()
start_event.record()
if self.compile_mode:
pad_lengths = [2**i for i in range(2, self.max_prompt_pad_length)]
for pad_length in pad_lengths[::-1]:
model_kwargs = self.prepare_model_inputs(
"dummy prompt", max_length_prompt=pad_length, pad=True
)
for _ in range(n_steps):
_ = self.model.generate(**model_kwargs)
logger.info(f"Warmed up length {pad_length} tokens!")
else:
model_kwargs = self.prepare_model_inputs("dummy prompt")
for _ in range(n_steps):
_ = self.model.generate(**model_kwargs)
if self.device == "cuda":
end_event.record()
torch.cuda.synchronize()
logger.info(
f"{self.__class__.__name__}: warmed up! time: {start_event.elapsed_time(end_event) * 1e-3:.3f} s"
)
def process(self, tts_input: TTSInput | EndOfResponse):
if isinstance(tts_input, EndOfResponse):
yield AUDIO_RESPONSE_DONE
return
runtime_config = tts_input.runtime_config
response = tts_input.response
language_code = tts_input.language_code
text = tts_input.text
voice: Optional[str] = None
if response and response.audio and response.audio.output:
voice = str(response.audio.output.voice) if response.audio.output.voice is not None else None
if not voice and runtime_config:
audio_cfg = runtime_config.session.audio
audio_output = audio_cfg.output if audio_cfg is not None else None
voice = str(audio_output.voice) if audio_output is not None and audio_output.voice else None
if voice:
self.speaker = voice
elif language_code:
self.speaker = WHISPER_LANGUAGE_TO_PARLER_SPEAKER.get(language_code, "Jason")
console.print(f"[green]ASSISTANT: {text}")
nb_tokens = len(self.prompt_tokenizer(text).input_ids)
pad_args: dict[str, Any] = {}
if self.compile_mode:
# pad to closest upper power of two
pad_length = next_power_of_2(nb_tokens)
logger.debug(f"padding to {pad_length}")
pad_args["pad"] = True
pad_args["max_length_prompt"] = pad_length
tts_gen_kwargs = self.prepare_model_inputs(
text,
**pad_args,
)
streamer = ParlerTTSStreamer(
self.model, device=self.device, play_steps=self.play_steps
)
tts_gen_kwargs = {"streamer": streamer, **tts_gen_kwargs}
torch.manual_seed(0)
thread = Thread(target=self.model.generate, kwargs=tts_gen_kwargs)
thread.start()
pipeline_start = perf_counter()
for i, audio_chunk in enumerate(streamer):
if self.cancel_response and self.cancel_response.is_set():
logger.info("TTS generation cancelled (interruption)")
return
if i == 0:
logger.info(
f"Time to first audio: {perf_counter() - pipeline_start:.3f}s"
)
audio_chunk = librosa.resample(audio_chunk, orig_sr=44100, target_sr=16000)
audio_chunk = (audio_chunk * 32768).astype(np.int16)
for i in range(0, len(audio_chunk), self.blocksize):
yield np.pad(
audio_chunk[i : i + self.blocksize],
(0, self.blocksize - len(audio_chunk[i : i + self.blocksize])),
)
if not runtime_config:
self.should_listen.set()

0
archive/__init__.py Normal file
View File

0
archive/arguments_classes/__init__.py Normal file
View File

17
archive/arguments_classes/melo_tts_arguments.py Normal file
View File

@ -0,0 +1,17 @@
from dataclasses import dataclass, field
@dataclass
class MeloTTSHandlerArguments:
melo_language: str = field(
default="en",
metadata={"help": "The language of the text to be synthesized. Default is 'EN_NEWEST'."},
)
melo_device: str = field(
default="auto",
metadata={"help": "The device to be used for speech synthesis. Default is 'auto'."},
)
melo_speaker_to_id: str = field(
default="en",
metadata={"help": "Mapping of speaker names to speaker IDs. Default is ['EN-Newest']."},
)

68
archive/arguments_classes/parler_tts_arguments.py Normal file
View File

@ -0,0 +1,68 @@
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class ParlerTTSHandlerArguments:
tts_model_name: str = field(
default="parler-tts/parler-mini-v1-jenny",
metadata={
"help": "The pretrained TTS model to use. Default is 'parler-tts/parler-mini-v1-jenny'."
},
)
tts_device: str = field(
default="cuda",
metadata={
"help": "The device type on which the model will run. Default is 'cuda' for GPU acceleration."
},
)
tts_torch_dtype: str = field(
default="float16",
metadata={
"help": "The PyTorch data type for the model and input tensors. One of `float32` (full-precision), `float16` or `bfloat16` (both half-precision)."
},
)
tts_compile_mode: Optional[str] = field(
default=None,
metadata={
"help": "Compile mode for torch compile. Either 'default', 'reduce-overhead' and 'max-autotune'. Default is None (no compilation)"
},
)
tts_gen_min_new_tokens: int = field(
default=64,
metadata={
"help": "Maximum number of new tokens to generate in a single completion. Default is 64, which corresponds to ~0.74 secs"
},
)
tts_gen_max_new_tokens: int = field(
default=1024,
metadata={
"help": "Maximum number of new tokens to generate in a single completion. Default is 1024, which corresponds to ~12 secs"
},
)
description: str = field(
default=(
"Jenny speaks at a slightly slow pace with an animated delivery with clear audio quality."
),
metadata={
"help": "Description of the speaker's voice and speaking style to guide the TTS model."
},
)
play_steps_s: float = field(
default=1.0,
metadata={
"help": "The time interval in seconds for playing back the generated speech in steps. Default is 1.0 seconds."
},
)
max_prompt_pad_length: int = field(
default=8,
metadata={
"help": "When using compilation, the prompt as to be padded to closest power of 2. This parameters sets the maximun power of 2 possible."
},
)
use_default_speakers_list: bool = field(
default=False,
metadata={
"help": "Whether to use the default list of speakers or not."
},
)

15
assets/star-history.svg Normal file
View File

@ -0,0 +1,15 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 420" font-family="-apple-system,'Segoe UI',Helvetica,Arial,sans-serif">
<style>
.lbl { font-size: 12px; fill: #8b949e; }
.title { font-size: 16px; font-weight: 600; fill: #8b949e; }
.total { font-size: 13px; fill: #8b949e; }
</style>
<text x="70" y="28" class="title">huggingface/speech-to-speech star history</text>
<text x="770" y="28" text-anchor="end" class="total">6,165 stars</text>
<line x1="70" y1="360.0" x2="770" y2="360.0" stroke="#8b949e" stroke-opacity="0.25" stroke-width="1"/><line x1="70" y1="298.0" x2="770" y2="298.0" stroke="#8b949e" stroke-opacity="0.25" stroke-width="1"/><line x1="70" y1="236.0" x2="770" y2="236.0" stroke="#8b949e" stroke-opacity="0.25" stroke-width="1"/><line x1="70" y1="174.0" x2="770" y2="174.0" stroke="#8b949e" stroke-opacity="0.25" stroke-width="1"/><line x1="70" y1="112.0" x2="770" y2="112.0" stroke="#8b949e" stroke-opacity="0.25" stroke-width="1"/><line x1="70" y1="50.0" x2="770" y2="50.0" stroke="#8b949e" stroke-opacity="0.25" stroke-width="1"/>
<text x="60" y="364.0" text-anchor="end" class="lbl">0</text><text x="60" y="302.0" text-anchor="end" class="lbl">1.6k</text><text x="60" y="240.0" text-anchor="end" class="lbl">3.2k</text><text x="60" y="178.0" text-anchor="end" class="lbl">4.8k</text><text x="60" y="116.0" text-anchor="end" class="lbl">6.4k</text><text x="60" y="54.0" text-anchor="end" class="lbl">8k</text>
<text x="70.0" y="382" text-anchor="middle" class="lbl">Aug 2024</text><text x="210.0" y="382" text-anchor="middle" class="lbl">Jan 2025</text><text x="350.0" y="382" text-anchor="middle" class="lbl">May 2025</text><text x="490.0" y="382" text-anchor="middle" class="lbl">Oct 2025</text><text x="630.0" y="382" text-anchor="middle" class="lbl">Feb 2026</text><text x="770.0" y="382" text-anchor="middle" class="lbl">Jul 2026</text>
<path d="M70.0,360.0 L70.9,358.0 L71.1,356.0 L71.1,354.0 L71.1,352.1 L71.2,350.1 L71.4,348.1 L71.7,346.1 L71.9,344.2 L72.1,342.2 L72.3,340.2 L72.4,338.2 L72.7,336.2 L73.1,334.3 L73.1,332.3 L73.2,330.3 L73.3,328.3 L73.5,326.4 L73.6,324.4 L73.8,322.4 L73.9,320.4 L73.9,318.5 L74.1,316.5 L74.2,314.5 L74.3,312.5 L74.4,310.6 L74.6,308.6 L74.7,306.6 L74.8,304.6 L74.9,302.6 L74.9,300.7 L74.9,298.7 L75.0,296.7 L75.0,294.7 L75.6,292.8 L75.9,290.8 L76.0,288.8 L76.1,286.8 L76.2,284.9 L76.6,282.9 L77.0,280.9 L77.8,278.9 L79.1,277.0 L81.6,275.0 L83.0,273.0 L83.7,271.0 L84.2,269.1 L85.3,267.1 L86.8,265.1 L88.2,263.1 L89.9,261.1 L90.0,259.2 L90.2,257.2 L91.0,255.2 L91.9,253.2 L93.6,251.3 L98.0,249.3 L104.4,247.3 L111.0,245.3 L117.7,243.4 L119.7,241.4 L127.8,239.4 L133.7,237.4 L139.7,235.5 L146.1,233.5 L147.4,231.5 L153.1,229.5 L161.8,227.6 L178.2,225.6 L196.2,223.6 L219.0,221.6 L246.0,219.6 L260.9,217.7 L272.1,215.7 L280.0,213.7 L292.0,211.7 L313.3,209.8 L336.2,207.8 L367.4,205.8 L397.9,203.8 L436.2,201.9 L478.3,199.9 L523.9,197.9 L588.9,195.9 L611.5,194.0 L611.9,192.0 L613.0,190.0 L627.4,188.0 L644.6,186.1 L648.8,184.1 L659.3,182.1 L681.9,180.1 L699.4,178.1 L716.2,176.2 L722.5,174.2 L732.5,172.2 L752.4,170.2 L756.9,168.3 L757.3,166.3 L757.5,164.3 L757.8,162.3 L758.2,160.4 L758.5,158.4 L758.9,156.4 L759.3,154.4 L760.1,152.5 L761.1,150.5 L761.5,148.5 L762.0,146.5 L762.4,144.6 L762.9,142.6 L763.3,140.6 L763.7,138.6 L764.1,136.6 L764.4,134.7 L764.9,132.7 L764.9,130.7 L765.3,128.7 L765.8,126.8 L766.6,124.8 L768.2,122.8 L770.0,121.1 L770.0,121.1 L770.0,360.0 L70.0,360.0 Z" fill="#f4b400" fill-opacity="0.12"/>
<path d="M70.0,360.0 L70.9,358.0 L71.1,356.0 L71.1,354.0 L71.1,352.1 L71.2,350.1 L71.4,348.1 L71.7,346.1 L71.9,344.2 L72.1,342.2 L72.3,340.2 L72.4,338.2 L72.7,336.2 L73.1,334.3 L73.1,332.3 L73.2,330.3 L73.3,328.3 L73.5,326.4 L73.6,324.4 L73.8,322.4 L73.9,320.4 L73.9,318.5 L74.1,316.5 L74.2,314.5 L74.3,312.5 L74.4,310.6 L74.6,308.6 L74.7,306.6 L74.8,304.6 L74.9,302.6 L74.9,300.7 L74.9,298.7 L75.0,296.7 L75.0,294.7 L75.6,292.8 L75.9,290.8 L76.0,288.8 L76.1,286.8 L76.2,284.9 L76.6,282.9 L77.0,280.9 L77.8,278.9 L79.1,277.0 L81.6,275.0 L83.0,273.0 L83.7,271.0 L84.2,269.1 L85.3,267.1 L86.8,265.1 L88.2,263.1 L89.9,261.1 L90.0,259.2 L90.2,257.2 L91.0,255.2 L91.9,253.2 L93.6,251.3 L98.0,249.3 L104.4,247.3 L111.0,245.3 L117.7,243.4 L119.7,241.4 L127.8,239.4 L133.7,237.4 L139.7,235.5 L146.1,233.5 L147.4,231.5 L153.1,229.5 L161.8,227.6 L178.2,225.6 L196.2,223.6 L219.0,221.6 L246.0,219.6 L260.9,217.7 L272.1,215.7 L280.0,213.7 L292.0,211.7 L313.3,209.8 L336.2,207.8 L367.4,205.8 L397.9,203.8 L436.2,201.9 L478.3,199.9 L523.9,197.9 L588.9,195.9 L611.5,194.0 L611.9,192.0 L613.0,190.0 L627.4,188.0 L644.6,186.1 L648.8,184.1 L659.3,182.1 L681.9,180.1 L699.4,178.1 L716.2,176.2 L722.5,174.2 L732.5,172.2 L752.4,170.2 L756.9,168.3 L757.3,166.3 L757.5,164.3 L757.8,162.3 L758.2,160.4 L758.5,158.4 L758.9,156.4 L759.3,154.4 L760.1,152.5 L761.1,150.5 L761.5,148.5 L762.0,146.5 L762.4,144.6 L762.9,142.6 L763.3,140.6 L763.7,138.6 L764.1,136.6 L764.4,134.7 L764.9,132.7 L764.9,130.7 L765.3,128.7 L765.8,126.8 L766.6,124.8 L768.2,122.8 L770.0,121.1 L770.0,121.1" fill="none" stroke="#f4b400" stroke-width="2.5" stroke-linejoin="round"/>
<circle cx="770.0" cy="121.1" r="4" fill="#f4b400"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.1 KiB

BIN
chile_female.mp3 Normal file
View File

Binary file not shown.

BIN
chile_female.wav Normal file
View File

Binary file not shown.

BIN
demo.zip Normal file
View File

Binary file not shown.

12
demo/.dockerignore Normal file
View File

@ -0,0 +1,12 @@
.git
.gitignore
__pycache__/
*.pyc
.venv/
venv/
.vscode/
.idea/
.DS_Store
*.log
docs/
.claude/

21
demo/.gitignore vendored Normal file
View File

@ -0,0 +1,21 @@
# macOS metadata
.DS_Store
# Editor scratch
.vscode/
.idea/
*.swp
# Static HTTP server logs (we don't ship them)
*.log
# Python
__pycache__/
*.pyc
.venv/
venv/
# Local-only replica deploy script (never tracked)
deploy_replica.py
.env

108
demo/CONTEXT.md Normal file
View File

@ -0,0 +1,108 @@
# Context glossary
Canonical terms for this space. A glossary, not a spec — it defines what words mean,
not how anything is built. Keep design/implementation detail in `DESIGN.md` and the
code.
## Speech-to-speech demo
The product: a voice conversation you have with a model by tapping the orb and
talking. "The demo" and "the space" refer to this same thing. It runs on Hugging
Face's open `speech-to-speech` backend.
## The pipeline
The ordered path a turn travels, from your voice to the orb's reply. Order is
meaningful — each stage consumes the previous one's output:
`you speak → VAD → STT → VLM → TTS → orb replies`
- **VAD** — voice activity detection. Decides *when* you are speaking, so the system
knows a turn has started and ended. Model: silero-vad.
- **STT** — speech to text. Transcribes your speech into words. Model:
nvidia/parakeet-tdt-1.1b.
- **VLM** — the vision-language model that composes the reply. Served via Cerebras.
Model: google/gemma-4-31B-it.
- **TTS** — text to speech. Speaks the reply back in the chosen voice. Model:
Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice.
## Builder
A Hugging Face user credited with making the space, shown by HF username. Current
builders: tfrere, A-Mahla and andito. Distinct from the *models'* authors (nvidia, google,
Qwen, snakers4), who are credited per pipeline stage.
## Powered by
The infrastructure running the pipeline, named in the about panel: Hugging Face
Inference Endpoints (hosting) and Cerebras (LLM inference). Distinct from "built by"
(the people) and from the model authors (who trained each model).
## Tool
A function the model can call mid-conversation to do something the pipeline
can't do on its own (look something up, look through the camera). Tools are
declared to the backend in the session config; the model decides when to call
one. Distinct from the *pipeline stages* (VAD/STT/VLM/TTS), which always run.
## Tool executor
The client-side component that runs a tool when the model calls it and returns
the result to the backend, so the model can speak the answer. It is the missing
half of the round-trip: the backend already emits the call, the executor runs it
and replies. Distinct from the *tool* itself (the thing being run).
## Web search tool
A tool that looks something up on the web for the model. The model calls it with
a query; the tool executor forwards the query to the search proxy and returns the
results as the tool result. Activates only when a search key is available.
## Camera snapshot tool
A tool that lets the model look through the user's webcam. While enabled, a live
self-view is shown in the page (bottom-left); when the model calls the tool, the
executor captures a frame and sends it to the model as an image so the VLM can
see it. Distinct from the *preview* (what the user sees) and the *snapshot* (the
single frame sent to the model).
## Search proxy
The same-origin server route (`/search`) that holds the search key and calls the
external search provider on the client's behalf, so the key never reaches the
browser. Lives in the same container as the page. Distinct from the *s2s backend*
(the separate load-balanced speech-to-speech service).
## Tools panel
The dialog opened from the "Tools" button in the top-right, holding one switch per
tool (and the web-search key status). Turning a switch on/off declares or removes
that tool on the live session. Distinct from *Settings* (connection, voice,
instructions) and the *About panel* (project info).
## Identity block
The top-left corner of the topbar (replacing the old wordmark): the demo name, a
one-line blurb, and the "powered by" / "built by" credits, shown directly rather
than hidden behind a click.
## About panel
The popup opened from the (i) icon to the right of the identity block. Holds the
general introduction to the speech-to-speech project (with a repo link) and the
pipeline. Identity itself now lives in the corner, not here.
## Queue
The line of users waiting for a free conversation slot when every compute is
busy. You join the queue instead of being turned away; you leave it by reaching
the front or by giving up. Distinct from a *session* (an actual live
conversation) — being in the queue is not yet talking, and time spent waiting
never counts against your usage limit.
## Ticket
Your held place in the queue. Created when you join, it is what the demo checks
to tell you your position and to notice if you have left. A ticket is not a
session: it only promises a spot in line, not a compute.
## Position
How many people are ahead of you in the queue, shown while you wait ("You're #3
in line"). It only ever counts down. Distinct from an *estimated wait* — the
demo shows position, never a time, because wait time is unpredictable.
## Claim
The moment you reach the front and a free slot becomes yours — the queue hands
off to a real session and the conversation begins. This is also the point where
your usage limit first starts to matter (never while waiting).
## At capacity
The state where the queue itself is full, so new users can't even join the line
and are asked to try again shortly. Distinct from simply *busy* (all computes
taken but the queue still has room to wait in).

205
demo/DESIGN.md Normal file
View File

@ -0,0 +1,205 @@
# Design language
The reference for keeping this app visually coherent as it grows. Read it before
touching `style.css`, `index.html`, or any DOM-building code in `main.js`. Every
rule here is already live in the codebase — this file explains the *why* so changes
extend the system instead of drifting from it.
---
## The thesis: color belongs to the voice
This is a voice app. The one thing in the room that should have color is the thing
that is talking. So:
- **The orb** carries saturated color. It glows, and the glow's hue changes with
conversational state.
- **Everything else is monochrome** — a precise cool-grey dark canvas. Surfaces,
borders, buttons, panels, the transcript: all greyscale.
- The **only** exceptions are tiny *role echoes* (a one-word mono label, a small
icon) that borrow the orb's state hue so the transcript reads in the same color
language the orb speaks. They are accents the size of a word, never fills.
- **Brand logos keep their own color.** The Hugging Face mark (`#ffd21e`) and the
Cerebras mark (`#f15a29`) render in their brand colors in the identity credits and
the about panel — a deliberate, owner-approved exception. It applies to those two
logos only; do not generalize it to other chrome.
If you find yourself adding a tinted background, a colored border, or a bright
button anywhere outside the orb, stop — that color almost certainly belongs to the
orb instead, or shouldn't exist.
---
## Color tokens
All defined in `:root` in `style.css`. Use the variables, never raw hex in rules.
### Canvas (the monochrome world)
| Token | Value | Use |
|---|---|---|
| `--bg` | `#0a0b10` | Page background (a cool near-black) |
| `--bg-elev` | `#13151c` | Raised surfaces: bubbles, panels, icon buttons |
| `--bg-elev-2` | `#1b1e29` | Surfaces on surfaces: history bodies, inputs |
| `--border` | `rgba(255,255,255,.08)` | Default hairline |
| `--border-strong` | `rgba(255,255,255,.16)` | Emphasised hairline |
| `--text` | `#f5f6fa` | Primary text; also the *primary button* fill |
| `--text-dim` | `rgba(245,246,250,.65)` | Secondary text |
| `--text-faint` | `rgba(245,246,250,.42)` | Captions, labels, footer |
### Voice (the only saturated hues)
These are the orb's state colors. They appear on the orb, and as small role echoes
in the transcript — nowhere else.
| Token | Value | Meaning |
|---|---|---|
| `--accent` / `--speaking` | `#8b7dff` violet | Assistant speaking |
| `--accent-2` / `--listening` | `#22d3ee` cyan | You / listening |
| `--processing` | `#f59e0b` amber | Thinking / tool call |
| `--error` | `#ff6a75` | Error |
| `--success` | `#34d399` | Ready / connected |
### Role echoes (semantic aliases — use these in chat code)
| Token | Maps to | Where it shows |
|---|---|---|
| `--voice-user` | cyan | `YOU` label + user bubble accents |
| `--voice-assistant` | violet | `ASSISTANT` label + assistant accents |
| `--voice-tool` | amber | `TOOL CALL` label, wrench icon |
**Why this mapping:** it mirrors the orb exactly — when *you* speak the orb is cyan
(`state-listening`), when the *assistant* speaks it's violet (`state-ai-speaking`),
when it's working it's amber (`state-processing`). The transcript is a quiet replay
of the orb's color story.
### Orb state → glow (`.circle.state-*` → `--glow`)
| State | Glow |
|---|---|
| `signed-out` | violet `#8b7dff` |
| `authenticated` / `ready` | green `#34d399` |
| `connecting` / `connected` / `starting` | yellow `#facc15` |
| `listening` / `user-speaking` | cyan (`--listening`) |
| `processing` | amber (`--processing`) |
| `ai-speaking` | violet (`--speaking`) |
| `error` | red (`--error`) |
Adding a new state? Give it a `--glow`, and if it surfaces in the transcript, add a
matching `--voice-*` alias rather than a one-off color.
---
## Typography
Two faces, two jobs. Never reach for a third.
- **Inter** — body and UI. Wordmark, buttons, inputs, panel titles, prose, history
message bodies. The workhorse; it should feel neutral and get out of the way.
- **Geist Mono** (`--font-mono`) — the **machine voice**. Reserved for text the
*system* emits or identifiers it reports, never for human prose.
### When mono is correct
Mono signals "this is the machine talking or naming itself." Use it for:
- the orb's status caption (`.circle-caption`)
- role eyebrows (`YOU` / `ASSISTANT` / `TOOL CALL`)
- tool-call names and argument JSON
- the `·WebSocket` transport tag, bitrate readouts, connection identifiers
- the empty-state label
Mono text is set uppercase with `letter-spacing: ~0.1–0.14em` and weight `500`, so it
reads as a typed status line, not a headline. Body copy, button labels, and
explanatory `small` text stay **Inter** — putting prose in mono breaks the metaphor.
The font is loaded in `index.html`; the stack falls back to system mono gracefully if
the CDN is blocked.
---
## Layout
- **One continuous canvas.** No dividers under the topbar or above the footer, no
panel chrome competing with content. The topbar and footer float over the stage.
Keep it that way — a new section earns a hairline (`--border`) only if it genuinely
needs separating.
- **The orb is the hero and the center of gravity.** It sits dead-center on the
stage. Controls flank it (mic / stop), captions sit beneath. Don't crowd it.
- **Hairlines, not boxes.** Separation comes from `1px` borders at 8–16% white and
from spacing, not from heavy fills or shadows. Shadows are soft and low
(`0 4px 18px rgba(0,0,0,.32)`), used only to lift floating elements (bubbles,
panels, modal).
- **Radii:** `--radius-sm: 8px` (buttons, inputs, chips), `--radius-md: 14px`
(bubbles, message bodies, modal), `--radius-lg: 22px` (reserved). Pick by element
size; don't invent new values.
- **Two reading surfaces for the transcript:** ephemeral bubbles top-right (desktop
only) that log and fade, and a slide-in history panel for review. On phones the
bubble stream is dropped and the panel goes full-screen — the panel is the single
source of truth there.
---
## Components
- **Buttons.** Default (`.btn`) is a neutral elevated surface. The *primary* button
is **near-white on dark** (`--text` fill, `--bg` text) — the highest-contrast thing
on the page that *isn't* the orb. There is no colored button; emphasis comes from
contrast, not hue.
- **Icon buttons** (`.icon-btn`) are `36px`, elevated surface, dim icon that brightens
on hover. Side controls (`.side-btn`) are circular, collapse to zero size until the
session is live (by width on desktop, by height in the mobile column).
- **Chat bubbles & history messages** share one neutral surface. They are
distinguished by **side** (you = left, assistant = right) plus the **mono role
label** in the role-echo hue — not by tinted fills. Tool entries use the wrench icon
+ mono + amber, on the same neutral surface.
- **Badge** (new-message dot) is monochrome white — a signal, not a color accent.
- **Focus** is visible and neutral: inputs focus to `--text-dim`; the orb uses a
`--glow`-colored outline (it's the orb, so color is allowed).
---
## Motion
- **The orb is audio-reactive, not timer-driven.** Mic RMS (`--audio-level`) and the
assistant output level (`--ai-audio-level`) drive scale/opacity at display rate, so
every syllable moves it. This is the signature animation — keep new motion
subordinate to it.
- **Quiet by default.** Breathing/glow throbs are slow (1.4–2.4s) and low-contrast.
Resist adding scattered micro-animations; an orchestrated moment beats many small
ones, and excess motion reads as AI-generated.
- **First paint is frozen.** `body.booting` disables all transitions until the first
frame commits (stripped after one rAF in `main.js`). Anything new that would
otherwise animate-in on load must respect this.
- Honor `prefers-reduced-motion` for any motion you add.
---
## Writing / copy voice
- Sentence case, plain verbs, no filler. Tuned and quiet — match the minimal canvas.
- Name things by what the user controls, not by the system's internals. A button says
exactly what it does, and keeps the same word through the flow.
- **Empty states invite action** ("Tap the orb and start talking"), they don't just
set a mood.
- **Errors state what happened and how to recover**, in the interface's voice — they
don't apologize and are never vague.
- Mono labels are terse identifiers (`YOU`, `TOOL CALL`); prose stays in Inter.
---
## Responsive floor (non-negotiable)
Every change ships meeting these:
- Works down to a `360px`-wide phone. The `@media (max-width: 600px)` block already
handles the phone layout — extend it, don't fight it.
- Visible keyboard focus on every interactive element.
- `prefers-reduced-motion` respected.
- Tap targets ≥ `44px`; `touch-action: manipulation` on anything tappable.
---
## Before you ship — the mirror check
1. Is every saturated color either on the orb or a word-sized role echo? If a fill or
border is colored, remove that accessory.
2. Is mono used only for machine/system text, and Inter for everything human?
3. Does any new state have both a `--glow` and (if it appears in chat) a `--voice-*`
alias?
4. Are separations hairlines + spacing, not boxes and heavy shadows?
5. Did you add motion? Is it quieter than the orb and reduced-motion-safe?
6. Remove one accessory. The minimal look survives on precision, not addition.

14
demo/Dockerfile Normal file
View File

@ -0,0 +1,14 @@
# Single container: serves the static front-end AND the /api/search proxy.
# HF Spaces (sdk: docker) routes traffic to $PORT, default 7860.
FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 7860
CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "7860"]

190
demo/README.md Normal file
View File

@ -0,0 +1,190 @@
---
title: HF Realtime Voice
emoji: 🎙️
colorFrom: indigo
colorTo: purple
sdk: docker
app_port: 7860
pinned: false
short_description: Voice chat over WebSocket against a HF speech-to-speech
hf_oauth: true
---
# Realtime Voice Demo (WebSocket transport)
Browser voice-chat UI for the
[huggingface/speech-to-speech](https://github.com/huggingface/speech-to-speech)
backend. The browser streams mic audio over a WebSocket using the OpenAI
Realtime **GA** protocol and plays back the assistant's audio as it arrives.
## Quick start (local)
1. **Start the speech-to-speech backend** in realtime mode (from the repo root;
see the [backend README](https://github.com/huggingface/speech-to-speech/blob/main/src/speech_to_speech/api/openai_realtime/README.md)
for more model combinations):
```bash
uv run speech-to-speech \
--mode realtime \
--stt parakeet-tdt \
--llm_backend transformers \
--tts kokoro \
--model_name "Qwen/Qwen3-4B-Instruct-2507" \
--llm_device mps \
--llm_torch_dtype float16 \
--enable_live_transcription
```
The realtime server listens on `ws://localhost:8765/v1/realtime` by default
(`--ws_host` / `--ws_port` to change).
2. **Start this app**, pointing it at the backend with `SPEECH_TO_SPEECH_URL`:
```bash
uv pip install -r demo/requirements.txt
export SPEECH_TO_SPEECH_URL=ws://localhost:8765/v1/realtime
export SERPER_API_KEY=... # optional; web search is disabled without it
uv run uvicorn --app-dir demo server:app --reload --port 7860
```
Or with Docker:
```bash
docker build -t s2s-demo demo/
docker run -p 7860:7860 -e SPEECH_TO_SPEECH_URL=ws://host.docker.internal:8765/v1/realtime s2s-demo
```
3. Open <http://localhost:7860/>, click the orb, allow the mic, talk.
> Browsers require **HTTPS or `localhost`** for `getUserMedia()` (mic + camera).
> `127.0.0.1` and `localhost` both work; plain `http://192.168.x.y` does NOT.
Smoke-test the backend from the shell:
```bash
websocat ws://localhost:8765/v1/realtime
# -> you should get a session.created event back immediately
```
## How it works
1. The browser opens a WebSocket on the configured `/v1/realtime` URL.
2. Server pushes `session.created` on connect. Client replies with
`session.update` (OpenAI Realtime **GA** schema: `session.audio.input`,
`session.audio.output`, `session.output_modalities`).
3. Client streams mic audio as PCM16 16 kHz mono base64 chunks
(`input_audio_buffer.append`, one frame every ~40 ms).
4. Server pushes `response.output_audio.delta` (PCM16 24 kHz mono base64)
and transcript deltas.
The backend exposes one concurrent session per pipeline unit
(`--num_pipelines` to serve more).
## Connecting to a backend
Three modes, picked by env (`/api/config` tells the client which one is active):
- **`SPEECH_TO_SPEECH_URL` env** — the mode you want for local use, and the
highest priority. The browser connects **directly** to this realtime
WebSocket URL; it's shown read-only in Settings. Setting it disables the
load-balancer logic entirely (no `/api/session` proxy, no queue, no
metering, no sign-in). Unlike the LB address it is not a secret. Accepts a
full `ws(s)://host/v1/realtime` URL or a bare host like `localhost:8765`
(the app adds `/v1/realtime`).
- **Neither env set** — **Settings → Speech-to-speech server URL**: paste a
full connect URL or a bare host, and the browser connects to it directly.
- **`LOAD_BALANCER_URL` env** — multi-compute deployments only: the browser
POSTs the same-origin `/api/session` proxy, the server forwards to the LB,
and the browser dials the per-session compute URL the LB hands back. The LB
address never reaches the browser; the Settings URL field is hidden.
| `SPEECH_TO_SPEECH_URL` | `LOAD_BALANCER_URL` | `SPACE_ID` | Connection | URL field | Metering |
|:---:|:---:|:---:|---|---|---|
| ✅ | any | any | direct → pinned URL | visible, locked | off |
| – | – | any | direct → user URL | editable | off |
| – | ✅ | ✅ | LB proxy | hidden | **on** |
| – | ✅ | – | LB proxy | hidden | off |
**Settings → Restart** reconnects with the current voice, instructions and URL.
## Tools
The assistant can call two tools mid-conversation (toggle them from the **Tools**
button, top-right):
- **Web search** — Google results via Serper.dev, proxied server-side so the key
never reaches the browser. Set `SERPER_API_KEY` as an env var / Space secret.
Without it, the tool is disabled unless the user pastes their own key in the
Tools panel.
- **Camera** — while enabled, a live self-view shows bottom-left; when the model
calls the tool, the current frame is sent to the vision-language model so it can
see what you're showing it.
## Usage limits (deployed Space only)
Conversation time is metered per UTC day by sign-in tier (see `limiter.py` /
`auth.py`), but **only on the deployed Space** — metering turns on only when BOTH
`LOAD_BALANCER_URL` and `SPACE_ID` (injected automatically by the HF Space
runtime) are present. Running locally — even with `LOAD_BALANCER_URL` exported —
leaves the app unmetered. Tunable via env:
| Env | Default | What |
|-----|---------|------|
| `LIMIT_ANON_SEC` | `300` | Daily seconds for anonymous visitors (5 min) |
| `LIMIT_FREE_SEC` | `600` | Daily seconds for signed-in non-PRO users (10 min) |
| `UNLIMITED_ORGS` | _(adds to defaults)_ | Extra HF org names whose members get **unlimited** usage, like PRO |
| `USAGE_HASH_SECRET` | _(random)_ | HMAC secret for hashing identity keys + signing the anon cookie |
PRO members are always unlimited. Members of `cerebras`, `HuggingFaceM4`,
`smolagents`, and `pollen-robotics` are unlimited out of the box (shown as
"Team", not "PRO"); set `UNLIMITED_ORGS=my-team` to add more. Matched
case-insensitively against the user's organisations from HF OAuth.
## Settings (stored in `localStorage`)
| Key | What |
|-----|------|
| Speech-to-speech server URL | Direct realtime WebSocket URL (hidden/locked when pinned by env) |
| Voice | Qwen3-TTS speaker name (Aiden, Ryan, Dylan, Eric, Ono_Anna, Serena, Sohee, Uncle_Fu, Vivian) |
| Instructions | System prompt sent in `session.update` once the WS opens |
LocalStorage keys are namespaced `s2s.ws.*` so this app's settings do
NOT collide with the WebRTC variant.
## Files
| File | Role |
|------|------|
| `index.html` | Single page, orb + settings modal (identical UI to the WebRTC app) |
| `main.js` | State machine, settings, tools, camera, noise-gate UI wiring |
| `ui/chat.js` | `ChatView`: history panel, ephemeral bubbles, transcript/tool streaming |
| `ui/account.js` | `Account`: HF login chip + popover, daily-limit modal |
| `ui/dom.js` | Shared helpers: `$`, `escHtml`, `truncateError`, `DEBUG` |
| `auth.py` | HF OAuth + per-request identity (tier, hashed keys) |
| `limiter.py` | SQLite per-day talk-time budget (chunked server-clock reservation) |
| `ws/s2s-ws-client.js` | WebSocket handshake + OpenAI Realtime GA protocol |
| `ws/codec.js` | base64 <-> PCM helpers + transcript extraction (pure) |
| `ws/orb-visualizer.js` | `OrbVisualiser`: FFT bands -> orb CSS custom properties |
| `worklets/mic-capture.js` | AudioWorklet: 48 kHz Float32 -> 16 kHz Int16 PCM, posts ~40 ms chunks |
| `worklets/audio-playback.js` | AudioWorklet: 24 kHz Float32 ring buffer -> 48 kHz, linear interp, fade in/out |
| `style.css` | Orb animations, layout, dark theme (verbatim from the WebRTC app) |
## Audio pipeline notes
- **Input**: `getUserMedia({ echoCancellation, noiseSuppression, autoGainControl })`
feeds the `mic-capture` worklet at the `AudioContext` rate. The worklet
resamples to 16 kHz (boxcar lowpass + decimation on the 48 -> 16 fast
path, linear interpolation fallback for odd rates) and packs Int16 LE.
- **Output**: `response.output_audio.delta` decodes to Int16 -> Float32
and is posted to the `audio-playback` worklet. The worklet maintains a
per-context ring buffer, linearly interpolates 24 -> 48, and applies
short 32-frame fades on entry/exit to suppress clicks.
- **Barge-in**: when the server VAD detects user speech mid-response
(`input_audio_buffer.speech_started` while `ai-speaking`), the client
posts `{ kind: "clear" }` to the playback worklet to wipe the queue
immediately. The server itself cancels the in-flight response.
## Credits
- Backend: [huggingface/speech-to-speech](https://github.com/huggingface/speech-to-speech)
- UI verbatim from `amir-tfrere/minimal-conversation-app-s2s-backend` (Pollen Robotics × Hugging Face)

229
demo/auth.py Normal file
View File

@ -0,0 +1,229 @@
"""
HF OAuth + per-request identity for the duration limiter.
Login uses Hugging Face's native Spaces OAuth via `huggingface_hub`
(`attach_huggingface_oauth` / `parse_huggingface_oauth`). The OAuth env
(`OAUTH_CLIENT_ID`, ...) is injected by the platform when the Space README sets
`hf_oauth: true`, so this only activates on a deployed Space — locally and in
direct mode there's no OAuth and the limiter treats everyone as anonymous.
Identity:
- signed in -> tier 'pro' | 'free', keyed by hashed HF `sub`
- anonymous -> tier 'anon', keyed by BOTH hashed client IP and a hashed
signed-cookie id (OR-matched in the limiter)
"""
import logging
import os
import secrets
import limiter
logger = logging.getLogger("s2s.auth")
# huggingface_hub adds these routes when OAuth is attached. Centralised so a
# version change is a one-line fix; the paths are handed to the client via
# /api/me rather than hardcoded there.
OAUTH_LOGIN_PATH = "/oauth/huggingface/login"
OAUTH_LOGOUT_PATH = "/oauth/huggingface/logout"
ANON_COOKIE = "s2s_anon"
_COOKIE_MAX_AGE = 60 * 60 * 24 * 30 # 30 days
# Members of these orgs get unlimited usage (like PRO) out of the box. The
# UNLIMITED_ORGS env adds to this set; it doesn't replace it.
_DEFAULT_UNLIMITED_ORGS = {"cerebras", "huggingfacem4", "smolagents", "pollen-robotics"}
def _unlimited_orgs() -> "set[str]":
"""Org usernames whose members get unlimited usage (like PRO).
Defaults to {cerebras, HuggingFaceM4, smolagents}; the UNLIMITED_ORGS env
(comma/space-separated, e.g. `UNLIMITED_ORGS=my-team`) adds more. Matched
case-insensitively against the signed-in user's organisations."""
raw = os.environ.get("UNLIMITED_ORGS", "")
extra = {o.strip().lower() for o in raw.replace(",", " ").split() if o.strip()}
return _DEFAULT_UNLIMITED_ORGS | extra
try:
from huggingface_hub import attach_huggingface_oauth, parse_huggingface_oauth
_OAUTH_IMPORTABLE = True
except Exception as exc: # pragma: no cover - import guard
logger.info("huggingface_hub OAuth unavailable (%s); sign-in disabled.", exc)
_OAUTH_IMPORTABLE = False
# Set by attach(): True once OAuth is actually wired (importable + env present).
oauth_enabled = False
def attach(app) -> bool:
"""Wire HF OAuth onto the app if it's importable and configured. Returns
whether sign-in is available."""
global oauth_enabled
if not _OAUTH_IMPORTABLE or not os.environ.get("OAUTH_CLIENT_ID"):
return False
try:
attach_huggingface_oauth(app)
oauth_enabled = True
logger.info("HF OAuth attached (sign-in enabled).")
except Exception as exc: # pragma: no cover - defensive
logger.warning("Failed to attach HF OAuth: %r", exc)
oauth_enabled = False
return oauth_enabled
def _field(obj, name, default=None):
"""Read a field whether the user-info is an object or a dict."""
if obj is None:
return default
if isinstance(obj, dict):
return obj.get(name, default)
return getattr(obj, name, default)
# Surface what we detect (orgs, tier) in logs and on /api/me when set. Handy for
# verifying org gating on the live Space without guessing.
AUTH_DEBUG = bool(os.environ.get("AUTH_DEBUG"))
# whoami-v2 org lookups are cached for the process lifetime, keyed by token, so
# /api/me + /api/session don't each hit the Hub.
_orgs_cache: "dict[str, set[str]]" = {}
def current_oauth(request):
"""The parsed HF OAuth info (user_info + access_token), or None."""
if not oauth_enabled:
return None
try:
return parse_huggingface_oauth(request)
except Exception:
return None
def current_user(request):
"""The signed-in HF user-info, or None."""
return _field(current_oauth(request), "user_info")
def _user_org_names(user) -> "set[str]":
"""The user's organisations from the OAuth userinfo, by username/name/id."""
names = set()
for org in _field(user, "orgs", []) or []:
for key in ("preferred_username", "name", "sub"):
val = _field(org, key)
if val:
names.add(str(val).lower())
return names
def _orgs_via_token(token: str) -> "set[str]":
"""Fallback org lookup via the Hub `whoami-v2` API, using the user's OAuth
access token. Covers the case where the userinfo claim omits `orgs`."""
if not token:
return set()
if token in _orgs_cache:
return _orgs_cache[token]
names: "set[str]" = set()
try:
import httpx
resp = httpx.get(
"https://huggingface.co/api/whoami-v2",
headers={"Authorization": f"Bearer {token}"},
timeout=5.0,
)
resp.raise_for_status()
for org in resp.json().get("orgs", []) or []:
for key in ("name", "fullname"):
val = org.get(key)
if val:
names.add(str(val).lower())
except Exception as exc: # pragma: no cover - network/permission dependent
logger.info("whoami-v2 org lookup failed: %r", exc)
_orgs_cache[token] = names
return names
def _org_names(user, token=None, allow=None) -> "set[str]":
"""The user's org usernames from the OAuth userinfo claim. If that doesn't
already satisfy `allow`, fall back to the Hub `whoami-v2` API (the claim is
often empty or partial), so membership is resolved either way."""
names = _user_org_names(user)
if token and (allow is None or not (allow & names)):
names = names | _orgs_via_token(token)
return names
def resolve_tier(user, token=None) -> str:
"""Tier for a signed-in user: 'pro' (paying), 'org' (allow-listed org
member, unlimited), or 'free'. PRO wins over org if both apply."""
if bool(_field(user, "is_pro", False)):
return "pro"
allow = _unlimited_orgs()
names = _org_names(user, token, allow)
tier = "org" if (allow & names) else "free"
if AUTH_DEBUG:
logger.info("tier=%s orgs=%s allow=%s", tier, sorted(names), sorted(allow))
return tier
def user_view(request) -> dict:
"""Public profile for /api/me."""
info = current_oauth(request)
user = _field(info, "user_info")
if not user:
return {"loggedIn": False, "tier": "anon"}
token = _field(info, "access_token")
out = {
"loggedIn": True,
"username": _field(user, "preferred_username") or _field(user, "name") or "you",
"avatar": _field(user, "picture"),
"tier": resolve_tier(user, token),
}
if AUTH_DEBUG:
out["orgs"] = sorted(_org_names(user, token))
return out
def _client_ip(request) -> str:
"""Real client IP. On HF the app sits behind a proxy, so the user's address
is the first hop in X-Forwarded-For, not request.client.host."""
xff = request.headers.get("x-forwarded-for", "")
if xff:
return xff.split(",")[0].strip()
return request.client.host if request.client else "unknown"
def resolve_identity(request):
"""Resolve (tier, keys, set_cookie) for this request.
`keys` are the limiter usage_daily keys to debit (one for signed-in, two for
anonymous). `set_cookie` is a signed value to Set-Cookie when we minted a new
anonymous id, else None.
"""
info = current_oauth(request)
user = _field(info, "user_info")
if user:
sub = _field(user, "sub") or _field(user, "preferred_username")
token = _field(info, "access_token")
return resolve_tier(user, token), [limiter.hash_key(f"sub:{sub}")], None
# Anonymous: key by IP and a signed cookie id, minting the cookie if absent.
ip = _client_ip(request)
cookie_id = limiter.verify_cookie(request.cookies.get(ANON_COOKIE, ""))
set_cookie = None
if not cookie_id:
cookie_id = secrets.token_urlsafe(18)
set_cookie = limiter.sign_cookie(cookie_id)
keys = [limiter.hash_key(f"ip:{ip}"), limiter.hash_key(f"cookie:{cookie_id}")]
return "anon", keys, set_cookie
def set_anon_cookie(response, signed: str) -> None:
# The Space runs inside an iframe on huggingface.co, so the cookie lives in a
# cross-site context — it must be SameSite=None; Secure or the browser drops it.
response.set_cookie(
ANON_COOKIE, signed,
max_age=_COOKIE_MAX_AGE, httponly=True, samesite="none", secure=True,
)

29
demo/docs/adr/0001-docker-space-with-search-proxy.md Normal file
View File

@ -0,0 +1,29 @@
# Convert the static Space into a Docker app with a search proxy
To give the model a web search tool, the executor (which runs in the browser of a
public Space) needs a search key. A `sdk: static` Space serves files as-is with no
runtime process, so it cannot hold a secret the browser uses without exposing it.
We convert the Space from `sdk: static` to `sdk: docker`: a single container runs a
small server (FastAPI + uvicorn) that both serves the existing front-end *unchanged*
and exposes a same-origin `/search` proxy holding `SERPER_API_KEY` server-side. The
whole app lives in that one container; the s2s speech-to-speech backend stays the
separate load-balanced service it already is.
## Considered options
- **Stay static, user-supplied key only** — no owner key; search works only if each
user pastes their own. Rejected as the default because it leaves the deployed demo
with no working search.
- **Separate proxy service** — same secrecy, but splits the app across two deploys.
Rejected: the app must live in one container.
- **Key baked into client JS at build time** — would be readable in the served
bundle. Rejected: defeats the point.
## Consequences
- Deployment is no longer static: there is a Dockerfile and a server process; the
README front-matter changes from `sdk: static` to `sdk: docker`.
- The client calls `/search` same-origin; the server reads the key from env. A user
may still supply their own key as a fallback, sent per-request to the proxy.
- The front-end, audio pipeline, and s2s handshake are untouched — only the hosting
shape and the new route are added.

373
demo/index.html Normal file
View File

@ -0,0 +1,373 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
<title>Minimal Conversation · S2S backend (WebSocket)</title>
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link
href="https://fonts.googleapis.com/css2?family=Geist+Mono:wght@400;500;600&family=Inter:wght@400;500;600;700;800&display=swap"
rel="stylesheet"
/>
<link rel="stylesheet" href="style.css" />
</head>
<!--
`booting` disables transitions/animations until the first paint commits.
Without this the orb visibly fades/scales in on the very first frame
because `state-idle` and the generic `.ind` defaults differ. The
bootstrap script at the end of main.js strips the class after one rAF.
-->
<body class="booting">
<div id="app">
<header class="topbar">
<div class="brand">
<div class="ident">
<div class="ident-head">
<a class="ident-title" href="https://github.com/huggingface/speech-to-speech" target="_blank" rel="noopener">Hugging Face Realtime</a>
<button id="about-btn" class="about-btn" title="About this space" aria-label="About this space">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="12" y1="16" x2="12" y2="12"/><line x1="12" y1="8" x2="12.01" y2="8"/></svg>
</button>
</div>
<p class="ident-blurb">An open, real-time voice chat built on Hugging Face's speech-to-speech stack.</p>
<div class="ident-meta">
<span class="ident-row">
<span class="ident-label">Powered by</span>
<a href="https://huggingface.co/inference-endpoints" target="_blank" rel="noopener">Inference Endpoints</a>
<span class="sep" aria-hidden="true">·</span>
<a class="cerebras-credit" href="https://cerebras.ai" target="_blank" rel="noopener"><svg class="cerebras-mark" viewBox="0 0 50 50" aria-hidden="true"><path fill-rule="evenodd" clip-rule="evenodd" d="M29.4186 5.62713C24.2806 5.62713 19.353 7.6682 15.7199 11.3013C12.0868 14.9345 10.0457 19.862 10.0457 25C10.0457 30.1381 12.0868 35.0656 15.7199 38.6988C19.353 42.3319 24.2806 44.373 29.4186 44.373V47.2917C17.1061 47.2917 7.12695 37.3105 7.12695 24.998C7.12695 12.6855 17.104 2.7063 29.4165 2.7063V5.62505L29.4186 5.62713ZM39.3186 13.2875C37.7794 11.9809 35.9971 10.9915 34.0741 10.3761C32.1512 9.76067 30.1256 9.53147 28.1137 9.70166C26.1019 9.87185 24.1435 10.4381 22.3513 11.3677C20.559 12.2974 18.9683 13.5722 17.6704 15.1189C16.3726 16.6655 15.3933 18.4534 14.7888 20.3798C14.1844 22.3062 13.9667 24.3332 14.1483 26.344C14.33 28.3548 14.9073 30.3099 15.8472 32.0968C16.787 33.8838 18.0709 35.4673 19.6249 36.7563L17.7478 38.9938C15.9126 37.4545 14.3987 35.5687 13.2924 33.4442C12.1862 31.3197 11.5092 28.9981 11.3003 26.6119C11.0914 24.2258 11.3545 21.8219 12.0747 19.5374C12.7949 17.253 13.9581 15.1328 15.4978 13.298C17.0373 11.4628 18.9233 9.94877 21.0479 8.84247C23.1726 7.73618 25.4944 7.05923 27.8807 6.8503C30.267 6.64136 32.6711 6.90453 34.9557 7.62477C37.2403 8.34501 39.3607 9.50822 41.1957 11.048L39.3186 13.2875ZM34.6207 15.0459C31.9818 13.6829 28.9113 13.4172 26.0776 14.3068C23.2438 15.1964 20.876 17.1691 19.4895 19.7958C18.1029 22.4224 17.8099 25.4903 18.6741 28.332C19.5384 31.1736 21.4899 33.5589 24.104 34.9688L22.7374 37.5521C19.4721 35.7617 17.0418 32.7592 15.9707 29.1927C14.8996 25.6261 15.2737 21.7815 17.0123 18.4883C18.7509 15.1952 21.7146 12.7176 25.2637 11.5903C28.8129 10.463 32.663 10.7763 35.9832 12.4625L34.6207 15.0459ZM29.4165 17.7896C27.5042 17.7896 25.6702 18.5493 24.318 19.9015C22.9658 21.2537 22.2061 23.0877 22.2061 25C22.2061 26.9124 22.9658 28.7464 24.318 30.0986C25.6702 31.4508 27.5042 32.2105 29.4165 32.2105V35.1313C26.7296 35.1313 24.1526 34.0639 22.2527 32.1639C20.3527 30.2639 19.2853 27.687 19.2853 25C19.2853 22.3131 20.3527 19.7362 22.2527 17.8362C24.1526 15.9362 26.7296 14.8688 29.4165 14.8688V17.7896Z" fill="#F15A29"/><path d="M32.0981 22.5749C31.7875 22.2404 31.4295 21.9534 31.0356 21.7228C30.6925 21.519 30.3014 21.4097 29.9023 21.4062C29.371 21.4062 28.896 21.5041 28.4773 21.6978C28.0725 21.8802 27.7089 22.1425 27.4081 22.469C27.1074 22.7955 26.8758 23.1795 26.7273 23.5978C26.5731 24.0207 26.4981 24.4645 26.4981 24.9124C26.4981 25.3666 26.5731 25.8082 26.7273 26.227C26.877 26.6449 27.1091 27.0286 27.4096 27.3553C27.7102 27.682 28.0733 27.9451 28.4773 28.1291C28.8939 28.3228 29.3731 28.4207 29.9023 28.4207C30.3523 28.4207 30.771 28.3249 31.1564 28.1395C31.5481 27.952 31.8856 27.6707 32.146 27.3228L34.0794 29.4187C33.7877 29.7103 33.4544 29.9624 33.0752 30.1749C32.374 30.566 31.6108 30.8338 30.8189 30.9666C30.4648 31.0207 30.1585 31.0499 29.9023 31.0499C29.0617 31.0552 28.2271 30.9069 27.4398 30.6124C26.6954 30.3371 26.0144 29.914 25.4377 29.3687C24.8643 28.8208 24.4078 28.1624 24.096 27.4332C23.7572 26.6366 23.5883 25.778 23.6002 24.9124C23.6002 23.9874 23.7669 23.1478 24.096 22.3916C24.4085 21.6624 24.8627 21.0041 25.4356 20.4562C26.0148 19.9124 26.696 19.4895 27.4398 19.2145C28.2271 18.92 29.0617 18.7717 29.9023 18.777C30.6419 18.777 31.3856 18.9187 32.1356 19.202C32.8877 19.4895 33.5627 19.952 34.1023 20.5541L32.0981 22.5749Z" fill="#fff"/></svg>Cerebras</a>
</span>
<span class="ident-row">
<span class="ident-label">Built by</span>
<a class="hf-credit" href="https://huggingface.co" target="_blank" rel="noopener"><svg class="hf-mark" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M12.025 1.13c-5.77 0-10.449 4.647-10.449 10.378 0 1.112.178 2.181.503 3.185.064-.222.203-.444.416-.577a.96.96 0 0 1 .524-.15c.293 0 .584.124.84.284.278.173.48.408.71.694.226.282.458.611.684.951v-.014c.017-.324.106-.622.264-.874s.403-.487.762-.543c.3-.047.596.06.787.203s.31.313.4.467c.15.257.212.468.233.542.01.026.653 1.552 1.657 2.54.616.605 1.01 1.223 1.082 1.912.055.537-.096 1.059-.38 1.572.637.121 1.294.187 1.967.187.657 0 1.298-.063 1.921-.178-.287-.517-.44-1.041-.384-1.581.07-.69.465-1.307 1.081-1.913 1.004-.987 1.647-2.513 1.657-2.539.021-.074.083-.285.233-.542.09-.154.208-.323.4-.467a1.08 1.08 0 0 1 .787-.203c.359.056.604.29.762.543s.247.55.265.874v.015c.225-.34.457-.67.683-.952.23-.286.432-.52.71-.694.257-.16.547-.284.84-.285a.97.97 0 0 1 .524.151c.228.143.373.388.43.625l.006.04a10.3 10.3 0 0 0 .534-3.273c0-5.731-4.678-10.378-10.449-10.378M8.327 6.583a1.5 1.5 0 0 1 .713.174 1.487 1.487 0 0 1 .617 2.013c-.183.343-.762-.214-1.102-.094-.38.134-.532.914-.917.71a1.487 1.487 0 0 1 .69-2.803m7.486 0a1.487 1.487 0 0 1 .689 2.803c-.385.204-.536-.576-.916-.71-.34-.12-.92.437-1.103.094a1.487 1.487 0 0 1 .617-2.013 1.5 1.5 0 0 1 .713-.174m-10.68 1.55a.96.96 0 1 1 0 1.921.96.96 0 0 1 0-1.92m13.838 0a.96.96 0 1 1 0 1.92.96.96 0 0 1 0-1.92M8.489 11.458c.588.01 1.965 1.157 3.572 1.164 1.607-.007 2.984-1.155 3.572-1.164.196-.003.305.12.305.454 0 .886-.424 2.328-1.563 3.202-.22-.756-1.396-1.366-1.63-1.32q-.011.001-.02.006l-.044.026-.01.008-.03.024q-.018.017-.035.036l-.032.04a1 1 0 0 0-.058.09l-.014.025q-.049.088-.11.19a1 1 0 0 1-.083.116 1.2 1.2 0 0 1-.173.18q-.035.029-.075.058a1.3 1.3 0 0 1-.251-.243 1 1 0 0 1-.076-.107c-.124-.193-.177-.363-.337-.444-.034-.016-.104-.008-.2.022q-.094.03-.216.087-.06.028-.125.063l-.13.074q-.067.04-.136.086a3 3 0 0 0-.135.096 3 3 0 0 0-.26.219 2 2 0 0 0-.12.121 2 2 0 0 0-.106.128l-.002.002a2 2 0 0 0-.09.132l-.001.001a1.2 1.2 0 0 0-.105.212q-.013.036-.024.073c-1.139-.875-1.563-2.317-1.563-3.203 0-.334.109-.457.305-.454m.836 10.354c.824-1.19.766-2.082-.365-3.194-1.13-1.112-1.789-2.738-1.789-2.738s-.246-.945-.806-.858-.97 1.499.202 2.362c1.173.864-.233 1.45-.685.64-.45-.812-1.683-2.896-2.322-3.295s-1.089-.175-.938.647 2.822 2.813 2.562 3.244-1.176-.506-1.176-.506-2.866-2.567-3.49-1.898.473 1.23 2.037 2.16c1.564.932 1.686 1.178 1.464 1.53s-3.675-2.511-4-1.297c-.323 1.214 3.524 1.567 3.287 2.405-.238.839-2.71-1.587-3.216-.642-.506.946 3.49 2.056 3.522 2.064 1.29.33 4.568 1.028 5.713-.624m5.349 0c-.824-1.19-.766-2.082.365-3.194 1.13-1.112 1.789-2.738 1.789-2.738s.246-.945.806-.858.97 1.499-.202 2.362c-1.173.864.233 1.45.685.64.451-.812 1.683-2.896 2.322-3.295s1.089-.175.938.647-2.822 2.813-2.562 3.244 1.176-.506 1.176-.506 2.866-2.567 3.49-1.898-.473 1.23-2.037 2.16c-1.564.932-1.686 1.178-1.464 1.53s3.675-2.511 4-1.297c.323 1.214-3.524 1.567-3.287 2.405.238.839 2.71-1.587 3.216-.642.506.946-3.49 2.056-3.522 2.064-1.29.33-4.568 1.028-5.713-.624"/></svg>Hugging Face</a>
<span class="sep" aria-hidden="true">·</span>
<a class="handle" href="https://huggingface.co/tfrere" target="_blank" rel="noopener">tfrere</a>
<span class="sep" aria-hidden="true">·</span>
<a class="handle" href="https://huggingface.co/A-Mahla" target="_blank" rel="noopener">A-Mahla</a>
<span class="sep" aria-hidden="true">·</span>
<a class="handle" href="https://huggingface.co/andito" target="_blank" rel="noopener">andito</a>
</span>
</div>
</div>
</div>
<div class="topbar-right">
<!-- HF login chip / sign-in pill. Populated by ui/account.js; only
shown when the deploy runs behind a load balancer with OAuth. -->
<div id="account" class="account" hidden></div>
<button id="about-btn-m" class="icon-btn about-btn-mobile" title="About this space" aria-label="About this space">
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="12" y1="16" x2="12" y2="12"/><line x1="12" y1="8" x2="12.01" y2="8"/></svg>
</button>
<button id="tools-btn" class="icon-btn" title="Tools" aria-label="Tools">
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M14.7 6.3a1 1 0 0 0 0 1.4l1.6 1.6a1 1 0 0 0 1.4 0l3.77-3.77a6 6 0 0 1-7.94 7.94l-6.91 6.91a2.12 2.12 0 0 1-3-3l6.91-6.91a6 6 0 0 1 7.94-7.94l-3.76 3.76z"/></svg>
</button>
<button id="chat-btn" class="icon-btn" title="Conversation history" aria-label="Conversation history">
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg>
<span id="chat-badge" class="chat-badge" aria-hidden="true"></span>
</button>
<button id="settings-btn" class="icon-btn" title="Settings" aria-label="Settings">
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="3"/><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 1 1-2.83 2.83l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 1 1-4 0v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 1 1-2.83-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 1 1 0-4h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 1 1 2.83-2.83l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 1 1 4 0v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 1 1 2.83 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9c0 .66.39 1.25 1 1.51H21a2 2 0 1 1 0 4h-.09a1.65 1.65 0 0 0-1.51 1z"/></svg>
</button>
</div>
</header>
<main class="stage">
<div class="orb-wrap">
<div id="mic-gate" class="mic-gate">
<svg id="mic-gate-arc" class="mic-gate-arc" viewBox="0 0 100 100" aria-hidden="true">
<path id="mga-track" class="mga-track" fill="none" />
<path id="mga-fill" class="mga-fill" fill="none" />
<path id="mga-hit" class="mga-hit" fill="none" />
<circle id="mga-handle" class="mga-handle" r="3" />
</svg>
<button id="mic-btn" class="side-btn" type="button" aria-label="Mute" title="Mute" aria-hidden="true">
<svg class="mic-on" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="2" width="6" height="12" rx="3"/><path d="M5 10a7 7 0 0 0 14 0"/><line x1="12" y1="19" x2="12" y2="22"/></svg>
<svg class="mic-off" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round"><line x1="2" y1="2" x2="22" y2="22"/><path d="M9 5a3 3 0 0 1 6 0v4"/><path d="M9 10v1a3 3 0 0 0 5.1 2.1"/><path d="M19 10a7 7 0 0 1-1.24 3.97"/><path d="M5 10a7 7 0 0 0 11 5.67"/><line x1="12" y1="19" x2="12" y2="22"/></svg>
</button>
</div>
<button
id="main-circle"
class="circle state-idle"
type="button"
aria-label="Start voice conversation"
>
<span class="circle-glow" aria-hidden="true"></span>
<span class="circle-ring" aria-hidden="true"></span>
<span class="circle-ring-outer" aria-hidden="true"></span>
<span class="circle-core">
<span class="circle-indicator" aria-hidden="true">
<svg class="ind ind-mic" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
<rect x="9" y="2" width="6" height="12" rx="3" fill="currentColor" stroke="none"/>
<path d="M5 10a7 7 0 0 0 14 0"/>
<line x1="12" y1="19" x2="12" y2="22"/>
<line x1="8" y1="22" x2="16" y2="22"/>
</svg>
<svg class="ind ind-error" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="9"/><line x1="12" y1="8" x2="12" y2="13"/><line x1="12" y1="16" x2="12" y2="16"/></svg>
<span class="ind ind-spinner"></span>
<span class="ind ind-thinking">
<span class="dot"></span>
<span class="dot"></span>
<span class="dot"></span>
</span>
<span class="ind ind-bars">
<span class="bar"></span>
<span class="bar"></span>
<span class="bar"></span>
<span class="bar"></span>
<span class="bar"></span>
</span>
<svg class="ind ind-voice" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
<path d="M3 10v4a1 1 0 0 0 1 1h3l5 4V5L7 9H4a1 1 0 0 0-1 1z" fill="currentColor" stroke="none"/>
<path class="wave wave-1" d="M16 8a5 5 0 0 1 0 8"/>
<path class="wave wave-2" d="M19 5a9 9 0 0 1 0 14"/>
</svg>
</span>
</span>
</button>
<button id="stop-btn" class="side-btn" type="button" aria-label="End" title="End" aria-hidden="true">
<svg viewBox="0 0 24 24" fill="currentColor"><rect x="6" y="6" width="12" height="12" rx="2"/></svg>
</button>
</div>
<p id="circle-caption" class="circle-caption" role="status">Tap to start</p>
<p id="circle-subcaption" class="circle-subcaption" hidden></p>
<div id="queue-actions" class="queue-actions" hidden>
<button id="join-queue-btn" class="join-queue-btn" type="button" hidden>
Join now
</button>
<button id="leave-queue-btn" class="leave-queue-btn" type="button" hidden>
Leave queue
</button>
</div>
</main>
<footer class="footer">
<span>
Powered by
<a href="https://github.com/huggingface/speech-to-speech" target="_blank" rel="noopener">huggingface/speech-to-speech</a>
</span>
</footer>
</div>
<!-- Ephemeral chat bubbles anchored to the top-right -->
<div id="bubble-stack" class="bubble-stack" aria-live="polite" aria-atomic="false"></div>
<!-- Webcam self-view, shown bottom-left while the camera tool is enabled -->
<div id="cam-pip" class="cam-pip" aria-hidden="true">
<video id="cam-video" class="cam-video" autoplay playsinline muted></video>
<span class="cam-flash" aria-hidden="true"></span>
<span class="cam-label">camera</span>
</div>
<!-- Conversation history panel (overlay) -->
<div id="chat-panel" class="chat-panel">
<div id="chat-panel-backdrop" class="chat-panel-backdrop"></div>
<div class="chat-panel-inner">
<header class="chat-panel-header">
<h3>Conversation</h3>
<button id="chat-panel-close" class="icon-btn" aria-label="Close">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
</button>
</header>
<div id="chat-history" class="chat-history"></div>
</div>
</div>
<dialog id="about-modal" class="modal about-modal">
<div class="modal-content">
<header class="modal-header">
<h2>About</h2>
<button id="about-close" class="icon-btn" aria-label="Close">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
</button>
</header>
<!-- General intro to the speech-to-speech project -->
<div class="about-intro">
<p>Speech-to-speech is Hugging Face's open framework for real-time voice agents. Rather than one end-to-end model, it chains four open models from the Hub (speech detection, transcription, a vision-language model, and synthesis), so any stage can be swapped or run locally. This demo wires that pipeline to hosted inference.</p>
<a class="about-repo" href="https://github.com/huggingface/speech-to-speech" target="_blank" rel="noopener">View the project on GitHub<svg class="ext" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M7 17 17 7"/><path d="M8 7h9v9"/></svg></a>
</div>
<!-- The backend, shown as the path a turn actually travels -->
<div class="about-pipeline">
<p class="pipeline-title">The pipeline</p>
<ol class="pipeline">
<li class="pipe-endpoint">You speak</li>
<li class="pipe-stage">
<span class="pipe-tag">VAD</span>
<span class="pipe-job">detects speech</span>
<a class="pipe-model" href="https://github.com/snakers4/silero-vad" target="_blank" rel="noopener">silero-vad<svg class="ext" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M7 17 17 7"/><path d="M8 7h9v9"/></svg></a>
</li>
<li class="pipe-stage">
<span class="pipe-tag">STT</span>
<span class="pipe-job">transcribes it</span>
<a class="pipe-model" href="https://huggingface.co/nvidia/parakeet-tdt-1.1b" target="_blank" rel="noopener">nvidia/parakeet-tdt-1.1b<svg class="ext" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M7 17 17 7"/><path d="M8 7h9v9"/></svg></a>
</li>
<li class="pipe-stage">
<span class="pipe-tag">VLM</span>
<span class="pipe-job">analyses and composes the reply <span class="pipe-note">· via <a class="cerebras-credit" href="https://cerebras.ai" target="_blank" rel="noopener"><svg class="cerebras-mark" viewBox="0 0 50 50" aria-hidden="true"><path fill-rule="evenodd" clip-rule="evenodd" d="M29.4186 5.62713C24.2806 5.62713 19.353 7.6682 15.7199 11.3013C12.0868 14.9345 10.0457 19.862 10.0457 25C10.0457 30.1381 12.0868 35.0656 15.7199 38.6988C19.353 42.3319 24.2806 44.373 29.4186 44.373V47.2917C17.1061 47.2917 7.12695 37.3105 7.12695 24.998C7.12695 12.6855 17.104 2.7063 29.4165 2.7063V5.62505L29.4186 5.62713ZM39.3186 13.2875C37.7794 11.9809 35.9971 10.9915 34.0741 10.3761C32.1512 9.76067 30.1256 9.53147 28.1137 9.70166C26.1019 9.87185 24.1435 10.4381 22.3513 11.3677C20.559 12.2974 18.9683 13.5722 17.6704 15.1189C16.3726 16.6655 15.3933 18.4534 14.7888 20.3798C14.1844 22.3062 13.9667 24.3332 14.1483 26.344C14.33 28.3548 14.9073 30.3099 15.8472 32.0968C16.787 33.8838 18.0709 35.4673 19.6249 36.7563L17.7478 38.9938C15.9126 37.4545 14.3987 35.5687 13.2924 33.4442C12.1862 31.3197 11.5092 28.9981 11.3003 26.6119C11.0914 24.2258 11.3545 21.8219 12.0747 19.5374C12.7949 17.253 13.9581 15.1328 15.4978 13.298C17.0373 11.4628 18.9233 9.94877 21.0479 8.84247C23.1726 7.73618 25.4944 7.05923 27.8807 6.8503C30.267 6.64136 32.6711 6.90453 34.9557 7.62477C37.2403 8.34501 39.3607 9.50822 41.1957 11.048L39.3186 13.2875ZM34.6207 15.0459C31.9818 13.6829 28.9113 13.4172 26.0776 14.3068C23.2438 15.1964 20.876 17.1691 19.4895 19.7958C18.1029 22.4224 17.8099 25.4903 18.6741 28.332C19.5384 31.1736 21.4899 33.5589 24.104 34.9688L22.7374 37.5521C19.4721 35.7617 17.0418 32.7592 15.9707 29.1927C14.8996 25.6261 15.2737 21.7815 17.0123 18.4883C18.7509 15.1952 21.7146 12.7176 25.2637 11.5903C28.8129 10.463 32.663 10.7763 35.9832 12.4625L34.6207 15.0459ZM29.4165 17.7896C27.5042 17.7896 25.6702 18.5493 24.318 19.9015C22.9658 21.2537 22.2061 23.0877 22.2061 25C22.2061 26.9124 22.9658 28.7464 24.318 30.0986C25.6702 31.4508 27.5042 32.2105 29.4165 32.2105V35.1313C26.7296 35.1313 24.1526 34.0639 22.2527 32.1639C20.3527 30.2639 19.2853 27.687 19.2853 25C19.2853 22.3131 20.3527 19.7362 22.2527 17.8362C24.1526 15.9362 26.7296 14.8688 29.4165 14.8688V17.7896Z" fill="#F15A29"/><path d="M32.0981 22.5749C31.7875 22.2404 31.4295 21.9534 31.0356 21.7228C30.6925 21.519 30.3014 21.4097 29.9023 21.4062C29.371 21.4062 28.896 21.5041 28.4773 21.6978C28.0725 21.8802 27.7089 22.1425 27.4081 22.469C27.1074 22.7955 26.8758 23.1795 26.7273 23.5978C26.5731 24.0207 26.4981 24.4645 26.4981 24.9124C26.4981 25.3666 26.5731 25.8082 26.7273 26.227C26.877 26.6449 27.1091 27.0286 27.4096 27.3553C27.7102 27.682 28.0733 27.9451 28.4773 28.1291C28.8939 28.3228 29.3731 28.4207 29.9023 28.4207C30.3523 28.4207 30.771 28.3249 31.1564 28.1395C31.5481 27.952 31.8856 27.6707 32.146 27.3228L34.0794 29.4187C33.7877 29.7103 33.4544 29.9624 33.0752 30.1749C32.374 30.566 31.6108 30.8338 30.8189 30.9666C30.4648 31.0207 30.1585 31.0499 29.9023 31.0499C29.0617 31.0552 28.2271 30.9069 27.4398 30.6124C26.6954 30.3371 26.0144 29.914 25.4377 29.3687C24.8643 28.8208 24.4078 28.1624 24.096 27.4332C23.7572 26.6366 23.5883 25.778 23.6002 24.9124C23.6002 23.9874 23.7669 23.1478 24.096 22.3916C24.4085 21.6624 24.8627 21.0041 25.4356 20.4562C26.0148 19.9124 26.696 19.4895 27.4398 19.2145C28.2271 18.92 29.0617 18.7717 29.9023 18.777C30.6419 18.777 31.3856 18.9187 32.1356 19.202C32.8877 19.4895 33.5627 19.952 34.1023 20.5541L32.0981 22.5749Z" fill="#fff"/></svg>Cerebras</a></span></span>
<a class="pipe-model" href="https://huggingface.co/google/gemma-4-31B-it" target="_blank" rel="noopener">google/gemma-4-31B-it<svg class="ext" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M7 17 17 7"/><path d="M8 7h9v9"/></svg></a>
</li>
<li class="pipe-stage">
<span class="pipe-tag">TTS</span>
<span class="pipe-job">speaks it back</span>
<a class="pipe-model" href="https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" target="_blank" rel="noopener">Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice<svg class="ext" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M7 17 17 7"/><path d="M8 7h9v9"/></svg></a>
</li>
<li class="pipe-endpoint">The orb replies</li>
</ol>
</div>
</div>
</dialog>
<dialog id="settings-modal" class="modal">
<form method="dialog" class="modal-content">
<header class="modal-header">
<h2>Settings</h2>
<button class="icon-btn" value="close" aria-label="Close">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
</button>
</header>
<div class="tab-panels">
<section class="tab-panel active" role="tabpanel">
<label class="field" id="conn-field">
<span id="conn-label">Speech-to-speech server URL</span>
<input id="lb-url" type="text" autocomplete="off" spellcheck="false" placeholder="http://localhost:port" />
<small id="conn-hint">
URL of your speech-to-speech server, e.g.
<code>http://localhost:8080</code> (the app adds <code>/v1/realtime</code>).
</small>
</label>
<div class="field-row">
<label class="field">
<span>Voice</span>
<select id="voice">
<option value="Aiden" selected>Aiden</option>
<option value="Ryan">Ryan</option>
<option value="Dylan">Dylan</option>
<option value="Eric">Eric</option>
<option value="Ono_Anna">Ono_Anna</option>
<option value="Serena">Serena</option>
<option value="Sohee">Sohee</option>
<option value="Uncle_Fu">Uncle_Fu</option>
<option value="Vivian">Vivian</option>
</select>
</label>
</div>
<div class="field">
<span class="field-head">
Noise gate
<span id="gate-value" class="field-value">Off</span>
</span>
<div class="gate">
<div class="gate-track" aria-hidden="true">
<div id="gate-meter-fill" class="gate-meter-fill"></div>
<input
id="noise-gate"
type="range"
min="-66"
max="-3"
step="1"
value="-50"
aria-label="Noise gate threshold"
/>
</div>
<div class="gate-ends">
<span>Off</span>
<span>−3 dB</span>
</div>
</div>
<small>
Mutes the mic below the handle so room noise isn't sent. Slide fully
left to turn it off; the bar shows your live input while in a call.
</small>
</div>
<label class="field">
<span>Instructions</span>
<textarea id="instructions" rows="5" placeholder="You are a friendly voice assistant..."></textarea>
</label>
<div class="field">
<button id="restart-conversation" type="button" class="btn primary wide" disabled>
Restart conversation with these settings
</button>
<small id="restart-hint">Connect first, then come back here to apply live changes.</small>
</div>
</section>
</div>
<footer class="modal-footer">
<button id="settings-save" type="submit" class="btn primary" value="save">Save</button>
</footer>
</form>
</dialog>
<dialog id="tools-modal" class="modal">
<div class="modal-content">
<header class="modal-header">
<h2>Tools</h2>
<button id="tools-close" class="icon-btn" aria-label="Close">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
</button>
</header>
<p class="tools-intro">Let the assistant act during the conversation. Changes apply live.</p>
<div class="tool-list">
<div class="tool-row" id="tool-web-row">
<div class="tool-info">
<span class="tool-name">Web search</span>
<span class="tool-desc">Look things up on Google, via Serper.</span>
</div>
<label class="switch">
<input id="tool-web" type="checkbox" />
<span class="switch-track" aria-hidden="true"></span>
</label>
</div>
<label class="field tools-key">
<span>Search API key</span>
<input id="search-key" type="password" autocomplete="off" spellcheck="false" />
<small id="tool-web-hint"></small>
</label>
<div class="tool-row tool-row-sep" id="tool-cam-row">
<div class="tool-info">
<span class="tool-name">Camera</span>
<span class="tool-desc">Let the assistant see through your webcam.</span>
</div>
<label class="switch">
<input id="tool-cam" type="checkbox" />
<span class="switch-track" aria-hidden="true"></span>
</label>
</div>
<small id="tool-cam-hint" class="tool-hint">A live preview shows bottom-left while the camera is on.</small>
</div>
</div>
</dialog>
<!-- Shown when the per-day conversation budget is spent (mid-call or at
start). Title / message / CTA / note are filled per tier by
ui/account.js. The hero is the HF mark — a friendly smiling face. -->
<dialog id="limit-modal" class="modal limit-modal">
<div class="modal-content limit-card">
<button id="limit-close" class="icon-btn limit-close" aria-label="Close">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
</button>
<div class="limit-badge" aria-hidden="true">
<svg class="hf-logo" viewBox="0 0 95 88" fill="none" aria-hidden="true"><path fill="#FFD21E" d="M47.21 76.5a34.75 34.75 0 1 0 0-69.5 34.75 34.75 0 0 0 0 69.5Z"/><path fill="#FF9D0B" d="M81.96 41.75a34.75 34.75 0 1 0-69.5 0 34.75 34.75 0 0 0 69.5 0Zm-73.5 0a38.75 38.75 0 1 1 77.5 0 38.75 38.75 0 0 1-77.5 0Z"/><path fill="#3A3B45" d="M58.5 32.3c1.28.44 1.78 3.06 3.07 2.38a5 5 0 1 0-6.76-2.07c.61 1.15 2.55-.72 3.7-.32ZM34.95 32.3c-1.28.44-1.79 3.06-3.07 2.38a5 5 0 1 1 6.76-2.07c-.61 1.15-2.56-.72-3.7-.32Z"/><path fill="#FF323D" d="M46.96 56.29c9.83 0 13-8.76 13-13.26 0-2.34-1.57-1.6-4.09-.36-2.33 1.15-5.46 2.74-8.9 2.74-7.19 0-13-6.88-13-2.38s3.16 13.26 13 13.26Z"/><path fill="#3A3B45" fill-rule="evenodd" d="M39.43 54a8.7 8.7 0 0 1 5.3-4.49c.4-.12.81.57 1.24 1.28.4.68.82 1.37 1.24 1.37.45 0 .9-.68 1.33-1.35.45-.7.89-1.38 1.32-1.25a8.61 8.61 0 0 1 5 4.17c3.73-2.94 5.1-7.74 5.1-10.7 0-2.34-1.57-1.6-4.09-.36l-.14.07c-2.31 1.15-5.39 2.67-8.77 2.67s-6.45-1.52-8.77-2.67c-2.6-1.29-4.23-2.1-4.23.29 0 3.05 1.46 8.06 5.47 10.97Z" clip-rule="evenodd"/><path fill="#FF9D0B" d="M70.71 37a3.25 3.25 0 1 0 0-6.5 3.25 3.25 0 0 0 0 6.5ZM24.21 37a3.25 3.25 0 1 0 0-6.5 3.25 3.25 0 0 0 0 6.5ZM17.52 48c-1.62 0-3.06.66-4.07 1.87a5.97 5.97 0 0 0-1.33 3.76 7.1 7.1 0 0 0-1.94-.3c-1.55 0-2.95.59-3.94 1.66a5.8 5.8 0 0 0-.8 7 5.3 5.3 0 0 0-1.79 2.82c-.24.9-.48 2.8.8 4.74a5.22 5.22 0 0 0-.37 5.02c1.02 2.32 3.57 4.14 8.52 6.1 3.07 1.22 5.89 2 5.91 2.01a44.33 44.33 0 0 0 10.93 1.6c5.86 0 10.05-1.8 12.46-5.34 3.88-5.69 3.33-10.9-1.7-15.92-2.77-2.78-4.62-6.87-5-7.77-.78-2.66-2.84-5.62-6.25-5.62a5.7 5.7 0 0 0-4.6 2.46c-1-1.26-1.98-2.25-2.86-2.82A7.4 7.4 0 0 0 17.52 48Zm0 4c.51 0 1.14.22 1.82.65 2.14 1.36 6.25 8.43 7.76 11.18.5.92 1.37 1.31 2.14 1.31 1.55 0 2.75-1.53.15-3.48-3.92-2.93-2.55-7.72-.68-8.01.08-.02.17-.02.24-.02 1.7 0 2.45 2.93 2.45 2.93s2.2 5.52 5.98 9.3c3.77 3.77 3.97 6.8 1.22 10.83-1.88 2.75-5.47 3.58-9.16 3.58-3.81 0-7.73-.9-9.92-1.46-.11-.03-13.45-3.8-11.76-7 .28-.54.75-.76 1.34-.76 2.38 0 6.7 3.54 8.57 3.54.41 0 .7-.17.83-.6.79-2.85-12.06-4.05-10.98-8.17.2-.73.71-1.02 1.44-1.02 3.14 0 10.2 5.53 11.68 5.53.11 0 .2-.03.24-.1.74-1.2.33-2.04-4.9-5.2-5.21-3.16-8.88-5.06-6.8-7.33.24-.26.58-.38 1-.38 3.17 0 10.66 6.82 10.66 6.82s2.02 2.1 3.25 2.1c.28 0 .52-.1.68-.38.86-1.46-8.06-8.22-8.56-11.01-.34-1.9.24-2.85 1.31-2.85Z"/><path fill="#FFD21E" d="M38.6 76.69c2.75-4.04 2.55-7.07-1.22-10.84-3.78-3.77-5.98-9.3-5.98-9.3s-.82-3.2-2.69-2.9c-1.87.3-3.24 5.08.68 8.01 3.91 2.93-.78 4.92-2.29 2.17-1.5-2.75-5.62-9.82-7.76-11.18-2.13-1.35-3.63-.6-3.13 2.2.5 2.79 9.43 9.55 8.56 11-.87 1.47-3.93-1.71-3.93-1.71s-9.57-8.71-11.66-6.44c-2.08 2.27 1.59 4.17 6.8 7.33 5.23 3.16 5.64 4 4.9 5.2-.75 1.2-12.28-8.53-13.36-4.4-1.08 4.11 11.77 5.3 10.98 8.15-.8 2.85-9.06-5.38-10.74-2.18-1.7 3.21 11.65 6.98 11.76 7.01 4.3 1.12 15.25 3.49 19.08-2.12Z"/><path fill="#FF9D0B" d="M77.4 48c1.62 0 3.07.66 4.07 1.87a5.97 5.97 0 0 1 1.33 3.76 7.1 7.1 0 0 1 1.95-.3c1.55 0 2.95.59 3.94 1.66a5.8 5.8 0 0 1 .8 7 5.3 5.3 0 0 1 1.78 2.82c.24.9.48 2.8-.8 4.74a5.22 5.22 0 0 1 .37 5.02c-1.02 2.32-3.57 4.14-8.51 6.1-3.08 1.22-5.9 2-5.92 2.01a44.33 44.33 0 0 1-10.93 1.6c-5.86 0-10.05-1.8-12.46-5.34-3.88-5.69-3.33-10.9 1.7-15.92 2.78-2.78 4.63-6.87 5.01-7.77.78-2.66 2.83-5.62 6.24-5.62a5.7 5.7 0 0 1 4.6 2.46c1-1.26 1.98-2.25 2.87-2.82A7.4 7.4 0 0 1 77.4 48Zm0 4c-.51 0-1.13.22-1.82.65-2.13 1.36-6.25 8.43-7.76 11.18a2.43 2.43 0 0 1-2.14 1.31c-1.54 0-2.75-1.53-.14-3.48 3.91-2.93 2.54-7.72.67-8.01a1.54 1.54 0 0 0-.24-.02c-1.7 0-2.45 2.93-2.45 2.93s-2.2 5.52-5.97 9.3c-3.78 3.77-3.98 6.8-1.22 10.83 1.87 2.75 5.47 3.58 9.15 3.58 3.82 0 7.73-.9 9.93-1.46.1-.03 13.45-3.8 11.76-7-.29-.54-.75-.76-1.34-.76-2.38 0-6.71 3.54-8.57 3.54-.42 0-.71-.17-.83-.6-.8-2.85 12.05-4.05 10.97-8.17-.19-.73-.7-1.02-1.44-1.02-3.14 0-10.2 5.53-11.68 5.53-.1 0-.19-.03-.23-.1-.74-1.2-.34-2.04 4.88-5.2 5.23-3.16 8.9-5.06 6.8-7.33-.23-.26-.57-.38-.98-.38-3.18 0-10.67 6.82-10.67 6.82s-2.02 2.1-3.24 2.1a.74.74 0 0 1-.68-.38c-.87-1.46 8.05-8.22 8.55-11.01.34-1.9-.24-2.85-1.31-2.85Z"/><path fill="#FFD21E" d="M56.33 76.69c-2.75-4.04-2.56-7.07 1.22-10.84 3.77-3.77 5.97-9.3 5.97-9.3s.82-3.2 2.7-2.9c1.86.3 3.23 5.08-.68 8.01-3.92 2.93.78 4.92 2.28 2.17 1.51-2.75 5.63-9.82 7.76-11.18 2.13-1.35 3.64-.6 3.13 2.2-.5 2.79-9.42 9.55-8.55 11 .86 1.47 3.92-1.71 3.92-1.71s9.58-8.71 11.66-6.44c2.08 2.27-1.58 4.17-6.8 7.33-5.23 3.16-5.63 4-4.9 5.2.75 1.2 12.28-8.53 13.36-4.4 1.08 4.11-11.76 5.3-10.97 8.15.8 2.85 9.05-5.38 10.74-2.18 1.69 3.21-11.65 6.98-11.76 7.01-4.31 1.12-15.26 3.49-19.08-2.12Z"/></svg>
</div>
<h2 id="limit-title" class="limit-title">That's a wrap for now</h2>
<p id="limit-msg" class="limit-msg"></p>
<a id="limit-cta" class="btn primary wide limit-cta" href="#"></a>
<p id="limit-note" class="limit-note"></p>
</div>
</dialog>
<script type="module" src="main.js"></script>
</body>
</html>

276
demo/limiter.py Normal file
View File

@ -0,0 +1,276 @@
"""
Per-day talk-time budget for the speech-to-speech demo.
Our server isn't in the audio path (the browser dials the compute WebSocket
directly), so it can't cut a live stream. What it *can* do is meter time with a
server-clock, chunked reservation:
- At grant we reserve the first chunk (CHUNK_SEC) and debit it from the day's
budget. A parallel grant therefore sees the budget already spent.
- The client heartbeats; each heartbeat extends the reservation one chunk at a
time until the daily budget runs out, then we report `expired` so the client
tears down.
- On a clean end (sendBeacon) we reconcile to the real elapsed time and refund
the unused chunk. A crash (no end, no heartbeats) is reaped by a sweep and
forfeits at most one chunk.
All time is the server's clock. Budgets are per UTC day; a new day is simply a
new row (no explicit reset). Logged-in users are keyed by a hashed HF `sub`;
anonymous users by BOTH a hashed IP and a hashed signed-cookie id, OR-matched
(spent = max of the two) so clearing one identifier doesn't reset the budget.
Storage is SQLite at $USAGE_DB_PATH, else /data (persistent Spaces storage),
else a /tmp fallback. On /tmp the budget is only per-uptime — flagged in logs.
"""
import hashlib
import hmac
import logging
import math
import os
import sqlite3
import tempfile
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
logger = logging.getLogger("s2s.limiter")
# ── Tunables (env-overridable) ───────────────────────────────────────────────
ANON_SEC = int(os.environ.get("LIMIT_ANON_SEC", "300")) # 5 min/day, not signed in
FREE_SEC = int(os.environ.get("LIMIT_FREE_SEC", "600")) # 10 min/day, signed in, no PRO
CHUNK_SEC = int(os.environ.get("RESERVE_CHUNK_SEC", "10")) # reservation granularity
HEARTBEAT_SEC = int(os.environ.get("HEARTBEAT_SEC", "5")) # advertised client cadence
REAP_AFTER_SEC = int(os.environ.get("SESSION_REAP_SEC", "15")) # silence before sweep
# Stable across restarts or the hashed keys (and signed cookies) rotate and the
# budget effectively resets. Set it as a Space secret. Falls back to a per-boot
# random value (keys then only hold within one uptime).
_HASH_SECRET = (os.environ.get("USAGE_HASH_SECRET", "").strip() or os.urandom(32).hex()).encode()
_lock = threading.Lock()
_db_path: "Path | None" = None
def budget_for(tier: str) -> "int | None":
"""Daily second-budget for a tier, or None for unlimited.
Unlimited tiers: 'pro' (paying PRO members) and 'org' (members of an
allow-listed organisation, see UNLIMITED_ORGS in auth.py)."""
if tier in ("pro", "org"):
return None
if tier == "free":
return FREE_SEC
return ANON_SEC
def hash_key(raw: str) -> str:
"""HMAC a raw identifier (sub / ip / cookie id) into an opaque storage key."""
digest = hmac.new(_HASH_SECRET, raw.encode("utf-8"), hashlib.sha256).hexdigest()
return f"k_{digest}"
def sign_cookie(value: str) -> str:
"""`<id>.<sig>` so a forged anon-cookie id is rejected on read."""
sig = hmac.new(_HASH_SECRET, value.encode("utf-8"), hashlib.sha256).hexdigest()[:32]
return f"{value}.{sig}"
def verify_cookie(signed: str) -> "str | None":
"""Return the id if the signature checks out, else None."""
if not signed or "." not in signed:
return None
value, _, sig = signed.rpartition(".")
want = hmac.new(_HASH_SECRET, value.encode("utf-8"), hashlib.sha256).hexdigest()[:32]
return value if hmac.compare_digest(sig, want) else None
def _today() -> str:
return datetime.now(timezone.utc).date().isoformat()
def _resolve_db_path() -> Path:
explicit = os.environ.get("USAGE_DB_PATH", "").strip()
if explicit:
return Path(explicit)
data = Path("/data")
if data.is_dir() and os.access(data, os.W_OK):
return data / "s2s-usage.sqlite3"
logger.warning("No persistent /data — usage budget falls back to /tmp (per-uptime only).")
return Path(tempfile.gettempdir()) / "s2s-usage.sqlite3"
def _connect() -> sqlite3.Connection:
con = sqlite3.connect(_db_path, timeout=5.0)
con.execute("PRAGMA journal_mode=WAL")
con.execute("PRAGMA busy_timeout=5000")
return con
def init() -> None:
"""Create the schema. Call once at startup."""
global _db_path
_db_path = _resolve_db_path()
with _lock, _connect() as con:
con.execute(
"""CREATE TABLE IF NOT EXISTS usage_daily (
user_key TEXT NOT NULL,
day TEXT NOT NULL,
spent_sec INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL,
PRIMARY KEY (user_key, day)
)"""
)
con.execute(
"""CREATE TABLE IF NOT EXISTS sessions (
session_id TEXT PRIMARY KEY,
keys TEXT NOT NULL, -- comma-joined usage_daily keys to debit
day TEXT NOT NULL,
tier TEXT NOT NULL,
grant_ts REAL NOT NULL,
last_seen_ts REAL NOT NULL,
reserved_sec INTEGER NOT NULL,
ended INTEGER NOT NULL DEFAULT 0
)"""
)
logger.info("Usage limiter ready at %s (anon=%ss free=%ss chunk=%ss)", _db_path, ANON_SEC, FREE_SEC, CHUNK_SEC)
# ── Internal helpers (call under _lock) ───────────────────────────────────────
def _spent(con, key: str, day: str) -> int:
row = con.execute(
"SELECT spent_sec FROM usage_daily WHERE user_key=? AND day=?", (key, day)
).fetchone()
return int(row[0]) if row else 0
def _spent_max(con, keys, day: str) -> int:
"""OR-match: the most-spent identifier governs."""
return max((_spent(con, k, day) for k in keys), default=0)
def _add(con, keys, day: str, delta: int) -> None:
now = int(time.time())
for k in keys:
cur = _spent(con, k, day)
nxt = max(0, cur + delta)
con.execute(
"""INSERT INTO usage_daily (user_key, day, spent_sec, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(user_key, day) DO UPDATE SET
spent_sec = excluded.spent_sec, updated_at = excluded.updated_at""",
(k, day, nxt, now),
)
# ── Public API ────────────────────────────────────────────────────────────────
def remaining(keys, tier: str) -> "int | None":
"""Seconds left today for these keys (None = unlimited). No mutation."""
budget = budget_for(tier)
if budget is None:
return None
with _lock, _connect() as con:
return max(0, budget - _spent_max(con, keys, _today()))
def begin(session_id: str, keys, tier: str) -> int:
"""Reserve the first chunk for a new session and record it. Returns the
chunk reserved (0 if the budget is already exhausted — the first heartbeat
will then expire it). PRO (unlimited) is never tracked; don't call it here."""
day = _today()
budget = budget_for(tier)
now = time.time()
with _lock, _connect() as con:
avail = budget - _spent_max(con, keys, day) if budget is not None else CHUNK_SEC
chunk = max(0, min(CHUNK_SEC, avail))
if chunk:
_add(con, keys, day, chunk)
con.execute(
"""INSERT OR REPLACE INTO sessions
(session_id, keys, day, tier, grant_ts, last_seen_ts, reserved_sec, ended)
VALUES (?, ?, ?, ?, ?, ?, ?, 0)""",
(session_id, ",".join(keys), day, tier, now, now, chunk),
)
return chunk
def heartbeat(session_id: str) -> bool:
"""Keep a session alive: extend the reservation toward `elapsed + 1 chunk`,
debiting the budget chunk by chunk. Returns True while alive, False once the
budget is spent (caller should tear down) or the session is unknown/ended."""
now = time.time()
with _lock, _connect() as con:
row = con.execute(
"SELECT keys, day, tier, grant_ts, reserved_sec, ended FROM sessions WHERE session_id=?",
(session_id,),
).fetchone()
if not row or row[5]:
return False
keys = row[0].split(",")
day, tier, grant_ts, reserved = row[1], row[2], row[3], int(row[4])
budget = budget_for(tier)
elapsed = now - grant_ts
# Grow the reservation one chunk at a time until it covers elapsed + a
# one-chunk lookahead, or the budget runs dry.
while reserved < elapsed + CHUNK_SEC:
if budget is not None and _spent_max(con, keys, day) >= budget:
break
_add(con, keys, day, CHUNK_SEC)
reserved += CHUNK_SEC
alive = reserved > elapsed # could we cover the time already elapsed?
con.execute(
"UPDATE sessions SET last_seen_ts=?, reserved_sec=?, ended=? WHERE session_id=?",
(now, reserved, 0 if alive else 1, session_id),
)
if not alive:
_reconcile(con, keys, day, grant_ts, reserved, end_ts=now)
return alive
def end(session_id: str) -> None:
"""Clean teardown: reconcile to actual elapsed time and refund the unused
reservation. Idempotent."""
now = time.time()
with _lock, _connect() as con:
row = con.execute(
"SELECT keys, day, grant_ts, reserved_sec, ended FROM sessions WHERE session_id=?",
(session_id,),
).fetchone()
if not row or row[4]:
return
keys, day, grant_ts, reserved = row[0].split(","), row[1], row[2], int(row[3])
_reconcile(con, keys, day, grant_ts, reserved, end_ts=now)
con.execute("UPDATE sessions SET ended=1, last_seen_ts=? WHERE session_id=?", (now, session_id))
def sweep() -> None:
"""Reap sessions that went silent (crash / closed without a beacon): bill
their elapsed time, refund the rest, mark ended. Forfeits ≤ one chunk."""
now = time.time()
cutoff = now - REAP_AFTER_SEC
with _lock, _connect() as con:
stale = con.execute(
"SELECT session_id, keys, day, grant_ts, last_seen_ts, reserved_sec FROM sessions "
"WHERE ended=0 AND last_seen_ts < ?",
(cutoff,),
).fetchall()
for session_id, keys_s, day, grant_ts, last_seen, reserved in stale:
_reconcile(con, keys_s.split(","), day, grant_ts, int(reserved), end_ts=last_seen)
con.execute("UPDATE sessions SET ended=1 WHERE session_id=?", (session_id,))
if stale:
logger.debug("swept %d stale session(s)", len(stale))
def _reconcile(con, keys, day: str, grant_ts: float, reserved: int, end_ts: float) -> None:
"""Refund reserved-but-unused time. Bill elapsed rounded up to a chunk,
capped at what was reserved."""
elapsed = max(0.0, end_ts - grant_ts)
billed = min(reserved, int(math.ceil(elapsed / CHUNK_SEC) * CHUNK_SEC))
refund = reserved - billed
if refund > 0:
_add(con, keys, day, -refund)

1440
demo/main.js Normal file
View File

File diff suppressed because it is too large Load Diff

5
demo/requirements.txt Normal file
View File

@ -0,0 +1,5 @@
fastapi==0.115.6
uvicorn[standard]==0.34.0
httpx==0.28.1
# [oauth] extra pulls authlib + itsdangerous for attach_huggingface_oauth.
huggingface_hub[oauth]>=0.30,<1.0

447
demo/server.py Normal file
View File

@ -0,0 +1,447 @@
"""
Tiny server for the speech-to-speech demo.
The demo used to ship as a `sdk: static` Space, but the web-search tool needs a
search key the browser must NOT see. A static Space has no runtime process, so it
can't hold a secret the front-end uses. This server fixes that: it serves the
unchanged front-end AND exposes a same-origin `/api/search` proxy that holds the
Serper key server-side (see docs/adr/0001).
Everything lives in one container; the speech-to-speech backend stays a separate,
load-balanced service the browser talks to over WebSocket as before. The load
balancer's address is a secret too (like the Serper key): the browser never sees
it. `/api/session` proxies the session handshake server-side so only the
per-session compute URL the LB hands back (which the browser must dial) is exposed.
On the deployed Space the server also meters conversation time by HF login tier
(anonymous / signed-in / PRO) — see `limiter.py` and `auth.py`. That whole feature
is off unless BOTH `LOAD_BALANCER_URL` and `SPACE_ID` are set, so it runs only on
the live Space, never locally (even with the LB exported for testing).
`SPEECH_TO_SPEECH_URL` overrides everything: when set, the LB logic above is
disabled entirely (no session proxy, no queue, no metering, no sign-in) and the
browser connects directly to that URL, shown read-only in Settings.
Endpoints:
GET /api/config -> { search, lb, allowDirect, s2sUrl, auth }
GET /api/me -> login + tier + remaining budget (LB mode only)
POST /api/search -> { results, answer } Google via Serper.dev
POST /api/session -> proxies <LB>/session: a grant, or a queue ticket
GET /api/queue/{id} -> proxies <LB>/queue/{id}: position, or a grant on claim
DELETE /api/queue/{id} -> leave the queue (explicit "Leave queue" button)
POST /api/queue/end -> leave the queue (sendBeacon on teardown)
POST /api/session/heartbeat-> extend the reservation; { expired }
POST /api/session/end -> reconcile + refund (sendBeacon on teardown)
/* -> static files (index.html, main.js, ...)
When every compute slot is busy the load balancer hands back a queue ticket
instead of a grant; the browser polls /api/queue/{id} until it reaches the front
and a slot frees. Waiting reserves nothing — the daily budget is only reserved at
the moment a slot is actually claimed (a grant), never while queued.
"""
import asyncio
import logging
import os
import httpx
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
import auth
import limiter
logger = logging.getLogger("s2s.search")
SERPER_KEY = os.environ.get("SERPER_API_KEY", "").strip()
# Speech-to-speech load balancer URL. When set, the browser POSTs /api/session
# (which proxies <lb>/session here, server-side) and connects to the URL the LB
# returns (the original flow). The LB address itself is never sent to the browser.
# When empty, the user may instead set a direct s2s server URL in Settings and the
# browser connects to it straight (no load balancer).
LOAD_BALANCER_URL = os.environ.get("LOAD_BALANCER_URL", "").strip()
# Direct s2s server URL pinned by the deploy. Takes priority over the load
# balancer: when set, ALL LB logic is disabled (no /api/session proxy, no queue,
# no limiter, no sign-in) and the browser connects to this URL directly. Unlike
# the LB address it is NOT a secret — /api/config sends it to the client, which
# shows it read-only in Settings.
SPEECH_TO_SPEECH_URL = os.environ.get("SPEECH_TO_SPEECH_URL", "").strip()
if SPEECH_TO_SPEECH_URL:
LOAD_BALANCER_URL = ""
# HF injects SPACE_ID ("owner/space") into every Space runtime; it's absent
# locally and on a plain `docker run`. We meter conversation time ONLY on the
# deployed Space — i.e. when BOTH the LB is configured AND we're on a Space.
# Off-Space (local dev, even with the LB exported) the app still proxies the LB,
# but nothing is metered: no budget, no reservations, no sign-in gating.
SPACE_ID = os.environ.get("SPACE_ID", "").strip()
LIMITER_ENABLED = bool(LOAD_BALANCER_URL) and bool(SPACE_ID)
SERPER_URL = "https://google.serper.dev/search"
# Cap results so the tool output stays small enough to feed back to the model.
MAX_RESULTS = 5
HERE = os.path.dirname(os.path.abspath(__file__))
app = FastAPI(title="s2s-demo")
# Wire HF OAuth before the app serves (no-op unless the OAuth env is present).
# Sign-in only matters when we're metering (prod Space), so gate it on that.
AUTH_ENABLED = LIMITER_ENABLED and auth.attach(app)
@app.on_event("startup")
async def _startup():
"""Stand up the usage DB and a periodic sweeper — metered (prod Space) only."""
if not LIMITER_ENABLED:
return
limiter.init()
asyncio.create_task(_sweeper())
async def _sweeper():
while True:
await asyncio.sleep(limiter.REAP_AFTER_SEC)
try:
await asyncio.to_thread(limiter.sweep)
except Exception as exc: # pragma: no cover - defensive
logger.warning("usage sweep failed: %r", exc)
class SearchRequest(BaseModel):
query: str
# Optional user-supplied key (fallback when the deploy has no server key).
# Used for this request only; never stored.
key: str | None = None
@app.get("/api/config")
def config():
"""Client bootstrap: whether web search is available, whether the deploy runs
behind a load balancer (so the browser uses the /api/session proxy + limiter),
whether HF sign-in is available, and whether the user may instead set a direct
s2s server URL. The LB address itself is intentionally NOT included."""
return {
"search": bool(SERPER_KEY),
"lb": bool(LOAD_BALANCER_URL),
"allowDirect": not LOAD_BALANCER_URL,
# Deploy-pinned direct s2s URL (empty when unset). Not a secret: the
# browser dials it itself, and Settings shows it locked.
"s2sUrl": SPEECH_TO_SPEECH_URL,
"auth": AUTH_ENABLED,
}
@app.get("/api/me")
async def me(request: Request):
"""Login state, tier, and remaining daily budget. Only meaningful in LB mode;
sets the anonymous tracking cookie when first seen."""
if not LIMITER_ENABLED:
return {"enabled": False}
view = auth.user_view(request)
tier, keys, set_cookie = auth.resolve_identity(request)
unlimited = limiter.budget_for(tier) is None
rem = None if unlimited else await asyncio.to_thread(limiter.remaining, keys, tier)
out = {
"enabled": True,
"auth": AUTH_ENABLED,
**view,
"remainingSec": rem,
"limitSec": limiter.budget_for(tier),
"loginUrl": auth.OAUTH_LOGIN_PATH if AUTH_ENABLED else None,
"logoutUrl": auth.OAUTH_LOGOUT_PATH if AUTH_ENABLED else None,
}
resp = JSONResponse(out)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
@app.post("/api/search")
async def search(req: SearchRequest):
"""Proxy a Google search via Serper.dev. The key stays on the server unless
the user brought their own (then theirs is used for this request only)."""
query = (req.query or "").strip()
if not query:
raise HTTPException(status_code=400, detail="Empty query.")
key = (req.key or "").strip() or SERPER_KEY
if not key:
# No server key and the user didn't supply one — search is unavailable.
raise HTTPException(status_code=503, detail="Search is not configured.")
headers = {"X-API-KEY": key, "Content-Type": "application/json"}
payload = {"q": query, "num": MAX_RESULTS}
try:
async with httpx.AsyncClient(timeout=12.0) as http:
resp = await http.post(SERPER_URL, headers=headers, json=payload)
except httpx.RequestError as exc:
logger.warning("Serper unreachable: %r", exc)
raise HTTPException(status_code=502, detail="Search provider unreachable.")
if resp.status_code != 200:
# Serper's error body carries the real reason (e.g. "Not enough
# credits") and contains no key, so it's safe to log and relay.
body = resp.text[:300]
logger.warning("Serper error %s: %s", resp.status_code, body)
msg = None
try:
msg = resp.json().get("message")
except Exception:
pass
detail = f"Search provider error ({resp.status_code})"
if msg:
detail += f": {msg}"
raise HTTPException(status_code=502, detail=detail)
data = resp.json()
results = []
for item in (data.get("organic") or [])[:MAX_RESULTS]:
results.append(
{
"title": item.get("title", ""),
"snippet": item.get("snippet", ""),
"url": item.get("link", ""),
}
)
# A direct answer when Google has one — saves the model a hop.
box = data.get("answerBox") or {}
answer = box.get("answer") or box.get("snippet") or None
if not answer:
kg = data.get("knowledgeGraph") or {}
answer = kg.get("description") or None
return JSONResponse({"query": query, "answer": answer, "results": results})
@app.post("/api/session")
async def session(request: Request):
"""Proxy the session handshake to the load balancer, keeping its URL secret,
and meter conversation time by tier.
The browser POSTs here (same-origin); we resolve the caller's tier, refuse if
today's budget is already spent (402), otherwise POST <LOAD_BALANCER_URL>/session
and relay the JSON back. The LB body carries a per-session `connect_url`
(compute host + short-lived token) the browser must dial directly — that one
URL is unavoidably exposed, but the stable load-balancer address is not. On a
successful grant we reserve the first time chunk against the day's budget."""
if not LOAD_BALANCER_URL:
# No LB configured — this deploy is direct-mode only; the browser should
# never call this. 404 so it's indistinguishable from a missing route.
raise HTTPException(status_code=404, detail="Not found.")
tier, keys, set_cookie = auth.resolve_identity(request)
# Metering runs only on the deployed Space; off-Space the LB still proxies but
# nothing is tracked. Within metering, unlimited tiers (pro, org) aren't either.
tracked = LIMITER_ENABLED and limiter.budget_for(tier) is not None
# Refuse before troubling the LB if the day's budget is already gone. Done
# here (at enqueue) so we never put a user who can't talk into the queue.
if tracked:
rem = await asyncio.to_thread(limiter.remaining, keys, tier)
if rem is not None and rem <= 0:
resp = JSONResponse(
{"tier": tier, "reason": "limit", "remainingSec": 0}, status_code=402
)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
url = f"{LOAD_BALANCER_URL.rstrip('/')}/session"
try:
async with httpx.AsyncClient(timeout=15.0) as http:
lb = await http.post(url, headers={"Content-Type": "application/json"}, content="{}")
except httpx.RequestError as exc:
logger.warning("Load balancer unreachable: %r", exc)
raise HTTPException(status_code=502, detail="Speech service unreachable.")
# The queue is full: the LB replies 503 {state:"at_capacity"}. Relay it as-is
# so the client shows a soft "try again shortly", not a hard error.
if lb.status_code == 503:
body = _safe_json(lb)
if body.get("state") == "at_capacity":
resp = JSONResponse({"state": "at_capacity"}, status_code=503)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
if lb.status_code != 200:
# The LB's error body may name the reason (e.g. capacity); it carries no
# secret, so relay a trimmed copy.
logger.warning("Session handshake failed %s: %s", lb.status_code, lb.text[:300])
raise HTTPException(status_code=502, detail=f"Session handshake failed ({lb.status_code}).")
data = lb.json()
# Busy pool: the LB queued us. Relay the ticket untouched — crucially with NO
# reservation, so waiting in line never costs the day's budget.
if data.get("state") == "queued":
data["tier"] = tier
resp = JSONResponse(data)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
# A slot was free: reserve the first chunk now and return the grant.
return await _finalize_grant(data, keys, tier, tracked, set_cookie)
@app.get("/api/queue/{queue_id}")
async def queue_status(queue_id: str, request: Request):
"""Poll a waiting ticket: relay the position, or — when the head of the line
claims a freed slot — reserve the budget now and return the grant. Re-checks the
daily budget at claim, since a multi-minute wait could have spent it elsewhere."""
if not LOAD_BALANCER_URL:
raise HTTPException(status_code=404, detail="Not found.")
tier, keys, set_cookie = auth.resolve_identity(request)
tracked = LIMITER_ENABLED and limiter.budget_for(tier) is not None
url = f"{LOAD_BALANCER_URL.rstrip('/')}/queue/{queue_id}"
try:
async with httpx.AsyncClient(timeout=15.0) as http:
lb = await http.get(url)
except httpx.RequestError as exc:
logger.warning("Load balancer unreachable: %r", exc)
raise HTTPException(status_code=502, detail="Speech service unreachable.")
if lb.status_code == 404:
# Ticket unknown/expired (reaped after we stopped polling). Tell the client
# to start over rather than spin.
resp = JSONResponse({"state": "expired"}, status_code=404)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
if lb.status_code != 200:
logger.warning("Queue poll failed %s: %s", lb.status_code, lb.text[:300])
raise HTTPException(status_code=502, detail=f"Queue poll failed ({lb.status_code}).")
data = lb.json()
if data.get("state") == "queued":
data["tier"] = tier
resp = JSONResponse(data)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
# Claimed a slot. Re-check the budget: it may have been spent in another tab
# during the wait. If so, refuse — the just-claimed slot is now a pending
# session on the LB and its pending-timeout reaper reclaims it shortly.
if tracked:
rem = await asyncio.to_thread(limiter.remaining, keys, tier)
if rem is not None and rem <= 0:
resp = JSONResponse(
{"tier": tier, "reason": "limit", "remainingSec": 0}, status_code=402
)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
return await _finalize_grant(data, keys, tier, tracked, set_cookie)
@app.delete("/api/queue/{queue_id}")
async def queue_leave(queue_id: str):
"""Leave the queue from the explicit 'Leave queue' button (a real fetch)."""
if not LOAD_BALANCER_URL:
raise HTTPException(status_code=404, detail="Not found.")
await _lb_leave(queue_id)
return {"ok": True}
@app.post("/api/queue/end")
async def queue_end(request: Request):
"""Leave the queue on teardown/tab-close (navigator.sendBeacon, which can only
POST). Body: { queueId }. Best-effort; the LB reaps the ticket on TTL anyway."""
if not LOAD_BALANCER_URL:
raise HTTPException(status_code=404, detail="Not found.")
qid = await _queue_id(request)
if qid:
await _lb_leave(qid)
return {"ok": True}
async def _finalize_grant(data, keys, tier, tracked, set_cookie):
"""Shared grant tail (fast path or queue claim): reserve the first chunk, attach
the metering fields the client needs, and set the anon cookie."""
remaining = None
if tracked and data.get("session_id"):
await asyncio.to_thread(limiter.begin, data["session_id"], keys, tier)
remaining = await asyncio.to_thread(limiter.remaining, keys, tier)
data.update({
"tier": tier,
"limited": tracked,
"remainingSec": remaining,
"heartbeatSec": limiter.HEARTBEAT_SEC,
})
resp = JSONResponse(data)
if set_cookie:
auth.set_anon_cookie(resp, set_cookie)
return resp
async def _lb_leave(queue_id: str) -> None:
"""Best-effort: tell the LB to drop a waiting ticket."""
url = f"{LOAD_BALANCER_URL.rstrip('/')}/queue/{queue_id}"
try:
async with httpx.AsyncClient(timeout=5.0) as http:
await http.delete(url)
except httpx.RequestError as exc:
logger.warning("Queue leave failed: %r", exc)
def _safe_json(response) -> dict:
try:
body = response.json()
except Exception:
return {}
return body if isinstance(body, dict) else {}
async def _queue_id(request: Request) -> str:
"""Pull `queueId` from a JSON body, tolerating sendBeacon's blob posts."""
try:
data = await request.json()
except Exception:
return ""
return (data or {}).get("queueId", "") if isinstance(data, dict) else ""
async def _session_id(request: Request) -> str:
"""Pull `sessionId` from a JSON body, tolerating sendBeacon's blob posts."""
try:
data = await request.json()
except Exception:
return ""
return (data or {}).get("sessionId", "") if isinstance(data, dict) else ""
@app.post("/api/session/heartbeat")
async def session_heartbeat(request: Request):
"""Extend the live reservation one chunk at a time. `expired` once the day's
budget is spent — the client then tears down."""
if not LIMITER_ENABLED:
raise HTTPException(status_code=404, detail="Not found.")
sid = await _session_id(request)
alive = bool(sid) and await asyncio.to_thread(limiter.heartbeat, sid)
return {"expired": not alive}
@app.post("/api/session/end")
async def session_end(request: Request):
"""Clean teardown: reconcile to real elapsed time and refund the unused
chunk. Sent via navigator.sendBeacon, so it must succeed without a response."""
if not LIMITER_ENABLED:
raise HTTPException(status_code=404, detail="Not found.")
sid = await _session_id(request)
if sid:
await asyncio.to_thread(limiter.end, sid)
return {"ok": True}
# Static front-end. Registered last so the /api routes win. `html=True` serves
# index.html at "/". The repo is public anyway, so serving the dir is fine.
app.mount("/", StaticFiles(directory=HERE, html=True), name="static")

2646
demo/style.css Normal file
View File

File diff suppressed because it is too large Load Diff

189
demo/ui/account.js Normal file
View File

@ -0,0 +1,189 @@
// @ts-check
/**
* Account — the HF login chip and the daily-limit modal.
*
* Reads `/api/me` to learn the current tier (anonymous / signed-in / PRO) and
* remaining daily talk-time, renders a sign-in pill or a signed-in chip with a
* small popover (tier, remaining, sign out, upgrade), and shows the limit modal
* when a conversation is refused or cut. The time metering itself lives in
* main.js (heartbeat loop) + the server; this module is just the surface.
*
* Inert unless the deploy is in LB mode (`/api/me` → `{enabled:true}`).
*/
import { $, escHtml } from "./dom.js";
const PRO_URL = "https://huggingface.co/subscribe/pro";
// Official multi-color Hugging Face logo, used in the badge + sign-in CTA.
const HF_MARK = `<svg class="hf-logo" viewBox="0 0 95 88" fill="none" aria-hidden="true"><path fill="#FFD21E" d="M47.21 76.5a34.75 34.75 0 1 0 0-69.5 34.75 34.75 0 0 0 0 69.5Z"/><path fill="#FF9D0B" d="M81.96 41.75a34.75 34.75 0 1 0-69.5 0 34.75 34.75 0 0 0 69.5 0Zm-73.5 0a38.75 38.75 0 1 1 77.5 0 38.75 38.75 0 0 1-77.5 0Z"/><path fill="#3A3B45" d="M58.5 32.3c1.28.44 1.78 3.06 3.07 2.38a5 5 0 1 0-6.76-2.07c.61 1.15 2.55-.72 3.7-.32ZM34.95 32.3c-1.28.44-1.79 3.06-3.07 2.38a5 5 0 1 1 6.76-2.07c-.61 1.15-2.56-.72-3.7-.32Z"/><path fill="#FF323D" d="M46.96 56.29c9.83 0 13-8.76 13-13.26 0-2.34-1.57-1.6-4.09-.36-2.33 1.15-5.46 2.74-8.9 2.74-7.19 0-13-6.88-13-2.38s3.16 13.26 13 13.26Z"/><path fill="#3A3B45" fill-rule="evenodd" d="M39.43 54a8.7 8.7 0 0 1 5.3-4.49c.4-.12.81.57 1.24 1.28.4.68.82 1.37 1.24 1.37.45 0 .9-.68 1.33-1.35.45-.7.89-1.38 1.32-1.25a8.61 8.61 0 0 1 5 4.17c3.73-2.94 5.1-7.74 5.1-10.7 0-2.34-1.57-1.6-4.09-.36l-.14.07c-2.31 1.15-5.39 2.67-8.77 2.67s-6.45-1.52-8.77-2.67c-2.6-1.29-4.23-2.1-4.23.29 0 3.05 1.46 8.06 5.47 10.97Z" clip-rule="evenodd"/><path fill="#FF9D0B" d="M70.71 37a3.25 3.25 0 1 0 0-6.5 3.25 3.25 0 0 0 0 6.5ZM24.21 37a3.25 3.25 0 1 0 0-6.5 3.25 3.25 0 0 0 0 6.5ZM17.52 48c-1.62 0-3.06.66-4.07 1.87a5.97 5.97 0 0 0-1.33 3.76 7.1 7.1 0 0 0-1.94-.3c-1.55 0-2.95.59-3.94 1.66a5.8 5.8 0 0 0-.8 7 5.3 5.3 0 0 0-1.79 2.82c-.24.9-.48 2.8.8 4.74a5.22 5.22 0 0 0-.37 5.02c1.02 2.32 3.57 4.14 8.52 6.1 3.07 1.22 5.89 2 5.91 2.01a44.33 44.33 0 0 0 10.93 1.6c5.86 0 10.05-1.8 12.46-5.34 3.88-5.69 3.33-10.9-1.7-15.92-2.77-2.78-4.62-6.87-5-7.77-.78-2.66-2.84-5.62-6.25-5.62a5.7 5.7 0 0 0-4.6 2.46c-1-1.26-1.98-2.25-2.86-2.82A7.4 7.4 0 0 0 17.52 48Zm0 4c.51 0 1.14.22 1.82.65 2.14 1.36 6.25 8.43 7.76 11.18.5.92 1.37 1.31 2.14 1.31 1.55 0 2.75-1.53.15-3.48-3.92-2.93-2.55-7.72-.68-8.01.08-.02.17-.02.24-.02 1.7 0 2.45 2.93 2.45 2.93s2.2 5.52 5.98 9.3c3.77 3.77 3.97 6.8 1.22 10.83-1.88 2.75-5.47 3.58-9.16 3.58-3.81 0-7.73-.9-9.92-1.46-.11-.03-13.45-3.8-11.76-7 .28-.54.75-.76 1.34-.76 2.38 0 6.7 3.54 8.57 3.54.41 0 .7-.17.83-.6.79-2.85-12.06-4.05-10.98-8.17.2-.73.71-1.02 1.44-1.02 3.14 0 10.2 5.53 11.68 5.53.11 0 .2-.03.24-.1.74-1.2.33-2.04-4.9-5.2-5.21-3.16-8.88-5.06-6.8-7.33.24-.26.58-.38 1-.38 3.17 0 10.66 6.82 10.66 6.82s2.02 2.1 3.25 2.1c.28 0 .52-.1.68-.38.86-1.46-8.06-8.22-8.56-11.01-.34-1.9.24-2.85 1.31-2.85Z"/><path fill="#FFD21E" d="M38.6 76.69c2.75-4.04 2.55-7.07-1.22-10.84-3.78-3.77-5.98-9.3-5.98-9.3s-.82-3.2-2.69-2.9c-1.87.3-3.24 5.08.68 8.01 3.91 2.93-.78 4.92-2.29 2.17-1.5-2.75-5.62-9.82-7.76-11.18-2.13-1.35-3.63-.6-3.13 2.2.5 2.79 9.43 9.55 8.56 11-.87 1.47-3.93-1.71-3.93-1.71s-9.57-8.71-11.66-6.44c-2.08 2.27 1.59 4.17 6.8 7.33 5.23 3.16 5.64 4 4.9 5.2-.75 1.2-12.28-8.53-13.36-4.4-1.08 4.11 11.77 5.3 10.98 8.15-.8 2.85-9.06-5.38-10.74-2.18-1.7 3.21 11.65 6.98 11.76 7.01 4.3 1.12 15.25 3.49 19.08-2.12Z"/><path fill="#FF9D0B" d="M77.4 48c1.62 0 3.07.66 4.07 1.87a5.97 5.97 0 0 1 1.33 3.76 7.1 7.1 0 0 1 1.95-.3c1.55 0 2.95.59 3.94 1.66a5.8 5.8 0 0 1 .8 7 5.3 5.3 0 0 1 1.78 2.82c.24.9.48 2.8-.8 4.74a5.22 5.22 0 0 1 .37 5.02c-1.02 2.32-3.57 4.14-8.51 6.1-3.08 1.22-5.9 2-5.92 2.01a44.33 44.33 0 0 1-10.93 1.6c-5.86 0-10.05-1.8-12.46-5.34-3.88-5.69-3.33-10.9 1.7-15.92 2.78-2.78 4.63-6.87 5.01-7.77.78-2.66 2.83-5.62 6.24-5.62a5.7 5.7 0 0 1 4.6 2.46c1-1.26 1.98-2.25 2.87-2.82A7.4 7.4 0 0 1 77.4 48Zm0 4c-.51 0-1.13.22-1.82.65-2.13 1.36-6.25 8.43-7.76 11.18a2.43 2.43 0 0 1-2.14 1.31c-1.54 0-2.75-1.53-.14-3.48 3.91-2.93 2.54-7.72.67-8.01a1.54 1.54 0 0 0-.24-.02c-1.7 0-2.45 2.93-2.45 2.93s-2.2 5.52-5.97 9.3c-3.78 3.77-3.98 6.8-1.22 10.83 1.87 2.75 5.47 3.58 9.15 3.58 3.82 0 7.73-.9 9.93-1.46.1-.03 13.45-3.8 11.76-7-.29-.54-.75-.76-1.34-.76-2.38 0-6.71 3.54-8.57 3.54-.42 0-.71-.17-.83-.6-.8-2.85 12.05-4.05 10.97-8.17-.19-.73-.7-1.02-1.44-1.02-3.14 0-10.2 5.53-11.68 5.53-.1 0-.19-.03-.23-.1-.74-1.2-.34-2.04 4.88-5.2 5.23-3.16 8.9-5.06 6.8-7.33-.23-.26-.57-.38-.98-.38-3.18 0-10.67 6.82-10.67 6.82s-2.02 2.1-3.24 2.1a.74.74 0 0 1-.68-.38c-.87-1.46 8.05-8.22 8.55-11.01.34-1.9-.24-2.85-1.31-2.85Z"/><path fill="#FFD21E" d="M56.33 76.69c-2.75-4.04-2.56-7.07 1.22-10.84 3.77-3.77 5.97-9.3 5.97-9.3s.82-3.2 2.7-2.9c1.86.3 3.23 5.08-.68 8.01-3.92 2.93.78 4.92 2.28 2.17 1.51-2.75 5.63-9.82 7.76-11.18 2.13-1.35 3.64-.6 3.13 2.2-.5 2.79-9.42 9.55-8.55 11 .86 1.47 3.92-1.71 3.92-1.71s9.58-8.71 11.66-6.44c2.08 2.27-1.58 4.17-6.8 7.33-5.23 3.16-5.63 4-4.9 5.2.75 1.2 12.28-8.53 13.36-4.4 1.08 4.11-11.76 5.3-10.97 8.15.8 2.85 9.05-5.38 10.74-2.18 1.69 3.21-11.65 6.98-11.76 7.01-4.31 1.12-15.26 3.49-19.08-2.12Z"/></svg>`;
/** @param {number} sec @returns {string} "m:ss" */
function fmt(sec) {
const s = Math.max(0, Math.round(sec));
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
}
export class Account {
constructor() {
/** @type {HTMLElement} */
this._root = $("#account");
/** @type {HTMLDialogElement} */
this._modal = $("#limit-modal");
this._modalTitle = $("#limit-title");
this._modalMsg = $("#limit-msg");
this._modalNote = $("#limit-note");
/** @type {HTMLAnchorElement} */
this._modalCta = /** @type {any} */ ($("#limit-cta"));
/** @type {{enabled:boolean, auth?:boolean, loggedIn?:boolean, username?:string, avatar?:string, tier?:string, remainingSec?:number|null, limitSec?:number|null, loginUrl?:string|null, logoutUrl?:string|null}} */
this._me = { enabled: false };
this._popoverOpen = false;
$("#limit-close").addEventListener("click", () => this._modal.close());
this._modal.addEventListener("click", (e) => {
if (e.target === this._modal) this._modal.close();
});
// Close the popover on an outside click.
document.addEventListener("click", (e) => {
if (this._popoverOpen && !this._root.contains(/** @type {Node} */ (e.target))) {
this._closePopover();
}
});
}
get tier() {
return this._me.tier || "anon";
}
/** Fetch `/api/me` and (re)render the chip. Safe to call repeatedly (load,
* after the OAuth redirect, after a conversation ends). */
async refresh() {
try {
const res = await fetch("api/me");
this._me = res.ok ? await res.json() : { enabled: false };
} catch {
this._me = { enabled: false };
}
this._render();
}
_render() {
const me = this._me;
if (!me.enabled) {
this._root.hidden = true;
this._root.innerHTML = "";
return;
}
this._root.hidden = false;
if (!me.loggedIn) {
// Signed-out: a sign-in pill (only when OAuth is actually available).
if (me.auth && me.loginUrl) {
this._root.innerHTML = `<a class="signin-pill" href="${escHtml(me.loginUrl)}" title="Sign in for more time"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M15 3h4a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2h-4"/><polyline points="10 17 15 12 10 7"/><line x1="15" y1="12" x2="3" y2="12"/></svg><span>Sign in</span></a>`;
} else {
this._root.innerHTML = "";
this._root.hidden = true;
}
return;
}
// Signed-in: avatar + handle chip that toggles a popover.
const isPro = me.tier === "pro";
// Org members get unlimited usage too, but aren't PRO — don't brand them so.
const isUnlimited = isPro || me.tier === "org";
const avatar = me.avatar
? `<img class="account-avatar" src="${escHtml(me.avatar)}" alt="" />`
: `<span class="account-avatar account-avatar-fallback">${escHtml((me.username || "?")[0].toUpperCase())}</span>`;
const remaining =
isUnlimited || me.remainingSec == null
? "Unlimited"
: `${fmt(me.remainingSec)} left today`;
const tierLabel = isPro ? "PRO" : isUnlimited ? "Team" : "Free";
this._root.innerHTML = `
<button id="account-chip" class="account-chip" aria-haspopup="true" aria-expanded="false">
${avatar}
<span class="account-handle">${escHtml(me.username || "you")}</span>
${isPro
? '<span class="account-pro">PRO</span>'
: me.tier === "org"
? '<span class="account-pro account-team">TEAM</span>'
: ""}
</button>
<div id="account-pop" class="account-pop" hidden>
<div class="account-pop-row account-pop-name">${escHtml(me.username || "you")}</div>
<div class="account-pop-row account-pop-meta">
<span class="account-tier">${tierLabel}</span>
<span class="account-remaining">${escHtml(remaining)}</span>
</div>
${isUnlimited ? "" : `<a class="account-pop-link" href="${PRO_URL}" target="_blank" rel="noopener">Upgrade to PRO</a>`}
<a class="account-pop-link account-signout" href="${escHtml(me.logoutUrl || "#")}">Sign out</a>
</div>`;
const chip = $("#account-chip");
chip.addEventListener("click", (e) => {
e.stopPropagation();
this._popoverOpen ? this._closePopover() : this._openPopover();
});
}
_openPopover() {
const pop = document.getElementById("account-pop");
const chip = document.getElementById("account-chip");
if (!pop || !chip) return;
pop.hidden = false;
chip.setAttribute("aria-expanded", "true");
this._popoverOpen = true;
}
_closePopover() {
const pop = document.getElementById("account-pop");
const chip = document.getElementById("account-chip");
if (pop) pop.hidden = true;
if (chip) chip.setAttribute("aria-expanded", "false");
this._popoverOpen = false;
}
/**
* Show the limit modal for a tier — used both when a conversation is refused
* at start (402) and when a live one is cut (heartbeat `expired`).
* @param {string} [tier]
*/
showLimit(tier = this.tier) {
const canSignIn = this._me.auth && this._me.loginUrl;
this._modalTitle.textContent = "Thanks for chatting!";
if (tier === "anon") {
this._modalMsg.textContent =
"Guest conversations run for 5 minutes. Sign in with Hugging Face to get 10 minutes a day for free, and PRO members chat with no limit at all.";
this._modalNote.textContent = "Your free minutes refresh tomorrow.";
if (canSignIn) {
this._modalCta.innerHTML = `${HF_MARK}<span>Sign in with Hugging Face</span>`;
this._modalCta.href = /** @type {string} */ (this._me.loginUrl);
this._modalCta.hidden = false;
} else {
this._modalCta.hidden = true;
}
} else {
// Signed-in, non-PRO.
this._modalMsg.textContent =
"You've enjoyed your 10 minutes for today. Go PRO for unlimited conversations and to support open source AI.";
this._modalNote.textContent = "Or come back tomorrow. Your minutes reset daily.";
this._modalCta.innerHTML = "<span>Upgrade to PRO</span>";
this._modalCta.href = PRO_URL;
this._modalCta.hidden = false;
}
if (!this._modal.open) this._modal.showModal();
}
/** Show a warm "we're at capacity" message when even the waiting line is full.
* Reuses the limit modal shell; no call-to-action, just reassurance. */
showBusy() {
this._modalTitle.textContent = "Hugged to the limit 🤗";
this._modalMsg.textContent =
"Every slot and the whole line are full right now. Too much love! Grab a coffee and pop back in a minute.";
this._modalNote.textContent = "A spot usually opens up within a few minutes.";
this._modalCta.hidden = true;
if (!this._modal.open) this._modal.showModal();
}
}

425
demo/ui/chat.js Normal file
View File

@ -0,0 +1,425 @@
// @ts-check
/**
* ChatView — owns the whole conversation surface: the slide-in history panel,
* the ephemeral on-orb bubbles, and all the transcript/tool/streaming
* bookkeeping. main.js wires the realtime client's events straight to the
* `on*` methods here and otherwise doesn't touch chat state.
*
* Two parallel surfaces share one shape (see `_buildMessageEl`):
* - ephemeral bubbles (`.bubble` / `.bubble-*`) fade on a timer
* - persistent history (`.hist-msg` / `.hist-*`) the durable panel log
*
* Keying:
* - user transcripts by the server's `item_id` — a speculative continuation
* REUSES it, so both segments land in one row/bubble; deltas are CUMULATIVE
* (each carries the full sentence so far), so we replace text wholesale.
* - assistant transcripts by `response_id`, so a cancelled speculative reply
* can be marked interrupted without erasing what was already shown.
*/
import { $, escHtml, DEBUG } from "./dom.js";
const WRENCH_PATH = `<path d="M14.7 6.3a1 1 0 0 0 0 1.4l1.6 1.6a1 1 0 0 0 1.4 0l3.77-3.77a6 6 0 0 1-7.94 7.94l-6.91 6.91a2.12 2.12 0 0 1-3-3l6.91-6.91a6 6 0 0 1 7.94-7.94l-3.76 3.76z"/>`;
const CHAT_BUBBLE_SVG = `<svg width="28" height="28" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg>`;
const EMPTY_STATE_HTML = `<div id="chat-empty" class="chat-empty">${CHAT_BUBBLE_SVG}<span class="chat-empty-title">No messages yet</span><span class="chat-empty-hint">Tap the orb and start talking</span></div>`;
export class ChatView {
constructor() {
/** @type {HTMLButtonElement} */
this._chatBtn = $("#chat-btn");
/** @type {HTMLSpanElement} */
this._chatBadge = $("#chat-badge");
/** @type {HTMLDivElement} */
this._chatPanel = $("#chat-panel");
/** @type {HTMLDivElement} */
this._chatPanelBackdrop = $("#chat-panel-backdrop");
/** @type {HTMLButtonElement} */
this._chatPanelClose = $("#chat-panel-close");
/** @type {HTMLDivElement} */
this._chatHistory = $("#chat-history");
/** @type {HTMLDivElement} */
this._bubbleStack = $("#bubble-stack");
this._panelOpen = false;
this._scrollQueued = false;
// ── User transcript state (keyed by item_id) ───────────────────────────
/** @type {Map<string, HTMLElement>} */
this._userHistByItem = new Map();
/** @type {HTMLElement | null} */
this._activeUserBubble = null;
this._activeUserItemId = "";
// Monotonic counter for synthesizing unique keys when the server omits an
// item_id / response_id, so id-less messages never collapse onto each other.
this._anonSeq = 0;
// ── Assistant transcript state (keyed by response_id) ──────────────────
/** @type {Map<string, { bubble: HTMLElement, hist: HTMLElement }>} */
this._asstByResp = new Map();
// ── Ephemeral bubble auto-dismiss ──────────────────────────────────────
// Per-element expiry (epoch ms). A bubble fades once its expiry passes —
// but only in stack order (see _reapBubbles). Refreshing the expiry keeps a
// bubble alive while it updates (e.g. the live user utterance).
/** @type {WeakMap<HTMLElement, number>} */
this._bubbleExpiry = new WeakMap();
// Single pending reaper handle: one timer for the whole stack (not one per
// bubble) so dismissal is strictly oldest-first regardless of per-bubble delays.
this._reaperHandle = 0;
this._chatBtn.addEventListener("click", () => (this._panelOpen ? this._closePanel() : this._openPanel()));
this._chatPanelClose.addEventListener("click", () => this._closePanel());
this._chatPanelBackdrop.addEventListener("click", () => this._closePanel());
document.addEventListener("keydown", (e) => {
if (e.key === "Escape" && this._panelOpen) this._closePanel();
});
}
// ── Panel ───────────────────────────────────────────────────────────────
_openPanel() {
this._panelOpen = true;
this._chatPanel.classList.add("open");
this._chatBadge.classList.remove("visible");
this._scrollToBottom();
}
_closePanel() {
this._panelOpen = false;
this._chatPanel.classList.remove("open");
}
// Coalesce scroll-to-bottom: a burst of cumulative transcript deltas would
// otherwise queue one rAF per delta, all writing the same scrollTop.
_scrollToBottom() {
if (!this._panelOpen || this._scrollQueued) return;
this._scrollQueued = true;
requestAnimationFrame(() => {
this._scrollQueued = false;
this._chatHistory.scrollTop = this._chatHistory.scrollHeight;
});
}
_markUnread() {
if (this._panelOpen) {
this._scrollToBottom();
return;
}
this._chatBadge.classList.add("visible");
}
// ── Shared rendering ──────────────────────────────────────────────────────
/**
* Build a role-labelled message element. Ephemeral bubbles and persistent
* history rows share the same shape and differ only in their class prefix
* (`bubble`/`bubble-*` vs `hist-msg`/`hist-*`).
* @param {{ container: string, prefix: string, role: "user"|"assistant", text: string, partial?: boolean }} o
* @returns {HTMLElement}
*/
_buildMessageEl({ container, prefix, role, text, partial = false }) {
const el = document.createElement("div");
el.className = `${container} ${role}`;
const label = role === "user" ? "You" : "Assistant";
el.innerHTML = `<div class="${prefix}-role">${label}</div><div class="${prefix}-body${partial ? " partial" : ""}">${escHtml(text)}</div>`;
return el;
}
// ── Ephemeral bubbles ───────────────────────────────────────────────────
/** @param {"user"|"assistant"|"tool"} role @param {string} text @returns {HTMLElement} */
_spawnBubble(role, text) {
let el;
if (role === "tool") {
el = document.createElement("div");
el.className = "bubble tool";
el.innerHTML = `<svg class="bubble-tool-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">${WRENCH_PATH}</svg><span class="bubble-tool-text">${escHtml(text)}</span>`;
} else {
el = this._buildMessageEl({ container: "bubble", prefix: "bubble", role, text });
}
this._bubbleStack.appendChild(el);
// Cap the stack at 3, but never evict the bubble the caller is still
// actively updating (the live user bubble) — drop the next-oldest instead.
const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]);
if (visible.length > 3) {
this._dismissBubble(visible.find((b) => b !== this._activeUserBubble) ?? visible[0]);
}
requestAnimationFrame(() => el.classList.add("in"));
return el;
}
/** @param {HTMLElement} el @param {string} text */
_updateBubbleText(el, text) {
const t = el.querySelector(".bubble-body");
if (t) t.textContent = text;
}
/** @param {HTMLElement} el */
_dismissBubble(el) {
if (!el || el.classList.contains("out")) return; // idempotent
this._bubbleExpiry.delete(el);
el.classList.remove("in");
el.classList.add("out");
const remove = () => el.remove();
el.addEventListener("transitionend", remove, { once: true });
// Fallback: transitionend never fires if the bubble's visual state didn't
// change (dismissed pre-paint) or the tab is backgrounded. The transition
// is 0.3s, so force removal a little after.
setTimeout(remove, 400);
}
/**
* Fade bubbles whose expiry has passed — strictly oldest-first. We walk the
* stack top (oldest) to bottom (newest) and stop at the first bubble still
* alive: nothing newer may leave while an older bubble is still on screen. A
* bubble that keeps updating pushes its own expiry forward, so it (and
* everything behind it) stays put until it finally goes quiet.
*/
_reapBubbles() {
this._reaperHandle = 0;
const now = Date.now();
const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]);
let nextWake = Infinity;
for (const el of visible) {
const exp = this._bubbleExpiry.get(el) ?? now; // no expiry recorded → treat as due
if (exp <= now) {
this._dismissBubble(el);
} else {
// Oldest survivor isn't due yet; stop so nothing newer leaves before it.
nextWake = exp;
break;
}
}
if (nextWake !== Infinity) {
this._reaperHandle = setTimeout(() => this._reapBubbles(), Math.max(50, nextWake - Date.now()));
}
}
/**
* (Re)arm a bubble's auto-dismiss by pushing its expiry out by `delay`.
* Calling it again resets the countdown — so a bubble that keeps updating
* stays on screen and only fades once it goes quiet. Removal is ordered by the
* shared reaper, so the oldest bubble always disappears first.
* @param {HTMLElement} el @param {number} [delay]
*/
_bumpDismiss(el, delay = 4000) {
this._bubbleExpiry.set(el, Date.now() + delay);
if (!this._reaperHandle) this._reaperHandle = setTimeout(() => this._reapBubbles(), delay);
}
// ── History ───────────────────────────────────────────────────────────────
/** Render the empty-state placeholder into the history panel. */
renderEmptyState() {
this._chatHistory.innerHTML = EMPTY_STATE_HTML;
}
/** Reset the panel to the empty state and clear the unread badge. */
clear() {
this.renderEmptyState();
this._chatBadge.classList.remove("visible");
}
/** @param {"user"|"assistant"} role @param {string} text @param {boolean} partial @returns {HTMLElement} */
_appendHistMsg(role, text, partial) {
const empty = this._chatHistory.querySelector(".chat-empty");
if (empty) empty.remove();
const el = this._buildMessageEl({ container: "hist-msg", prefix: "hist", role, text, partial });
this._chatHistory.appendChild(el);
this._scrollToBottom();
return el;
}
/** @param {HTMLElement | null} el @param {string} text @param {boolean} partial */
_updateHistMsg(el, text, partial) {
if (!el) return;
const body = /** @type {HTMLElement | null} */ (el.querySelector(".hist-body"));
if (!body) return;
body.textContent = text;
body.classList.toggle("partial", partial);
this._scrollToBottom();
}
/**
* Append a tool-call row to the conversation. We only add it once the tool
* has run, so the expandable toggle carries BOTH the call input and its result.
* @param {string} name @param {string} argsJson @param {string} output
*/
_appendHistTool(name, argsJson, output) {
const empty = this._chatHistory.querySelector(".chat-empty");
if (empty) empty.remove();
let pretty = argsJson;
try { pretty = JSON.stringify(JSON.parse(argsJson), null, 2); } catch {}
const el = document.createElement("div");
el.className = "hist-msg tool";
el.innerHTML = `
<div class="hist-role">Tool call</div>
<button class="hist-tool-header" aria-expanded="false">
<svg class="hist-tool-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">${WRENCH_PATH}</svg>
<span class="hist-tool-name">${escHtml(name)}</span>
<svg class="hist-tool-chevron" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg>
</button>
<div class="hist-tool-body">
<div class="hist-tool-label">Input</div>
<div class="hist-tool-block">${escHtml(pretty)}</div>
<div class="hist-tool-label">Output</div>
<div class="hist-tool-block hist-tool-output">${escHtml(output || "(no output)")}</div>
</div>
`;
const header = /** @type {HTMLButtonElement} */ (el.querySelector(".hist-tool-header"));
const body = /** @type {HTMLDivElement} */ (el.querySelector(".hist-tool-body"));
header.addEventListener("click", () => {
const expanded = header.getAttribute("aria-expanded") === "true";
header.setAttribute("aria-expanded", String(!expanded));
body.classList.toggle("open", !expanded);
});
this._chatHistory.appendChild(el);
this._scrollToBottom();
}
/** Tag an assistant history row as interrupted (user barged in mid-reply).
* @param {HTMLElement | null} hist */
_markHistInterrupted(hist) {
if (!hist || hist.querySelector(".hist-note")) return;
hist.classList.add("interrupted");
const note = document.createElement("div");
note.className = "hist-note";
note.textContent = "Interrupted";
hist.appendChild(note);
}
/** Render a captured webcam frame in the transcript (the camera tool result).
* @param {string} dataUrl */
_appendHistImage(dataUrl) {
const empty = this._chatHistory.querySelector(".chat-empty");
if (empty) empty.remove();
const el = document.createElement("div");
el.className = "hist-msg tool";
el.innerHTML = `<div class="hist-role">Snapshot</div><img class="hist-image" alt="Webcam snapshot sent to the model" />`;
const img = /** @type {HTMLImageElement} */ (el.querySelector("img"));
img.src = dataUrl;
this._chatHistory.appendChild(el);
this._scrollToBottom();
}
/**
* Reset all streaming bookkeeping for session start / teardown. Pass
* `dismiss` to also fade any bubbles still on screen.
* @param {{ dismiss?: boolean }} [opts]
*/
reset(opts) {
if (opts?.dismiss) {
if (this._activeUserBubble) this._dismissBubble(this._activeUserBubble);
for (const { bubble } of this._asstByResp.values()) this._dismissBubble(bubble);
}
this._userHistByItem.clear();
this._activeUserBubble = null;
this._activeUserItemId = "";
this._asstByResp.clear();
}
// ── Client event handlers ─────────────────────────────────────────────────
/**
* A streamed transcript delta (user or assistant).
* @param {{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }} d
*/
onTranscript(d) {
if (DEBUG) console.debug(`[ui] transcript role=${d.role} partial=${d.partial} item=${d.itemId} resp=${d.responseId} text=${JSON.stringify(d.text)}`);
if (d.role === "user") {
// Group by item_id: a speculative continuation reuses the same id, so it
// updates the same row/bubble. A missing id falls back to the active item
// (same utterance) or a fresh unique key, never a shared sentinel that
// would collapse distinct turns into one row.
const id = d.itemId || this._activeUserItemId || `_u${++this._anonSeq}`;
const text = d.text;
let hist = this._userHistByItem.get(id);
if (!hist) {
hist = this._appendHistMsg("user", text, d.partial);
this._userHistByItem.set(id, hist);
} else {
this._updateHistMsg(hist, text, d.partial);
}
// One ephemeral bubble per active item. Purely timer-based: the timer is
// refreshed on every delta, so it stays while the user keeps talking and
// fades a few seconds after they stop — no dependency on a response ever
// arriving, so it can never get stuck.
if (this._activeUserItemId !== id || !this._activeUserBubble) {
this._activeUserBubble = this._spawnBubble("user", text);
this._activeUserItemId = id;
} else {
this._updateBubbleText(this._activeUserBubble, text);
}
this._bumpDismiss(this._activeUserBubble, 6000);
this._markUnread();
} else if (d.role === "assistant") {
// Assistant transcript arrives once, as the full text, keyed by
// response_id so a cancelled speculative response can be removed later. A
// missing id gets a unique key so two id-less replies never collide.
const rid = d.responseId || `_a${++this._anonSeq}`;
const entry = this._asstByResp.get(rid);
if (!entry) {
const bubble = this._spawnBubble("assistant", d.text);
this._asstByResp.set(rid, { bubble, hist: this._appendHistMsg("assistant", d.text, false) });
this._bumpDismiss(bubble);
} else {
this._updateBubbleText(entry.bubble, d.text);
this._updateHistMsg(entry.hist, d.text, false);
this._bumpDismiss(entry.bubble);
}
this._markUnread();
}
}
/**
* A response closed (completed or cancelled).
* @param {{ responseId: string; status: string; audible?: boolean; transcript?: string }} detail
*/
onResponseFinished(detail) {
const { responseId, status, audible, transcript } = detail;
if (DEBUG) console.debug(`[ui] response-finished resp=${responseId} status=${status} audible=${audible} known=${this._asstByResp.has(responseId)}`);
// Without an id we can't target a specific response; the bubble will
// auto-dismiss on its own timer regardless.
if (!responseId) return;
const entry = this._asstByResp.get(responseId);
if (status === "cancelled") {
// Keep every transcript that was received — mark it interrupted rather
// than erasing it. If the `*.transcript.done` never fired, build the row
// from the text carried in response.done.
let hist = entry?.hist ?? null;
if (!hist && transcript) {
hist = this._appendHistMsg("assistant", transcript, false);
} else if (hist && transcript) {
this._updateHistMsg(hist, transcript, false);
}
if (hist) this._markHistInterrupted(hist);
this._asstByResp.delete(responseId);
return;
}
// Any other terminal close (completed / failed / incomplete / …): just
// release the map entry. The bubble already auto-dismisses on its timer and
// the history row persists as the conversation log. Crucially we do NOT
// touch user state here — that lifecycle is fully independent.
this._asstByResp.delete(responseId);
}
/** The model called a tool — show an ephemeral "running" bubble.
* @param {string} name */
onToolCall(name) {
this._bumpDismiss(this._spawnBubble("tool", name));
this._markUnread();
}
/** The tool finished — append its call+result row (and any captured image).
* @param {string} name @param {string} argsJson @param {string} output @param {string} [image] */
onToolResult(name, argsJson, output, image) {
this._appendHistTool(name, argsJson, output);
if (image) this._appendHistImage(image); // show the captured frame below the call
this._markUnread();
}
}

37
demo/ui/dom.js Normal file
View File

@ -0,0 +1,37 @@
// @ts-check
/** Small shared helpers used across the UI modules: a strict query selector,
* HTML escaping for text we drop into innerHTML, error-string trimming, and
* the opt-in debug flag. */
/** Opt-in tracing: `localStorage.setItem("s2s.debug", "1")` then reload. */
export const DEBUG = (() => {
try {
return localStorage.getItem("s2s.debug") === "1";
} catch {
return false;
}
})();
/**
* Query a single element, throwing if it's missing (so a broken selector fails
* loudly at startup rather than as a later null-deref).
* @template {HTMLElement} T
* @param {string} selector
* @returns {T}
*/
export function $(selector) {
const el = document.querySelector(selector);
if (!el) throw new Error(`Missing element: ${selector}`);
return /** @type {T} */ (el);
}
/** @param {string} s @returns {string} */
export function escHtml(s) {
return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
}
/** Trim a long error message to fit the orb caption. @param {string} text */
export function truncateError(text) {
if (text.length <= 90) return text;
return text.slice(0, 87) + "…";
}

171
demo/worklets/audio-playback.js Normal file
View File

@ -0,0 +1,171 @@
// @ts-check
/**
* AudioWorkletProcessor that plays back Float32 mono samples received from
* the main thread, upsampling whatever incoming rate the server uses
* (typically 24 kHz PCM16) to the AudioContext rate (typically 48 kHz).
*
* Lifecycle / messaging:
*
* main -> worklet:
* { kind: "config", inputRate: 24000 } one-shot at startup
* { kind: "audio", samples: Float32Array } (transferable) per chunk
* { kind: "clear" } wipe queue (barge-in)
*
* worklet -> main:
* { kind: "stats", queuedMs, played } every ~250 ms
* { kind: "underrun" } every time the queue
* runs dry mid-playback
*
* Underrun strategy: output silence. We do NOT hold the last sample (that
* tends to produce audible clicks/buzzes when long gaps appear between
* TTS chunks). A short ramp-out + ramp-in at boundaries would be nicer but
* the server's 30 ms cadence makes underruns visible only at end of turn.
*/
const STATS_INTERVAL_FRAMES = 12000;
const FADE_FRAMES = 32;
class AudioPlaybackProcessor extends AudioWorkletProcessor {
constructor() {
super();
this._inputRate = 24000;
this._stepRatio = this._inputRate / sampleRate;
this._queue = [];
this._readIdx = 0;
this._fracPos = 0;
this._playing = false;
this._framesSinceStats = 0;
this._totalPlayed = 0;
this._fadeIn = 0;
this._fadeOut = 0;
this._lastSample = 0;
this.port.onmessage = (e) => {
const data = e.data;
if (!data || typeof data !== "object") return;
switch (data.kind) {
case "config":
if (typeof data.inputRate === "number" && data.inputRate > 0) {
this._inputRate = data.inputRate;
this._stepRatio = this._inputRate / sampleRate;
}
break;
case "audio":
if (data.samples instanceof Float32Array && data.samples.length > 0) {
this._queue.push(data.samples);
if (!this._playing) {
this._playing = true;
this._fadeIn = FADE_FRAMES;
this._fadeOut = 0;
}
}
break;
case "clear":
this._queue.length = 0;
this._readIdx = 0;
this._fracPos = 0;
this._fadeOut = FADE_FRAMES;
break;
}
};
}
_queuedSamples() {
let total = -this._readIdx;
for (const buf of this._queue) total += buf.length;
return Math.max(0, total);
}
/** Linear-interp read at the current fractional position. */
_readInterpolated() {
if (this._queue.length === 0) return null;
const head = this._queue[0];
const idx = this._readIdx;
const frac = this._fracPos;
let a = head[idx];
let b;
if (idx + 1 < head.length) {
b = head[idx + 1];
} else if (this._queue.length > 1) {
b = this._queue[1][0];
} else {
b = a;
}
return a + (b - a) * frac;
}
/** Advance the read position by `stepRatio`; pop consumed buffers. */
_advance() {
this._fracPos += this._stepRatio;
while (this._fracPos >= 1) {
this._fracPos -= 1;
this._readIdx += 1;
}
while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
this._readIdx -= this._queue[0].length;
this._queue.shift();
}
}
process(_, outputs) {
const channels = outputs[0];
if (!channels || channels.length === 0) return true;
const out = channels[0];
const stereo = channels.length > 1 ? channels[1] : null;
for (let i = 0; i < out.length; i++) {
let sample = 0;
if (this._playing) {
const v = this._readInterpolated();
if (v === null) {
// Underrun: try to ramp out cleanly to avoid clicks.
sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES);
this._lastSample = sample;
if (Math.abs(sample) < 1e-4) {
this._playing = false;
this._lastSample = 0;
this.port.postMessage({ kind: "underrun" });
}
} else {
sample = v;
this._lastSample = v;
this._advance();
}
if (this._fadeIn > 0) {
const gain = 1 - this._fadeIn / FADE_FRAMES;
sample *= gain;
this._fadeIn -= 1;
}
if (this._fadeOut > 0) {
const gain = this._fadeOut / FADE_FRAMES;
sample *= gain;
this._fadeOut -= 1;
if (this._fadeOut === 0) {
this._playing = false;
this._lastSample = 0;
}
}
this._totalPlayed += 1;
}
out[i] = sample;
if (stereo) stereo[i] = sample;
}
this._framesSinceStats += out.length;
if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) {
this._framesSinceStats = 0;
const queuedSamples = this._queuedSamples();
const queuedMs = (queuedSamples / this._inputRate) * 1000;
this.port.postMessage({ kind: "stats", queuedMs, played: this._totalPlayed });
}
return true;
}
}
registerProcessor("audio-playback", AudioPlaybackProcessor);

159
demo/worklets/mic-capture.js Normal file
View File

@ -0,0 +1,159 @@
// @ts-check
/**
* AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz)
* down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it
* back to the main thread in fixed-size chunks.
*
* The Hugging Face speech-to-speech WebSocket route expects the
* `input_audio_buffer.append` payload at 16 kHz PCM16 mono.
*
* Design notes:
* - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a
* cheap low-pass before decimating. Good enough for voice STT; we lose
* a tiny bit of >8 kHz content which the pipeline discards anyway.
* - Output frames are emitted at the cadence dictated by `chunkMs`
* (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime
* server batches incoming audio so the cadence is flexible; 20-100 ms
* is the sweet spot.
* - Float -> Int16 saturates to [-1, 1] before scaling.
* - Optional noise gate: per-chunk RMS decides open/closed against a
* threshold; the gain ramps (fast attack, hold, slow release) so word
* onsets aren't clipped and quiet tails don't click. The gate only
* affects the audio we SEND; the main-thread visualiser taps the raw
* mic separately. We post the chunk RMS up every frame so the Settings
* mic meter can show the live level against the threshold.
*/
const TARGET_RATE = 16000;
const DEFAULT_CHUNK_MS = 40;
// Gate envelope timing (fixed; only the threshold is user-tunable).
const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive
const GATE_HOLD_MS = 250; // stay open this long after the level drops back under
const GATE_RELEASE_MS = 80; // then fade closed over this long (no click)
class MicCaptureProcessor extends AudioWorkletProcessor {
constructor(options) {
super();
const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS;
this._inputRate = sampleRate;
this._ratio = this._inputRate / TARGET_RATE;
this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000);
this._scratch = new Float32Array(0);
this._decimated = new Float32Array(this._chunkSamples16k);
this._enabled = true;
// Noise gate state. Disabled by default (pure passthrough).
this._gateEnabled = false;
this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open
this._gateGain = 1; // smoothed gain currently applied
this._holdRemaining = 0; // samples left before the gate may start closing
this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE));
this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE));
this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE);
this.port.onmessage = (e) => {
const data = e.data;
if (data?.kind === "enable") this._enabled = !!data.value;
else if (data?.kind === "gate") {
this._gateEnabled = !!data.enabled;
// dB -> linear amplitude. When off, threshold 0 keeps the gate open.
this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0;
}
};
}
/**
* Append `incoming` to the internal scratch buffer, then emit as many
* full output chunks as we have material for.
* @param {Float32Array} incoming
*/
_ingest(incoming) {
if (incoming.length === 0) return;
const next = new Float32Array(this._scratch.length + incoming.length);
next.set(this._scratch, 0);
next.set(incoming, this._scratch.length);
this._scratch = next;
this._maybeEmit();
}
_maybeEmit() {
const r = this._ratio;
const n = this._chunkSamples16k;
const needIn = Math.ceil(n * r);
const dec = this._decimated;
while (this._scratch.length >= needIn) {
// 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter.
let sumSq = 0;
if (Math.abs(r - 3) < 1e-6) {
// 48 kHz -> 16 kHz fast path with boxcar lowpass.
for (let i = 0; i < n; i++) {
const idx = i * 3;
const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3;
dec[i] = s;
sumSq += s * s;
}
} else {
// Generic path: linear interpolation. Slower but works at any rate
// (e.g. some Windows boxes report sampleRate=44100).
for (let i = 0; i < n; i++) {
const srcPos = i * r;
const idx = Math.floor(srcPos);
const frac = srcPos - idx;
const a = this._scratch[idx];
const b = this._scratch[idx + 1] ?? a;
const s = a + (b - a) * frac;
dec[i] = s;
sumSq += s * s;
}
}
const rms = Math.sqrt(sumSq / n);
// 2. Decide the gate target for this chunk, then ramp sample-by-sample.
let target = 1;
if (this._gateEnabled) {
if (rms >= this._thresholdLin) {
this._holdRemaining = this._holdSamples; // re-arm the hold
} else if (this._holdRemaining > 0) {
this._holdRemaining -= n; // coasting through the hold window
} else {
target = 0;
}
}
// 3. Apply the (smoothed) gain and pack to Int16.
const out = new Int16Array(n);
let gain = this._gateGain;
for (let i = 0; i < n; i++) {
const coef = target > gain ? this._attackCoef : this._releaseCoef;
gain = target + (gain - target) * coef;
const s = dec[i] * gain;
const clamped = s < -1 ? -1 : s > 1 ? 1 : s;
out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
}
this._gateGain = gain;
// Shift the scratch buffer to keep only the trailing unused samples.
const consumed = Math.floor(n * r);
this._scratch = this._scratch.slice(consumed);
// Live input level for the Settings meter (raw RMS, pre-gate).
this.port.postMessage({ kind: "level", rms });
if (this._enabled) {
this.port.postMessage(out.buffer, [out.buffer]);
}
// When disabled (mic muted) we silently consume input so the worklet
// stays alive and the buffer never grows unbounded.
}
}
process(inputs) {
const input = inputs[0];
if (!input || input.length === 0 || !input[0]) return true;
const mono = input[0];
if (mono.length > 0) this._ingest(mono);
return true;
}
}
registerProcessor("mic-capture", MicCaptureProcessor);

57
demo/ws/codec.js Normal file
View File

@ -0,0 +1,57 @@
// @ts-check
/**
* Pure, stateless helpers for the WebSocket realtime client: base64 <-> PCM
* conversion for the audio frames on the wire, transcript extraction from a
* `response.done` payload, and a tiny URL helper. Kept separate from the client
* so the protocol/state logic stays readable.
*/
/** @param {string} url */
export function trimTrailingSlash(url) {
return url.endsWith("/") ? url.slice(0, -1) : url;
}
/**
* Pull the assistant transcript out of a `response.done` payload. The text
* lives in `response.output[].content[].transcript` (audio) or `.text`. Used as
* the source of truth for interrupted replies, where the dedicated
* `*.transcript.done` event may never arrive.
* @param {any} response
* @returns {string}
*/
export function extractResponseTranscript(response) {
const output = response?.output;
if (!Array.isArray(output)) return "";
/** @type {string[]} */
const parts = [];
for (const item of output) {
for (const part of item?.content ?? []) {
const text = part?.transcript ?? part?.text;
if (typeof text === "string" && text.trim()) parts.push(text.trim());
}
}
return parts.join(" ").trim();
}
/** @param {ArrayBuffer} buf */
export function base64FromArrayBuffer(buf) {
const bytes = new Uint8Array(buf);
// Chunked encoding so we don't blow up the call stack on long buffers.
let binary = "";
const chunk = 0x8000;
for (let i = 0; i < bytes.length; i += chunk) {
binary += String.fromCharCode.apply(null, /** @type {number[]} */ (
/** @type {unknown} */ (bytes.subarray(i, i + chunk))
));
}
return btoa(binary);
}
/** @param {string} b64 */
export function base64ToBytes(b64) {
const binary = atob(b64);
const len = binary.length;
const out = new Uint8Array(len);
for (let i = 0; i < len; i++) out[i] = binary.charCodeAt(i);
return out;
}

98
demo/ws/orb-visualizer.js Normal file
View File

@ -0,0 +1,98 @@
// @ts-check
/**
* Orb spectrum visualiser. Each animation frame it reads two AnalyserNodes (the
* mic input and the TTS output) and maps the low-frequency speech energy onto
* the orb's CSS custom properties:
* - `--bar0`..`--bar4` the 5-band level meter
* - `--ai-audio-level` the global "Reachy talks" glow / scale pulse
*
* The bottom of the FFT is where speech energy lives, so the band edges stay
* low — that keeps the bars dancing on voice rather than on noise. While the AI
* is speaking we source the bars from the OUTPUT analyser so the orb pulses with
* Reachy's voice instead of sitting dead while the user is silent.
*/
// Exported so the client can size its AnalyserNodes to match our buffer.
export const VIS_FFT_SIZE = 256;
const VIS_BAND_COUNT = 5;
const VIS_BAND_EDGES = [2, 5, 9, 16, 28, 52];
const VIS_ATTACK = 0.6; // weight for new sample on upswing (snappy)
const VIS_RELEASE = 0.18; // weight for new sample on decay (gentle fade)
export class OrbVisualiser {
/**
* @param {AnalyserNode} micAnalyser
* @param {AnalyserNode} outAnalyser
* @param {() => boolean} isAiSpeaking Source the bars from the AI output when
* true, otherwise from the mic.
*/
constructor(micAnalyser, outAnalyser, isAiSpeaking) {
this._mic = micAnalyser;
this._out = outAnalyser;
this._isAiSpeaking = isAiSpeaking;
this._buf = new Uint8Array(micAnalyser.frequencyBinCount);
this._bands = new Float32Array(VIS_BAND_COUNT);
this._aiLevel = 0;
/** @type {number | null} */
this._frame = null;
}
/** Begin the rAF loop (idempotent). */
start() {
if (this._frame !== null) return;
const root = document.documentElement;
const tick = () => {
this._frame = requestAnimationFrame(tick);
this._update(root);
};
this._frame = requestAnimationFrame(tick);
}
/** Stop the loop and clear the CSS vars so the orb returns to rest. */
stop() {
if (this._frame !== null) {
cancelAnimationFrame(this._frame);
this._frame = null;
}
const root = document.documentElement;
for (let i = 0; i < VIS_BAND_COUNT; i++) root.style.removeProperty(`--bar${i}`);
root.style.removeProperty("--ai-audio-level");
}
/** @param {HTMLElement} root */
_update(root) {
// Mic bars: split FFT into 5 log-ish bands, smooth, write CSS vars.
const source = this._isAiSpeaking() ? this._out : this._mic;
source.getByteFrequencyData(this._buf);
for (let b = 0; b < VIS_BAND_COUNT; b++) {
const lo = VIS_BAND_EDGES[b];
const hi = VIS_BAND_EDGES[b + 1];
let sum = 0;
let n = 0;
for (let i = lo; i < hi && i < this._buf.length; i++) {
sum += this._buf[i];
n += 1;
}
const target = n > 0 ? sum / (n * 255) : 0;
const prev = this._bands[b];
const k = target > prev ? VIS_ATTACK : VIS_RELEASE;
const next = prev + (target - prev) * k;
this._bands[b] = next;
root.style.setProperty(`--bar${b}`, next.toFixed(3));
}
// Global AI audio level: peak of the output analyser, used by the CSS to
// make the orb's glow / scale react to Reachy's voice.
this._out.getByteFrequencyData(this._buf);
let peak = 0;
const limit = Math.min(this._buf.length, VIS_BAND_EDGES[VIS_BAND_COUNT]);
for (let i = 0; i < limit; i++) {
if (this._buf[i] > peak) peak = this._buf[i];
}
const aiTarget = peak / 255;
const k = aiTarget > this._aiLevel ? VIS_ATTACK : VIS_RELEASE;
this._aiLevel = this._aiLevel + (aiTarget - this._aiLevel) * k;
root.style.setProperty("--ai-audio-level", this._aiLevel.toFixed(3));
}
}

1108
demo/ws/s2s-ws-client.js Normal file
View File

File diff suppressed because it is too large Load Diff

72
docker-compose.yml Normal file
View File

@ -0,0 +1,72 @@
---
services:
llama:
image: ghcr.io/ggml-org/llama.cpp:server-cuda
command:
- -hf
- ggml-org/gemma-4-E4B-it-GGUF
- -np
- "2"
- -c
- "65536"
- -fa
- "on"
- --swa-full
- --host
- 0.0.0.0
- --port
- "8080"
ports:
- 8080:8080/tcp
volumes:
- ./cache/:/root/.cache/
deploy:
resources:
reservations:
devices:
- driver: nvidia
device_ids: ['0']
capabilities: [gpu]
pipeline:
depends_on:
- llama
build:
context: .
dockerfile: ${DOCKERFILE:-Dockerfile}
command:
- speech-to-speech
- --mode
- socket
- --recv_host
- 0.0.0.0
- --send_host
- 0.0.0.0
- --llm_backend
- responses-api
- --model_name
- ggml-org/gemma-4-E4B-it-GGUF
- --responses_api_base_url
- http://llama:8080/v1
- --responses_api_api_key
- ""
- --init_chat_role
- system
- --init_chat_prompt
- "You are a helpful assistant"
expose:
- 12345/tcp
- 12346/tcp
ports:
- 12345:12345/tcp
- 12346:12346/tcp
volumes:
- ./cache/:/root/.cache/
deploy:
resources:
reservations:
devices:
- driver: nvidia
device_ids: ['0']
capabilities: [gpu]

339
docs/PROJECT_OVERVIEW.es.md Normal file
View File

@ -0,0 +1,339 @@
# speech-to-speech — Descripción del Proyecto
> **Paquete**: `speech-to-speech`
> **Versión**: `0.2.11`
> **Autor**: Hugging Face
> **Licencia**: Apache-2.0
> **Python**: 3.10 – 3.12
> **Lema**: Pipeline Speech-to-Speech de baja latencia end-to-end para construir agentes de voz en tiempo real.
---
## 1. ¿Qué es `speech-to-speech`?
`speech-to-speech` es una **pipeline de audio modular y de baja latencia** que convierte la entrada hablada del usuario en una respuesta hablada encadenando cuatro etapas de IA:
**VAD → STT → LLM → TTS**.
De fábrica expone un **endpoint WebSocket compatible con el protocolo OpenAI Realtime** (`ws://host:port/v1/realtime`),
por lo que cualquier navegador o SDK que sepa hablar el protocolo Realtime de OpenAI puede conectarse y empezar a tener
conversaciones naturales en cuestión de segundos — sin escribir una sola línea de código en el servidor.
La pipeline está diseñada para ser **agnóstica del backend en cada etapa**: puedes intercambiar distintos modelos
de STT / LLM / TTS según el hardware disponible (NVIDIA CUDA, Apple Silicon MLX o solo CPU), los idiomas de tus usuarios
y el compromiso entre latencia y calidad que necesites.
Un subsistema opcional integrado de **RAG (Retrieval Augmented Generation) del lado servidor** permite que las
respuestas del LLM se basen en tu propia base de conocimientos privada (documentos Markdown / TXT / JSONL),
mientras permanece 100 % transparente para el cliente. Las actualizaciones dinámicas de la base de conocimientos
se exponen en el mismo servidor HTTP mediante una API REST de 11 endpoints.
---
## 2. Características principales
| Característica | Descripción |
|---|---|
| **Protocolo de voz en tiempo real** | Soporte nativo del WebSocket OpenAI `v1/realtime`: `session.update`, `response.create`, llamadas a funciones, deltas de audio, interrupciones. Compatible directamente con el SDK Realtime oficial y con los "playgrounds" web. |
| **4 modos de ejecución** | `local` (micrófono → altavoces), `socket` (IPC TCP), `websocket` (WS audio crudo), `realtime` (protocolo OpenAI — predeterminado). |
| **6 backends de STT intercambiables** | Whisper / Whisper-MLX / MLX-Audio-Whisper / Faster-Whisper / Parakeet TDT (predeterminado) / Paraformer. |
| **4 backends de LLM intercambiables** | `transformers` (local) · `mlx-lm` (Apple Silicon) · `responses-api` (endpoint OpenAI con tool-calling) · `chat-completions` (cualquier servidor `/v1/chat/completions` compatible con OpenAI). |
| **5 backends de TTS intercambiables** | ChatTTS · Facebook MMS · Pocket (muy pequeño, CPU) · Kokoro · Qwen3-TTS (predeterminado, voz personalizada de 1.7B). |
| **Multiplataforma** | NVIDIA CUDA (Linux), Apple Silicon MLX + MPS (macOS), fallback CPU en cada etapa. |
| **Pool de N pipelines aisladas** | `--num_pipelines N` ejecuta N sesiones independientes en paralelo (cada una con sus propios VAD / STT / LLM / TTS y estado de conversación). Ideal para despliegues pequeños con múltiples conexiones simultáneas. |
| **VAD e interrupción integrados** | Detección de actividad vocal con umbral configurable; finalización del STT por silencio; el LLM se puede interrumpir mientras habla y se cancela de forma limpia mediante un `CancelScope`. |
| **Detección automática de idioma** | A través de `lingua-language-detector` para el idioma de la respuesta del asistente. |
| **Transcripción parcial en vivo** | Parakeet-TDT emite transcripciones parciales cada 500 ms para que los clientes muestren el texto "el usuario está hablando…". |
| **Soporte completo de tool-calling** | Con el backend `responses-api`: entrega en streaming de argumentos de función, tool calls paralelos, voces personalizadas sobre TTS — todo conforme a la superficie de la Responses API de OpenAI. |
| **Inyección RAG opcional en el servidor** | Hook transparente de recuperación en cada turno del LLM. Embeddings con Sentence-Transformers, persistencia del índice NPZ, top-k / umbral coseno configurables, inyección como mensaje del sistema o del usuario. Multilingüe por defecto (español / italiano / inglés listos para usar). |
| **API REST para base de conocimientos dinámica** | 11 endpoints para listar, buscar, hacer upsert, actualizar, eliminar y recargar contenidos de la KB en tiempo real — con persistencia entre reinicios mediante `kb/_dynamic.jsonl`. |
| **Configuración estructurada CLI / JSON** | Cada parámetro es un argumento `@dataclass` de HfArgumentParser con valores predeterminados razonables, o un fichero JSON de configuración completo que puedes pasar como único argumento. |
| **Empaquetado y flujo de publicación PyPI** | Configurado `uv build` + `twine check` + GitHub Actions (la etiqueta `vX.Y.Z` activa la subida). |
---
## 3. Arquitectura a vista de pájaro
Un turno único de conversación en modo **realtime** se ve así:
```
┌─────────────────────────────────────────────────────────┐
│ uvicorn + FastAPI │
│ ┌───────────────────────────────────────────────────┐ │
Mic / Navegador ──► WS │ │ RealtimeService (sesión + rutas + eventos) │ │
(audio entradas + │ │ └───────────────────────────────────────────────────┘ │
eventos) │ │ │
│ │ Pool de pipelines ─► PipelineUnit #1 ─► PipelineUnit #N│
│ └─────────────────────────────────────────────────────────┘
│ │
▼ ▼
┌──────────────────────────────────────────────┐
│ Una unidad de pipeline (una por usuario) │
│ │
│ 1. VAD ──► inicio / fin voz │
│ 2. STT (5 sabores) ──► último texto usuario│
│ │ │
│ ▼ │
│ ┌────────────────────────┐ │
│ │ HOOK BÚSQUEDA RAG │ ◄─── kb/_index.npz
│ │ (inyecta solo si ≥ N) │ + md/txt/jsonl
│ └────────────────────────┘ + _dynamic.jsonl
│ │ │
│ ▼ │
│ 3. LLM (4 sabores) ──► texto + tool calls│
│ │ │
│ ▼ │
│ 4. TTS (5 sabores) ──► PCM audio streaming│
└──────────────────────────────────────────────┘
│
▼
◄── WS audio / delta events
```
Invariantes clave:
- **Cada conexión = una `PipelineUnit`** (asignada de forma atómica; colas y estado aislados; si las N unidades están ocupadas, la conexión N+1 se rechaza).
- **La etapa LLM nunca ve audio crudo**. La pipeline solo envía buffers de texto al manejador del LLM, por lo que cambiar de proveedor o de modelo sigue siendo transparente.
- **La etapa RAG es un efecto lateral puro sobre el búfer de texto**: lee el texto más reciente del usuario, ejecuta un embedding + top-k por coseno + filtro por umbral, y antepone el conocimiento relevante como un mensaje adicional del sistema (o del usuario) — con una línea de registro clara para ver siempre exactamente qué se inyectó.
- **TTS en streaming** (todos los backends modernos): los deltas de audio se emiten a medida que se sintetizan, por lo que el primer byte de una respuesta llega al altavoz mucho antes de que el LLM termine de generar el texto.
---
## 4. Modos de ejecución (`--mode`)
Configurables mediante [ModuleArguments.mode](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/module_arguments.py#L11-L16)
(predeterminado: `realtime`).
| Modo | Entrada | Salida | Ideal para |
|---|---|---|---|
| `local` | Micrófono local vía `sounddevice` / `miniaudio` | Altavoces locales | Demos de escritorio, prototipado rápido en portátil. |
| `socket` | Chunks PCM crudos en socket TCP | PCM crudos por socket TCP | Sistemas legacy / embebidos, transporte personalizado. |
| `websocket` | PCM crudo sobre endpoint WS | PCM crudo sobre WS | Frontend a medida mínimo que solo envía audio. |
| `realtime` | WS con protocolo OpenAI Realtime (`/v1/realtime`) | Mismo protocolo + API REST RAG en el mismo servidor HTTP | **Producción e integración con SDK.** Todos los clientes que soportan la Realtime API (OpenAI SDK, playgrounds web, wrappers Swift / Kotlin / JS) se conectan aquí. |
---
## 5. Backends soportados
### 5.1 STT — Speech to Text
| Nombre `--stt` | Familia de modelos | Modelo predeterminado | Ventajas hardware |
|---|---|---|---|
| `whisper` | HuggingFace Transformers Whisper | `distil-whisper/distil-large-v3` | CUDA / CPU |
| `whisper-mlx` | MLX Whisper | `mlx-community/whisper-large-v3-turbo` | Apple Silicon (macOS) |
| `mlx-audio-whisper` | Apple `mlx-audio` | `mlx-community/whisper-large-v3-turbo` | Apple Silicon |
| `faster-whisper` | Whisper cuantizada CTranslate2 | `tiny.en` | CPU con latencia muy baja |
| `parakeet-tdt` | **(predeterminado)** HuggingFace Parakeet TDT | `parakeet-tdt-1.1b` | Streaming amigable + **transcripción parcial en vivo** sobre CUDA |
| `paraformer` | Alibaba Paraformer | `paraformer-zh` | Despliegues solo chino |
### 5.2 LLM — Modelo de lenguaje
| `--llm_backend` | Descripción | Modelo predeterminado |
|---|---|---|
| `transformers` | Generación local HF Transformers en proceso | `Qwen/Qwen3-4B-Instruct-2507` |
| `mlx-lm` | Inferencia local cuantizada 4-bit / 8-bit en Apple Silicon | `mlx-community/...` |
| `responses-api` | **(predeterminado)** Endpoint remoto compatible con OpenAI Responses API (superficie completa de tool-calling + argumentos de función en streaming) | `gpt-5.4-mini` |
| `chat-completions` | Cualquier remoto `/v1/chat/completions` compatible con OpenAI — enchufa vLLM, TGI, Ollama, servidor llama.cpp, SGLang, TabbyAPI, etc. | (auto-detecta vía `base_url + /v1/models`) |
### 5.3 TTS — Text to Speech
| Nombre `--tts` | Backend | Modelo predeterminado | Puntos fuertes |
|---|---|---|---|
| `chatTTS` | ChatTTS (grupo opcional `chattts`) | — | Muy conversacional, inglés + chino, prosodia muy expresiva |
| `facebookMMS` | Facebook MMS (grupo opcional `facebook-mms`) | `facebook/mms-tts-eng` | Ultraligero, cubre más de 1.100 idiomas |
| `pocket` | PocketTTS (pequeño CPU) | — | Sin dependencias en dispositivo, ideal para embebidos / poca RAM |
| `kokoro` | Kokoro TTS (grupo opcional `kokoro`) | — | Estado del arte de TTS neural en inglés y japonés |
| `qwen3` | **(predeterminado)** Qwen3-TTS vía GGML `faster-qwen3-tts` | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Multilingüe, soporta clonación por voz de referencia, tasa de token 12 Hz → latencia ultrabaja. |
Todos los backends de TTS se acceden a través de una interfaz de streaming común, por lo que el resto de la pipeline es agnóstica al backend.
---
## 6. RAG del lado servidor (complemento opcional)
El subsistema RAG está completamente documentado en:
- 🌍 Guía lógica IT/EN: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md)
- 🇪🇸 Traducción al español: [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md)
En resumen:
```
$ ./start_pipeline_rag.sh # arranca pipeline + API REST RAG
RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/.../kb
RAG: Índice cargado desde disco: 9 chunk (shape=(9, 384)).
RAG: Activo. kb=... top_k=3 umbral=0.250 inject_as=system idioma=es
RAG HTTP API montata su prefix='/v1/rag'
```
Puntos clave:
- **Cero cambios en el cliente** — la inyección ocurre en el servidor, dentro del búfer de chat del LLM, antes de cada turno.
- **Modelos de embedding intercambiables** con un modelo multilingüe predeterminado (`paraphrase-multilingual-MiniLM-L12-v2`, 384 dims) y alternativas recomendadas para español-only y para bases de conocimientos muy grandes.
- **Persistencia del índice NPZ** — se reconstruye solo si el contenido cambió. Soporte de `--rag_force_rebuild`.
- **Tres formatos de fuente**: Markdown / TXT con chunking recursivo automático (tamaño + solapamiento configurables), o JSONL para chunks artesanales.
- **Comportamiento de recuperación configurable**: `--rag_top_k`, `--rag_threshold`, `--rag_inject_as (system/user)`, `--rag_language (es/it/en)`.
- **API REST dinámica thread-safe** (11 endpoints): `/status`, `/sources`, `/chunks`, `/chunks/list`, `/chunks/update`, `/upsert/document`, `/search`, `/add/document`, `/add/chunks`, `/remove`, `/reload`.
- **Persistencia entre reinicios**: las llamadas a la API con `persist=true` se adjuntan atómicamente a `kb/_dynamic.jsonl` y se re-indexan automáticamente en el siguiente arranque.
- **Patrón CRUD idempotente**: `/upsert/document` gestiona crear / reemplazar / eliminar (`text=""`) de forma atómica para que los clientes solo necesiten una URL.
- **Falla con elegancia**: cualquier excepción de recuperación se registra pero nunca rompe el turno de conversación.
---
## 7. Arranque rápido
### 7.1 Instalación (instalación editable estilo PyPI)
```bash
git clone https://github.com/huggingface/speech-to-speech.git
cd speech-to-speech
# Paquete base (incluye STT parakeet-tdt + TTS qwen3 por defecto)
uv pip install -e .
# Opcional: añade el subsistema RAG para recuperación en la KB
uv pip install -e ".[rag]"
# Opcional: selecciona los extras de TTS / STT que quieras
uv pip install -e ".[rag,chattts,kokoro,faster-whisper]"
```
### 7.2 Arrancar en modo Realtime (predeterminado) con un LLM externo
Esta es la configuración más común: el LLM corre en un servidor remoto compatible con OpenAI
(p.ej. `http://127.0.0.1:8001/v1` con vLLM o TGI), STT + TTS corren localmente sobre CUDA / MLX.
Crea un pequeño *wrapper* de shell (véase [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) para la plantilla completa):
```bash
#!/bin/bash
set -euo pipefail
LLM_BASE_URL="${LLM_BASE_URL:-http://127.0.0.1:8001/v1}"
LLM_MODEL="${LLM_MODEL:-Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4}"
LLM_API_KEY="${LLM_API_KEY:-placeholder}"
TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-Base}"
STT_MODEL="${STT_MODEL:-distil-whisper/distil-large-v3}"
cd /home/azurian/speech-to-speech
uv run speech-to-speech \
--mode realtime \
--ws_host 0.0.0.0 \
--ws_port 12345 \
--num_pipelines 2 \
\
--llm_backend chat-completions \
--chat_completions_handler_base_url "$LLM_BASE_URL" \
--chat_completions_handler_model_name "$LLM_MODEL" \
--chat_completions_handler_api_key "$LLM_API_KEY" \
\
--stt whisper \
--whisper_stt_model_name "$STT_MODEL" \
\
--tts qwen3 \
--qwen3_tts_model_name "$TTS_MODEL"
```
Ejecútalo y después conecta cualquier cliente Realtime de OpenAI a:
```
ws://<host>:12345/v1/realtime
```
### 7.3 Misma pipeline + RAG activado
Añade estos flags (véase §6 para la referencia completa y el script auxiliar [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh)):
```bash
--rag_enabled \
--rag_kb_path ./kb \
--rag_top_k 3 \
--rag_threshold 0.25 \
--rag_language es \
--rag_inject_as system
```
La API REST de RAG aparece inmediatamente en el mismo servidor HTTP:
```bash
curl http://127.0.0.1:12345/v1/rag/status | jq
```
---
## 8. Estructura del proyecto (puntos destacados)
```
speech-to-speech/
├── pyproject.toml ← metadatos del paquete + grupos de deps opcionales
├── LICENSE ← Apache-2.0
├── start_pipeline.sh ← script de arranque de referencia
├── start_pipeline_rag.sh ← script de arranque con RAG activado
│
├── src/speech_to_speech/
│ ├── s2s_pipeline.py ← punto de entrada (main), constructor de pipeline, pool realtime
│ ├── baseHandler.py
│ ├── chat.py
│ ├── pipeline/ ← tipos de cola, CancelScope, tipos de handler
│ │
│ ├── arguments_classes/ ← args HfArgumentParser @dataclass (uno por handler)
│ │ ├── module_arguments.py ← mode / stt / tts / llm_backend / num_pipelines
│ │ ├── rag_arguments.py ← 12 flags específicos de RAG
│ │ └── ... (Whisper, Qwen3, ChatTTS, VAD, …)
│ │
│ ├── STT/ ← seis manejadores STT
│ ├── TTS/ ← cinco manejadores TTS
│ ├── LLM/
│ │ ├── base_openai_compatible_language_model.py ← Hook de inyección RAG + tool-calling completo
│ │ ├── chat_completions_language_model.py
│ │ ├── responses_api_language_model.py
│ │ └── language_model.py ← manejadores locales transformers / mlx-lm
│ │
│ ├── RAG/
│ │ ├── retriever.py ← singleton, embeddings, NPZ, search, CRUD completo
│ │ └── router.py ← 11 endpoints FastAPI /v1/rag
│ │
│ └── api/openai_realtime/
│ ├── websocket_router.py ← FastAPI + Realtime + montaje condicional RAG
│ ├── service.py ← Bucle de eventos Realtime + rutas de sesión / handlers
│ └── ...
│
├── kb/
│ ├── 01_faq_producto.md ← ejemplo FAQ en español (incluido en scaffold)
│ ├── 02_politicas_internas.md ← ejemplo políticas en español
│ ├── README.md ← guía de formatos
│ └── _dynamic.jsonl ← chunks añadidos por API (auto-generado)
│
└── docs/
├── RAG_SERVER_SIDE.md ← Guía RAG en profundidad IT/EN (más de 600 líneas)
└── RAG_SERVER_SIDE.es.md ← Traducción al español
```
---
## 9. Versionado y publicación
El repositorio trae una pipeline completa de lanzamiento a PyPI en
`.github/workflows/publish.yml`:
1. Sube la `version` en [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L7)
y `__version__` en [src/speech_to_speech/__init__.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/__init__.py).
2. Fusiona una PR de release que contenga solo esos dos cambios.
3. Etiqueta y sube:
```bash
git checkout main && git pull origin main
git tag -a vX.Y.Z -m "Release vX.Y.Z"
git push origin vX.Y.Z
```
4. El workflow ejecuta automáticamente `uv build` + `twine check --strict` + subida a PyPI.
Consulta [AGENTS.md](file:///home/azurian/speech-to-speech/AGENTS.md) para las reglas de release a nivel de repositorio.
---
## 10. Siguientes pasos
- Empieza con los scripts de arranque: [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) (base) y [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) (con RAG).
- Profundiza en el subsistema RAG: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md) / [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md).
- Ajusta los manejadores por etapa leyendo las clases de argumentos en [arguments_classes](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes) — cada flag tiene su ayuda en línea.
- Construye tu propio cliente Realtime apoyándote en [service.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/service.py) y [websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py).

337
docs/PROJECT_OVERVIEW.md Normal file
View File

@ -0,0 +1,337 @@
# speech-to-speech — Project Overview
> **Package**: `speech-to-speech`
> **Version**: `0.2.11`
> **Author**: Hugging Face
> **License**: Apache-2.0
> **Python**: 3.10 – 3.12
> **Tagline**: Low-latency end-to-end Speech-to-Speech pipeline for building realtime voice agents.
---
## 1. What is `speech-to-speech`?
`speech-to-speech` is a fully modular, **low-latency audio pipeline** that turns a user's spoken input into a spoken reply by chaining four AI stages together:
**VAD → STT → LLM → TTS**.
Out of the box it exposes an **OpenAI-compatible Realtime WebSocket endpoint** (`ws://host:port/v1/realtime`),
so any browser or SDK that knows how to speak the OpenAI Realtime Protocol can connect and start having natural voice conversations in seconds — without writing a single line of server code.
The pipeline is designed to be **backend-agnostic at every stage**: you can plug different STT / LLM / TTS models
depending on the available hardware (NVIDIA CUDA, Apple Silicon MLX, or CPU-only), the language(s) of your users,
and the latency / quality trade-off you need.
A built-in, optional **RAG (Retrieval Augmented Generation) server-side subsystem** lets the LLM answers be grounded
in your own private knowledge base (Markdown / TXT / JSONL documents), while staying 100 % transparent for the client.
Dynamic knowledge base updates are exposed over the same HTTP server via a 11-endpoint REST API.
---
## 2. Key Features
| Feature | Description |
|---|---|
| **Real-time voice protocol** | Native OpenAI `v1/realtime` WebSocket support: `session.update`, `response.create`, function calling, audio deltas, interruptions. Drop-in compatible with the official Realtime SDK and browser playgrounds. |
| **4 execution modes** | `local` (mic → speakers), `socket` (TCP IPC), `websocket` (raw audio WS), `realtime` (OpenAI protocol — default). |
| **6 pluggable STT backends** | Whisper / Whisper-MLX / MLX-Audio-Whisper / Faster-Whisper / Parakeet TDT (default) / Paraformer. |
| **4 pluggable LLM backends** | `transformers` (local) · `mlx-lm` (Apple Silicon) · `responses-api` (OpenAI tool-calling aware endpoint) · `chat-completions` (any OpenAI-compatible `/v1/chat/completions` server). |
| **5 pluggable TTS backends** | ChatTTS · Facebook MMS · Pocket (tiny CPU) · Kokoro · Qwen3-TTS (default, custom voice 1.7B). |
| **Multi-hardware ready** | NVIDIA CUDA (Linux), Apple Silicon MLX + MPS (macOS), CPU fallback for every stage. |
| **Pool of N isolated pipelines** | `--num_pipelines N` runs N independent sessions in parallel (each one with its own VAD / STT / LLM / TTS handlers and conversation state). Ideal for small multi-concurrent deployments. |
| **Built-in VAD & interruption** | Voice-activity detection with configurable threshold and streaming silence → STT finalization; the LLM can be interrupted mid-speech by the user and cancelled cleanly through a `CancelScope`. |
| **Automatic language detection** | Via `lingua-language-detector` for the assistant reply language. |
| **Live partial transcription** | Parakeet-TDT emits live partial transcriptions every 500 ms so clients can display "user is speaking …" text. |
| **Full tool-calling support** | With `responses-api` backend: streamed function argument delivery, parallel tool calls, custom voices over TTS — everything compliant with the OpenAI Responses API surface. |
| **Optional RAG server-side injection** | Transparent retrieval hook on every LLM turn. Sentence-Transformers embeddings, NPZ-index persistence, configurable top-k / cosine threshold, system or user-message injection. Multilingual by default (Spanish / Italian / English out of the box). |
| **Dynamic Knowledge Base REST API** | 11 endpoints to list, search, upsert, update, remove, and reload KB contents at runtime — including cross-restart persistence through `kb/_dynamic.jsonl`. |
| **Structured CLI / JSON configuration** | Every parameter is a `@dataclass` HfArgumentParser argument with sane defaults, or a full JSON config file you can pass in as single argument. |
| **PyPI-ready packaging & release flow** | Configured `uv build` + `twine check` + GitHub Actions publish workflow (tag `vX.Y.Z` triggers upload). |
---
## 3. Architecture at a glance
A single conversation turn in **realtime** mode looks like this:
```
┌─────────────────────────────────────────────────────────┐
│ uvicorn + FastAPI │
│ ┌───────────────────────────────────────────────────┐ │
Mic / Browser ──► WS │ │ RealtimeService (session + routing + events) │ │
(audio in + │ │ └───────────────────────────────────────────────────┘ │
events) │ │ │
│ │ Pipeline pool ──► PipelineUnit #1 ──► PipelineUnit #N│
│ └─────────────────────────────────────────────────────────┘
│ │
▼ ▼
┌──────────────────────────────────────────────┐
│ Single pipeline unit (one per user) │
│ │
│ 1. VAD ──► speech start / end │
│ 2. STT (5 flavours) ──► last user text │
│ │ │
│ ▼ │
│ ┌────────────────────────┐ │
│ │ RAG SEARCH HOOK │ ◄─── kb/_index.npz
│ │ (inject only if ≥ N) │ + md/txt/jsonl
│ └────────────────────────┘ + _dynamic.jsonl
│ │ │
│ ▼ │
│ 3. LLM (4 flavours) ──► text reply + tools│
│ │ │
│ ▼ │
│ 4. TTS (5 flavours) ──► streamed PCM audio│
└──────────────────────────────────────────────┘
│
▼
◄── WS audio / delta events
```
Key invariants:
- **Each connection = one `PipelineUnit`** (allocated atomically; queues and state are isolated; if N units are busy, the `N+1`-th connection is rejected).
- **The LLM stage never sees raw audio**. The pipeline only forwards text buffers to the LLM handler, so model vendors and backend switches remain transparent.
- **The RAG stage is a pure side-effect on the text buffer**: it reads the most recent user text, runs an embedding + cosine top-k + threshold filter, and prepends matching knowledge as an extra system (or user) message — with a clear log line so you can always see exactly what was injected.
- **TTS streams** (all modern backends): audio deltas are emitted as they are synthesised, so the first byte of a reply reaches the speaker well before the LLM has finished generating text.
---
## 4. Execution modes (`--mode`)
Configurable via [ModuleArguments.mode](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/module_arguments.py#L11-L16)
(default: `realtime`).
| Mode | Input | Output | Best for |
|---|---|---|---|
| `local` | Local microphone via `sounddevice` / `miniaudio` | Local speakers | Desktop demos, quick prototyping on laptop. |
| `socket` | Raw PCM chunks on TCP socket | Raw PCM on TCP socket | Legacy / embedded setups, custom transport. |
| `websocket` | Raw PCM over WS endpoint | Raw PCM over WS | Minimal custom frontend that just pushes audio. |
| `realtime` | OpenAI Realtime Protocol WS (`/v1/realtime`) | Same protocol + RAG REST API on the same HTTP server | **Production & SDK integration.** All clients that support the Realtime API (OpenAI SDK, web playgrounds, Swift / Kotlin / JS wrappers) connect here. |
---
## 5. Supported backends
### 5.1 STT — Speech to Text
| `--stt` name | Model family | Default model | Device highlights |
|---|---|---|---|
| `whisper` | HuggingFace Transformers Whisper | `distil-whisper/distil-large-v3` | CUDA / CPU |
| `whisper-mlx` | MLX Whisper | `mlx-community/whisper-large-v3-turbo` | Apple Silicon (macOS) |
| `mlx-audio-whisper` | Apple `mlx-audio` | `mlx-community/whisper-large-v3-turbo` | Apple Silicon |
| `faster-whisper` | CTranslate2 Quantized Whisper | `tiny.en` | Very low latency CPU |
| `parakeet-tdt` | **(default)** HuggingFace Parakeet TDT | `parakeet-tdt-1.1b` | Streaming-friendly + **live partial transcription** on CUDA |
| `paraformer` | Alibaba Paraformer | `paraformer-zh` | Chinese-only deployments |
### 5.2 LLM — Language Model
| `--llm_backend` | Description | Default model |
|---|---|---|
| `transformers` | Local, process-bound HF Transformers generation | `Qwen/Qwen3-4B-Instruct-2507` |
| `mlx-lm` | Apple Silicon local 4-bit / 8-bit quantised inference | `mlx-community/...` |
| `responses-api` | **(default)** OpenAI Responses-API compatible remote endpoint (full tool-calling surface + streamed function args) | `gpt-5.4-mini` |
| `chat-completions` | Any OpenAI-compatible `/v1/chat/completions` remote — plug in vLLM, TGI, Ollama, llama.cpp server, SGLang, TabbyAPI, etc. | (auto-detects via `base_url + /v1/models`) |
### 5.3 TTS — Text to Speech
| `--tts` name | Backend | Default model | Strengths |
|---|---|---|---|
| `chatTTS` | ChatTTS (optional group `chattts`) | — | Highly conversational, English + Chinese, very expressive prosody |
| `facebookMMS` | Facebook MMS (optional group `facebook-mms`) | `facebook/mms-tts-eng` | Ultra-lightweight, covers 1,100+ languages |
| `pocket` | PocketTTS (small CPU) | — | Zero-dependency on-device, great for embedded / low-RAM |
| `kokoro` | Kokoro TTS (optional group `kokoro`) | — | State of the art English & Japanese neural TTS |
| `qwen3` | **(default)** Qwen3-TTS via `faster-qwen3-tts` GGML | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Multilingual, supports reference voice cloning, 12 Hz token rate → ultra-low latency. |
All TTS backends are reachable through a common streaming interface so the rest of the pipeline is backend-agnostic.
---
## 6. RAG Server-Side (optional add-on)
The RAG subsystem is fully documented in:
- 🌍 English / Spanish logic: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md)
- 🇪🇸 Spanish translation: [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md)
At a glance:
```
$ ./start_pipeline_rag.sh # starts pipeline + RAG REST API
RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/.../kb
RAG: Índice cargado desde disco: 9 chunk (shape=(9, 384)).
RAG: Activo. kb=... top_k=3 umbral=0.250 inject_as=system idioma=es
RAG HTTP API montata su prefix='/v1/rag'
```
Highlights:
- **Zero client changes** — injection happens server-side in the LLM chat buffer before every turn.
- **Pluggable embedding models** with a default multilingual model (`paraphrase-multilingual-MiniLM-L12-v2`, 384 dims) and recommended alternatives for Spanish-only and very-large-KB scenarios.
- **NPZ index persistence** — rebuilds only if content changed. Supports `--rag_force_rebuild`.
- **Three source formats**: Markdown / TXT with automatic recursive chunking (size + overlap configurable), or JSONL for hand-crafted chunks.
- **Configurable retrieval behaviour**: `--rag_top_k`, `--rag_threshold`, `--rag_inject_as (system/user)`, `--rag_language (es/it/en)`.
- **Thread-safe dynamic REST API** (11 endpoints): `/status`, `/sources`, `/chunks`, `/chunks/list`, `/chunks/update`, `/upsert/document`, `/search`, `/add/document`, `/add/chunks`, `/remove`, `/reload`.
- **Cross-restart persistence**: API calls with `persist=true` are appended atomically to `kb/_dynamic.jsonl` and automatically re-indexed on the next startup.
- **Idempotent CRUD pattern**: `/upsert/document` handles create / replace / delete (`text=""`) atomically so clients only need one URL.
- **Fails gracefully**: any retrieval exception is logged but never breaks the conversation turn.
---
## 7. Quick start
### 7.1 Install (PyPI-style editable install)
```bash
git clone https://github.com/huggingface/speech-to-speech.git
cd speech-to-speech
# Base package (includes default parakeet-tdt STT + qwen3 TTS)
uv pip install -e .
# Optional: add the RAG subsystem for server-side KB retrieval
uv pip install -e ".[rag]"
# Optional: select which TTS / STT extras you want to bring along
uv pip install -e ".[rag,chattts,kokoro,faster-whisper]"
```
### 7.2 Launch in Realtime mode (default) using an external LLM
This is the most common configuration: LLM runs on a remote OpenAI-compatible server
(e.g. `http://127.0.0.1:8001/v1` with vLLM or TGI), STT + TTS run locally on CUDA / MLX.
Create a small shell wrapper (see [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) for the full template):
```bash
#!/bin/bash
set -euo pipefail
LLM_BASE_URL="${LLM_BASE_URL:-http://127.0.0.1:8001/v1}"
LLM_MODEL="${LLM_MODEL:-Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4}"
LLM_API_KEY="${LLM_API_KEY:-placeholder}"
TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-Base}"
STT_MODEL="${STT_MODEL:-distil-whisper/distil-large-v3}"
cd /home/azurian/speech-to-speech
uv run speech-to-speech \
--mode realtime \
--ws_host 0.0.0.0 \
--ws_port 12345 \
--num_pipelines 2 \
\
--llm_backend chat-completions \
--chat_completions_handler_base_url "$LLM_BASE_URL" \
--chat_completions_handler_model_name "$LLM_MODEL" \
--chat_completions_handler_api_key "$LLM_API_KEY" \
\
--stt whisper \
--whisper_stt_model_name "$STT_MODEL" \
\
--tts qwen3 \
--qwen3_tts_model_name "$TTS_MODEL"
```
Run it, then connect any OpenAI Realtime client to:
```
ws://<host>:12345/v1/realtime
```
### 7.3 Same pipeline + RAG enabled
Append these flags (see §6 for the full reference and the helper script [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh)):
```bash
--rag_enabled \
--rag_kb_path ./kb \
--rag_top_k 3 \
--rag_threshold 0.25 \
--rag_language es \
--rag_inject_as system
```
The RAG REST API appears immediately on the same HTTP server:
```bash
curl http://127.0.0.1:12345/v1/rag/status | jq
```
---
## 8. Project layout (highlights)
```
speech-to-speech/
├── pyproject.toml ← package metadata + optional deps groups
├── LICENSE ← Apache-2.0
├── start_pipeline.sh ← reference launch script
├── start_pipeline_rag.sh ← launch script + RAG enabled
│
├── src/speech_to_speech/
│ ├── s2s_pipeline.py ← entry point (main), pipeline builder, realtime pool
│ ├── baseHandler.py
│ ├── chat.py
│ ├── pipeline/ ← queue types, CancelScope, handler types
│ │
│ ├── arguments_classes/ ← @dataclass HfArgumentParser args (one per handler)
│ │ ├── module_arguments.py ← mode / stt / tts / llm_backend / num_pipelines
│ │ ├── rag_arguments.py ← 12 RAG-specific flags
│ │ └── ... (Whisper, Qwen3, ChatTTS, VAD, …)
│ │
│ ├── STT/ ← six STT handlers
│ ├── TTS/ ← five TTS handlers
│ ├── LLM/
│ │ ├── base_openai_compatible_language_model.py ← RAG injection hook + full tool-calling
│ │ ├── chat_completions_language_model.py
│ │ ├── responses_api_language_model.py
│ │ └── language_model.py ← local transformers / mlx-lm handlers
│ │
│ ├── RAG/
│ │ ├── retriever.py ← singleton, embeddings, NPZ, search, full CRUD
│ │ └── router.py ← 11 FastAPI /v1/rag endpoints
│ │
│ └── api/openai_realtime/
│ ├── websocket_router.py ← FastAPI + Realtime + conditional RAG mount
│ ├── service.py ← Realtime event loop + session/handler routing
│ └── ...
│
├── kb/
│ ├── 01_faq_producto.md ← example Spanish FAQ (included in scaffold)
│ ├── 02_politicas_internas.md ← example Spanish policies
│ ├── README.md ← format guide
│ └── _dynamic.jsonl ← API-added chunks (auto-generated)
│
└── docs/
├── RAG_SERVER_SIDE.md ← Italian/English in-depth RAG guide (600+ lines)
└── RAG_SERVER_SIDE.es.md ← Spanish translation
```
---
## 9. Versioning and publishing
The repository ships with a full PyPI release pipeline in
`.github/workflows/publish.yml`:
1. Bump `version` in [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L7)
and `__version__` in [src/speech_to_speech/__init__.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/__init__.py).
2. Merge a release PR containing only those two changes.
3. Tag and push:
```bash
git checkout main && git pull origin main
git tag -a vX.Y.Z -m "Release vX.Y.Z"
git push origin vX.Y.Z
```
4. The workflow runs `uv build` + `twine check --strict` + PyPI upload automatically.
See [AGENTS.md](file:///home/azurian/speech-to-speech/AGENTS.md) for the repository-level release rules.
---
## 10. Where to go next
- Start with the launch scripts: [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) (base) and [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) (RAG enabled).
- Dive into the RAG subsystem: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md) / [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md).
- Tune the per-stage handlers by reading the argument classes in [arguments_classes](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes) — every flag has inline help.
- Build your own Realtime client against [service.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/service.py) and [websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py).

678
docs/RAG_SERVER_SIDE.es.md Normal file
View File

@ -0,0 +1,678 @@
# RAG Server-Side — Guía Oficial
> **Funcionalidad**: Retrieval Augmented Generation integrado en la pipeline speech-to-speech.
> **Enfoque**: 2 — Inyección transparente en el lado servidor (sin cambios en el cliente).
> **Versión mínima speech-to-speech**: `0.2.11`
---
## 1. Visión general
El RAG server-side enriquece **automáticamente** cada respuesta del LLM con fragmentos relevantes extraídos de una base de conocimientos local. Todo el ciclo ocurre de forma invisible para el cliente Realtime:
```
Usuario habla → STT → 🟡 RAG RETRIEVAL (hook interno) → LLM → TTS → Audio al usuario
↓
kb/*.md, kb/*.txt, kb/*.jsonl
↓
top-k chunk inyectados en el prompt
```
### Ventajas
- ✅ **Cero cambios en el cliente** — funciona con cualquier SDK Realtime
- ✅ **Singleton global compartido** (1 sola copia de embeddings para N pipelines paralelas)
- ✅ **Persistencia del índice NPZ** — reconstrucción solo si los documentos cambian
- ✅ **100% compatible con los logs existentes** (`LLM REQUEST PROMPT` incluye los chunk inyectados)
- ✅ **Soporte multilingüe** (ES/IT/EN configurable)
---
## 2. Instalación de dependencias
Las dependencias RAG son opcionales (grupo `rag` en [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94)):
```bash
cd /home/azurian/speech-to-speech
uv pip install -e ".[rag]"
```
**Contenido del grupo**:
- `sentence-transformers>=3.0.0` (arrastra automáticamente `torch`, `numpy`, `transformers` que ya están presentes)
> ✅ **En DGX Spark GB10**: el modelo de embedding se carga nativamente sobre CUDA (detectado por `--rag_device auto`, valor por defecto).
---
## 3. Estructura de la base de conocimientos
La KB reside en la carpeta configurada mediante el parámetro `--rag_kb_path` (por defecto: `./kb`).
```
kb/
├── README.md ← instrucciones del scaffold (auto-generado)
├── 01_faq_producto.md ← ejemplo español incluido
├── 02_politicas_internas.md ← ejemplo español incluido
│
├── manual/ ← subcarpetas soportadas
│ ├── 01_instalacion.md
│ └── 02_facturacion.txt
│
├── datos/
│ └── clientes.jsonl ← formato pre-chunkizado
│
├── _index.npz ← ⚙️ índice generado (NO modificar)
├── _chunks.jsonl ← ⚙️ catálogo chunk (NO modificar)
└── _dynamic.jsonl ← ⚙️ chunk dinámicos persistidos vía API
```
### Formatos soportados
#### A. Archivos Markdown / TXT (recomendado, esfuerzo cero)
Cualquier `*.md` o `*.txt` en la carpeta o subcarpetas se procesa así:
1. Lectura en UTF-8 (con fallback latin-1 y sustitución de errores)
2. División automática en chunk:
- **tamaño de chunk** por defecto: `512` caracteres (parámetro `--rag_chunk_size`)
- **overlap** por defecto: `64` caracteres (parámetro `--rag_chunk_overlap`)
- Algoritmo: splitter recursivo con separadores `\n\n → \n → . ? ! ; , → espacio → carácter`
3. Cada chunk recibe `source = path_relativo#chunk_index`
#### B. JSONL pre-chunkizado (control total)
Si prefieres gestionar chunk y metadatos manualmente (ej. extracción PDF estructurada), crea `*.jsonl` con **una línea por chunk**:
```jsonl
{"text": "Horario soporte lun-vie 09 a 18h", "source": "faq_horarios", "chunk_index": 0, "metadata": {"categoria": "soporte", "pagina": 12}}
{"text": "Devolución 14 días naturales", "source": "faq_compras", "chunk_index": 0, "metadata": {"categoria": "ventas"}}
```
Campos soportados:
| Campo | Obligatorio | Notas |
|---|---|---|
| `text` | ✅ | Cuerpo del chunk (string) |
| `source` | ❌ | Por defecto: `nombre_archivo.jsonl#lineaN` |
| `chunk_index` | ❌ | Por defecto: número de línea 0-based |
| cualquier otro | ❌ | Guardado en `chunk.metadata` y mostrado en los logs |
---
## 4. Persistencia del índice NPZ
Para evitar recalcular cientos/miles de embeddings en cada arranque:
### Primer arranque
```
archivos md/txt/jsonl → chunking → embedding → guarda:
kb/_index.npz (matriz numpy float32 N × embedding_dim)
kb/_chunks.jsonl (texto, source, metadata por cada línea)
```
Tiempo estimado: ~500 chunk/s en GB10 con MiniLM-L12-v2.
### Arranques sucesivos
```
Si existen _index.npz Y _chunks.jsonl Y las dimensiones coinciden:
→ carga directamente desde disco (<1 segundo)
Si no:
→ reconstruye desde cero
```
### Forzar reconstrucción
Usa **un solo** método cuando añadas/modifiques documentos:
1. **Flag CLI**: añade `--rag_force_rebuild` al arranque (recomendado)
2. **Manual**: borra `kb/_index.npz` y `kb/_chunks.jsonl`
---
## 5. Configuración (CLI / JSON)
Todos los parámetros están definidos en [rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) y son accesibles:
- Vía **flag CLI** en el comando `speech-to-speech ... --rag_enabled`
- Vía **archivo JSON** pasado como único argumento
- Vía `HfArgumentParser` (patrón estándar del proyecto)
### Tabla de parámetros
| Flag CLI | Valor por defecto | Descripción |
|---|---|---|
| `--rag_enabled` | `False` | **Interruptor maestro**. Sin este flag, todo lo demás se ignora. |
| `--rag_kb_path` | `./kb` | Ruta absoluta/relativa de la carpeta KB. |
| `--rag_embedding_model` | `paraphrase-multilingual-MiniLM-L12-v2` | Modelo HuggingFace `sentence-transformers`. *Multilingüe*, buen compromiso calidad/velocidad. |
| `--rag_device` | `auto` | `auto` / `cuda` / `cpu` / `mps`. |
| `--rag_top_k` | `3` | Número máximo de chunk inyectados por turno. **No superar 5-6 si la salida del TTS es larga** (crecen los tokens de contexto). |
| `--rag_threshold` | `0.25` | Umbral mínimo de similitud coseno. Rango `[0,1]`.<br>• `0.15`–`0.25` → recall alto, recupera casi todo<br>• `0.35`–`0.45` → precisión alta, solo coincidencias seguras<br>• `0.5+` → muy restrictivo |
| `--rag_language` | `es` | Idioma del encabezado de inyección: `es`, `it`, `en`. Cambia la cabecera del bloque RAG que pasa al LLM. |
| `--rag_inject_as` | `system` | Dónde inyectar los resultados:<br>• `system` → concatena a las instrucciones system (**recomendado**)<br>• `user` → añade como último mensaje del usuario |
| `--rag_chunk_size` | `512` | Caracteres máximos por chunk (solo md/txt). |
| `--rag_chunk_overlap` | `64` | Overlap en caracteres entre chunk adyacentes. |
| `--rag_embedding_batch_size` | `32` | Batch size de embeddings durante la construcción del índice. |
| `--rag_force_rebuild` | `False` | Si `True`, ignora el índice NPZ y regenera todo. |
### Modelos de embedding recomendados
| Modelo | Idiomas | Dim embedding | Velocidad GB10 | Mejor para |
|---|---|---|---|---|
| `paraphrase-multilingual-MiniLM-L12-v2` (por defecto) | 50+ | 384 | ⚡⚡⚡ | ES/IT/EN mezclado, KB genéricas |
| `hiiamsid/sentence_similarity_spanish_es` | Solo ES | 768 | ⚡⚡ | KB en español exclusivamente |
| `BAAI/bge-m3` | Multilingüe | 1024 | ⚡ | KB grandes, consultas complejas |
| `intfloat/multilingual-e5-large-instruct` | Multilingüe | 1024 | ⚡ | Consultas complejas con instrucciones |
---
## 6. Integración en start_pipeline.sh
Añade a tu [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) las líneas siguientes (al final del comando `speech-to-speech`):
```bash
#!/bin/bash
# ... (flags existentes: llm_backend, model_name, base_url, etc.)
# ========== RAG SERVER-SIDE ==========
--rag_enabled \
--rag_kb_path ./kb \
--rag_top_k 3 \
--rag_threshold 0.30 \
--rag_language es \
--rag_inject_as system \
--rag_device auto
# --rag_force_rebuild # ⚠️ COMENTA después de reconstruir el índice!
```
**Ejemplo de arranque completo** (con los parámetros que ya usas):
```bash
uv run speech-to-speech \
--llm_backend chat-completions \
--chat_completions_handler_base_url "$LLM_BASE_URL" \
--chat_completions_handler_model_name "$LLM_MODEL" \
--chat_completions_handler_api_key "$LLM_API_KEY" \
--mode realtime \
--realtime_host 0.0.0.0 --realtime_port 8000 \
--tts qwen3 --qwen3_tts_model_name "$TTS_MODEL" \
--stt whisper --whisper_stt_model_name "$STT_MODEL" \
\
--rag_enabled \
--rag_kb_path ./kb \
--rag_top_k 3 \
--rag_threshold 0.30 \
--rag_language es \
--rag_inject_as system
```
---
## 7. Arquitectura interna — Secuencia de operaciones
### 7.1 Setup (una sola vez al arranque)
Ocurre en [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py#L1029-L1059) dentro de `main()`:
```
main()
├─► si rag_enabled == True:
│ ├─► RAGRetriever(...) → carga modelo sentence-transformers
│ ├─► .build_index() → carga NPZ o rebuild desde cero
│ │ ├─► _load_index() si existe → carga inmediata
│ │ └─► _load_text_files() + _load_jsonl() + encode() + _save_index()
│ └─► set_global_rag(rag) → registra el singleton global
└─► build_pipeline(...)
└─► cada LLM handler, en su setup(): self.rag = get_global_rag()
```
### 7.2 Por cada turno de usuario (runtime)
Hook en [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py#L609-L613), línea **610**:
```
process(request: LLMIn)
├─► construye active_chat
├─► _apply_config(instructions, wants_audio) → system message con instrucciones
├─► _inject_rag_context(active_chat, turn_id, turn_revision) ← 🔴 NUESTRO HOOK
│ ├─► (1) query = extrae último texto de usuario desde .buffer
│ ├─► (2) results = rag.search(query, top_k, threshold)
│ │ ├─► embedding query (1 vector)
│ │ ├─► @ (matriz embeddings @ query.T) → similitud coseno
│ │ ├─► argpartition top-k + argsort score
│ │ └─► filtro por threshold
│ ├─► (3) si results vacío → log debug, NINGUNA inyección
│ ├─► (4) si no, formatea en bloque de texto:
│ │ "Fragmentos relevantes recuperados..."
│ │ "[1] (source=xxx.md, score=0.672)\n<texto>"
│ ├─► (5a) inject_as=system → añade como system message
│ └─► (5b) inject_as=user → añade como último user message
├─► resolve_auto_language(lang prompt)
├─► _generate() →
│ ├─► LOG: LLM REQUEST PROMPT ← incluye los chunk RAG!
│ ├─► stream LLM
│ └─► LOG: LLM RESPONSE ← respuesta final generada con el contexto
└─► yield chunks → TTS
```
---
## 8. Mensajes de log
Todos los logs RAG usan el mismo formato de la pipeline (prefijo `pipeline X`, nivel INFO o DEBUG).
### Setup OK
```
RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/kb
RAG: Índice cargado desde disco: 27 chunk (shape=(27, 384)).
RAG: Activo. kb=/home/kb top_k=3 umbral=0.300 inject_as=system idioma=es chunk=512
RAG hook activo en BaseOpenAICompatibleHandler (inject_as=system, idioma=es)
```
### Setup: ningún documento
```
RAG: Ningún documento encontrado en /home/kb. Índice vacío (retrieval no inyectará nada).
```
### Retrieval exitoso
```
RAG: inyectados 2 chunk para turn=turn_91a75 rev=1 — 01_faq_producto.md#c1(0.67); 02_politicas_internas.md#c0(0.52)
```
### Ninguna coincidencia
```
RAG: ningún chunk relevante para turn=turn_91a75 (query='Hola, ¿cómo te llamas?' umbral=0.300)
```
### Fallo de search (¡no bloquea la pipeline!)
```
RAG search fallida para turn=turn_91a75 rev=1: <excepción>
```
---
## 9. Depuración y troubleshooting
### Síntoma 1: "RAG no inyecta nada, aunque sé que el documento está ahí"
1. Revisa los logs — ¿qué mensaje aparece? `inyectados X` o `ningún chunk`?
2. Si es `ningún chunk` → baja el umbral: `--rag_threshold 0.20` o incluso `0.15`.
3. Sube top_k: `--rag_top_k 5`.
4. Reconstruye el índice: añade `--rag_force_rebuild` (quizás modificaste archivos después del primer arranque).
5. Prueba directamente: compara el texto de la pregunta vs el texto del chunk — los modelos MiniLM pueden tener resultados escasos con frases muy coloquiales. **Solución**: aumenta el overlap o redacta chunk con preguntas+respuestas explícitas.
### Síntoma 2: "Primer arranque lentísimo / crash CUDA OOM"
1. Reduce el batch size de embedding: `--rag_embedding_batch_size 4` (por defecto 32).
2. Si tienes miles de chunk, valora `paraphrase-multilingual-MiniLM-L12-v2` (384 dim, la mitad de memoria que bge-m3).
### Síntoma 3: "El LLM ignora los chunk e inventa datos"
1. Refuerza las `instructions` de sistema (vía session.update) con un prompt como:
> *"Responde SOLO con la información contenida en los Fragmentos relevantes recuperados. Si no hay información, di 'No tengo información sobre eso'."*
2. Prueba `--rag_inject_as user` (algunos modelos respetan más los bloques inyectados dentro de los mensajes de usuario).
3. Sube el **umbral** (reduce falsos positivos): `--rag_threshold 0.40`.
4. Verifica que los chunk tengan sentido (elimina cabeceras Markdown, metadatos no pertinentes).
### Síntoma 4: "El log LLM REQUEST PROMPT no muestra los chunk"
Verifica:
- Que `--rag_enabled` se haya pasado realmente (¿aparece el log `RAG: Activo` en el arranque?).
- Que el handler LLM sea una subclase de `BaseOpenAICompatibleHandler` (chat-completions y responses-api lo son; mlx-lm local y transformers no — usan [language_model_handler.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/language_model_handler.py) que NO comparte la base; para estos últimos el RAG no está enganchado todavía).
---
## 10. Ejemplos de sesiones reales
### Caso: FAQ de producto en español
- **Usuario (audio)**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"*
- **STT**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"*
- **RAG retrieval**:
```
Score 0.71 → 01_faq_producto.md#c3: "¿Ofrecéis reembolsos? Sí. Todos los planes tienen un periodo de devolución de 14 días naturales..."
Score 0.23 → 02_politicas_internas.md#c1: "Retención de registros: datos usuario dado de baja 6 meses..." ← descartado por threshold=0.30
```
- **Texto inyectado en el system**:
```
Fragmentos relevantes recuperados de la base de conocimientos:
[1] (source=01_faq_producto.md, score=0.712)
¿Ofrecéis reembolsos?
Sí. Todos los planes tienen un periodo de devolución de **14 días naturales**
desde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos
se evalúan caso por caso por el equipo de soporte.
```
- **Respuesta LLM**: *"Tienes 14 días naturales desde la compra para solicitar un reembolso sin necesidad de dar justificaciones. Pasado ese plazo, el equipo de soporte lo evalúa caso por caso."*
- **TTS**: el audio se genera y se envía al cliente.
---
## 11. Archivos del proyecto involucrados
| Archivo | Rol |
|---|---|
| [RAG/\_\_init\_\_.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/__init__.py) | Exports públicos (singleton + dataclass) |
| [RAG/retriever.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py) | Core: chunking, embedding, persistencia NPZ, search |
| [arguments_classes/rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) | Dataclass HfArgumentParser 12 parámetros |
| [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py) (`main()`, líneas 1029-1059) | Setup singleton global + build index |
| [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py) | Import singleton (línea 41); setup hook (líneas 163-166); injection (línea 610); helper `_extract_last_user_text` + `_inject_rag_context` (líneas 274-347) |
| [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94) | Grupo optional-deps `rag` |
| [kb/README.md](file:///home/azurian/speech-to-speech/kb/README.md) | Scaffold inicial de la KB |
| [kb/01_faq_producto.md](file:///home/azurian/speech-to-speech/kb/01_faq_producto.md) | Ejemplo FAQ español |
| [kb/02_politicas_internas.md](file:///home/azurian/speech-to-speech/kb/02_politicas_internas.md) | Ejemplo políticas español |
| [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) | Añadir flags --rag_* al comando (véase §6) |
| [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) | Script preconfigurado (variables de entorno + todos los flags RAG) |
| [RAG/router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/router.py) | **NUEVO** — Router FastAPI `/v1/rag/*` (11 endpoints HTTP) para administración dinámica |
| [api/openai_realtime/websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py#L511-L526) | Montaje condicional del router RAG en el mismo FastAPI del websocket |
---
## 13. 🆕 API HTTP dinámicas (puebla la KB en caliente)
La pipeline expone **11 endpoints JSON** en el mismo servidor que sirve el websocket
(`ws://host:12345` → HTTP sobre `http://host:12345`).
Base path común: `/v1/rag`.
> ⚠️ Si RAG está desactivado (sin `--rag_enabled`) todos los endpoints responden
> **503 Service Unavailable** con `{"code": "RAG_NOT_ENABLED", ...}`.
### 13.1 Endpoints disponibles
| Método | Path | Descripción |
|---|---|---|
| `GET` | `/v1/rag/status` | Estado: número chunk, dimensión embedding, fuentes, dispositivo, umbrales |
| `GET` | `/v1/rag/sources` | **LIST** — fuentes únicas + contador de chunk por cada una |
| `GET` | `/v1/rag/chunks` | **LIST** con query params: filtros, paginación, ordenación por relevancia |
| `POST` | `/v1/rag/chunks/list` | Mismo listado pero con body JSON (recomendado para consultas complejas) |
| `POST` | `/v1/rag/chunks/update` | **UPDATE** de un solo chunk (texto / metadatos / fuente) |
| `POST` | `/v1/rag/upsert/document` | **UPSERT atómico** de un documento entero (borra lo viejo + inserta lo nuevo). ✅ Uso más común para CRUD |
| `POST` | `/v1/rag/search` | Busca chunk con el mismo algoritmo que se usa para la inyección (depuración/pruebas) |
| `POST` | `/v1/rag/add/document` | INSERT puro de texto libre (chunking automático) |
| `POST` | `/v1/rag/add/chunks` | INSERT puro de chunk preformateados |
| `POST` | `/v1/rag/remove` | **REMOVE**: por prefijo o coincidencia exacta |
| `POST` | `/v1/rag/reload` | Reconstrucción total del índice desde los archivos en `kb/` |
> 💡 **Patrón CRUD recomendado**:
> - CREATE → `/upsert/document` (idempotente: si la source no existe la crea)
> - READ → `/sources` (lista fuentes) + `/chunks?source_exact=X` (detalle)
> - UPDATE → `/upsert/document` (misma source, texto nuevo → sustitución atómica)
> - DELETE → `/upsert/document` con `text: ""` o `/remove` con `exact: true`
---
### 13.2 Ejemplos curl
> Base URL: `http://127.0.0.1:12345/v1/rag` (cambia puerto y host si modificaste el script).
#### 📤 Añadir un documento (texto libre, chunking automático)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/document \
-H 'Content-Type: application/json' \
-d '{
"source": "crm/cliente_456_nota_20260826",
"text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual, renovación 15/02/2027. Contacto: +34 600 111 222, javier@empresa.es. Notas: prefiere que le llamen por la mañana antes de las 11h. Ha reportado un incidente con la factura número 2026-08-114 el 20/08/2026 que ya fue resuelto con un descuento del 15% aplicado en la siguiente factura.",
"metadata": {"cliente_id": 456, "pais": "es", "categoria": "crm"},
"persist": true,
"dynamic": true
}' | jq
```
**Respuesta:**
```json
{
"added": 1,
"sources": ["crm/cliente_456_nota_20260826"],
"total_after": 10
}
```
> 💡 **`persist: true`** → el chunk también se escribe en `kb/_dynamic.jsonl`,
> por lo tanto **en el próximo arranque** de la pipeline seguirá presente.
> Ponlo en `false` para chunk temporales (solo la sesión actual).
#### 📤 Añadir N chunk preformateados (ej. desde DB)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/chunks \
-H 'Content-Type: application/json' \
-d '{
"items": [
{"text": "Pedido 8942 — 25/08/2026. Productos: Teclado RGB Pro x1, Ratón Ergonómico x1. Estado: enviado, tracking 1Z999AA10123456784. Entrega estimada: 27/08/2026.",
"source": "pedidos/cliente_456/8942",
"metadata": {"pedido_id": 8942, "estado": "enviado"}},
{"text": "Dirección de envío habitual del cliente 456: Av. Diagonal 444, puerta 3, 08013 Barcelona, España. Contacto entrega: +34 600 111 222.",
"source": "direcciones/cliente_456/principal",
"metadata": {"tipo": "envio", "predeterminada": true}}
],
"persist": true,
"dynamic": true
}' | jq
```
#### 🔎 Probar el retrieval vía HTTP (depuración, sin audio)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/search \
-H 'Content-Type: application/json' \
-d '{
"query": "¿Qué pedido tiene Javier García y cuándo llega?",
"top_k": 3,
"threshold": 0.25
}' | jq
```
Respuesta con los chunk recuperados y el score:
```json
{
"query": "¿Qué pedido tiene Javier García y cuándo llega?",
"count": 2,
"results": [
{"text": "Pedido 8942 — 25/08/2026...",
"source": "pedidos/cliente_456/8942", "score": 0.813},
{"text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual...",
"source": "crm/cliente_456_nota_20260826", "score": 0.742}
]
}
```
#### 📊 Ver estado general
```bash
curl -sS http://127.0.0.1:12345/v1/rag/status | jq
```
#### 🗑️ Eliminar todos los chunk de un cliente
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \
-H 'Content-Type: application/json' \
-d '{"source_prefix": "crm/cliente_456"}' | jq
```
Sugerencia: usa prefijos inteligentes para organizar la KB:
```
source_prefix = "crm/" → borra toda la agenda
source_prefix = "pedidos/" → borra todos los datos de pedidos
source_prefix = "dynamic#" → borra todos los chunk temporales auto-numerados
```
#### 🛡️ Remove con coincidencia exacta (solo source EXACTA)
```bash
# Borra SÓLO el documento 'crm/cliente_456_nota_20260826'
# (no borra 'crm/cliente_456_nota_20260827')
curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \
-H 'Content-Type: application/json' \
-d '{"source_prefix": "crm/cliente_456_nota_20260826", "exact": true}' | jq
```
#### 📋 Listado de fuentes (inventario del índice)
```bash
# Todas las fuentes del índice, con conteo de chunk
curl -sS http://127.0.0.1:12345/v1/rag/sources | jq
```
Ejemplo salida:
```json
{
"count": 5,
"items": [
{"source": "01_faq_producto.md", "chunk_count": 5, "has_dynamic": false},
{"source": "crm/cliente_456_nota_20260826", "chunk_count": 1, "has_dynamic": true},
{"source": "pedidos/cliente_456/8942", "chunk_count": 1, "has_dynamic": true}
]
}
```
#### 🔍 Listar chunk con filtros y paginación
```bash
# GET con query params: solo chunk de cliente_456, paginado
curl -sS 'http://127.0.0.1:12345/v1/rag/chunks?source_prefix=crm/cliente_456&limit=10&offset=0' | jq
# POST con body JSON: ORDENA por relevancia sobre una consulta + filtro
curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/list \
-H 'Content-Type: application/json' \
-d '{
"source_prefix": "crm/",
"query": "descuento aplicado en factura",
"min_score": 0.20,
"limit": 10,
"include_text": true
}' | jq
```
Respuesta:
```json
{
"total": 2,
"offset": 0,
"limit": 10,
"query": "descuento aplicado en factura",
"items": [
{
"index": 9,
"source": "crm/cliente_456_nota_20260826",
"chunk_index": 1000009,
"score": 0.731,
"metadata": {"cliente_id": 456, "categoria": "crm"},
"is_dynamic": true,
"text": "Cliente: Javier García... 15% descuento..."
}
]
}
```
Campos especiales:
- `index`: la posición numérica en el vector de chunk — pásala directamente a `/chunks/update`
- `score`: `null` sin query, float 0-1 con ordenación por relevancia
#### 🆙 Actualizar un solo chunk
Usa `/chunks/update` cuando debas modificar **únicamente un trozo específico** (ej. corregir una errata, cambiar metadatos). Para actualizar **un documento entero** usa en su lugar `/upsert/document`.
```bash
# Opción A: actualiza por `index` (de list_chunks → campo "index": 9)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \
-H 'Content-Type: application/json' \
-d '{
"index": 9,
"new_text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2028 (2 años). Descuento 20% acordado en la llamada del 10/08.",
"new_metadata": {"cliente_id": 456, "categoria": "crm", "ultima_revision": "2026-08-26"}
}' | jq
# Opción B: actualiza por (source, chunk_index)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \
-H 'Content-Type: application/json' \
-d '{
"source": "crm/cliente_456_nota_20260826",
"chunk_index": 1000009,
"new_source": "crm/cliente_456/nota_principal"
}' | jq
```
Respuesta éxito `200`:
```json
{
"updated": 1,
"chunk": {
"index": 9,
"source": "crm/cliente_456/nota_principal",
"chunk_index": 1000009,
"text": "Cliente: Javier García...",
"metadata": {"cliente_id": 456, "...": "..."},
"is_dynamic": true
}
}
```
Errores HTTP posibles:
- `404 CHUNK_NOT_FOUND`
- `409 AMBIGUOUS_SOURCE` → pasaste `source` sola pero existen N chunk con la misma source; añade `chunk_index` o usa `index`
#### 🔄 Upsert documento (CRUD recomendado)
**Éste es el endpoint que usarás en el 90% de los casos.**
Idempotente sobre el campo `source`:
- Si la `source` no existe → CREATE
- Si la `source` existe → DELETE de todos los chunk antiguos + split del nuevo texto + INSERT
- Si mandas `text: ""` → DELETE atómico (sin re-insert)
```bash
# CREATE/UPDATE de un documento entero (ej. nota cliente)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \
-H 'Content-Type: application/json' \
-d '{
"source": "crm/cliente_456/perfil",
"text": "Javier García — ID 456. Plan Premium Anual, renovación automática 15/02/2028. Precio: 24€/mes. Email javier@empresa.es. Tel +34 600 111 222. Dirección envío habitual: Av. Diagonal 444, puerta 3, 08013 Barcelona. Notas: siempre prefiere atención por la mañana, antes de las 11h. Caso abierto 2026-08-0124: seguimiento calidad postventa, cerrado con valoración 5/5.",
"metadata": {"cliente_id": 456, "pais": "es", "actualizado": "2026-08-26"},
"dynamic": true,
"persist": true
}' | jq
```
Respuesta:
```json
{
"removed": 2,
"added": 1,
"sources": ["crm/cliente_456/perfil"],
"total_after": 10,
"mode": "upsert"
}
```
DELETE vía upsert:
```bash
# Elimina del todo todos los chunk asociados a esta source (atómico)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \
-H 'Content-Type: application/json' \
-d '{"source": "crm/cliente_456/perfil", "text": ""}' | jq
```
#### 🔄 Recargar toda la KB desde archivos (después de copiar nuevos md/txt)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/reload \
-H 'Content-Type: application/json' \
-d '{"force_rebuild": true}' | jq
```
---
### 13.3 Persistencia entre reinicios: `kb/_dynamic.jsonl`
Todos los chunk añadidos vía API con `persist: true` se añaden a `kb/_dynamic.jsonl` (un JSON por línea). En el próximo arranque de la pipeline:
1. Se reconstruye el índice estático desde los archivos md/txt/jsonl
2. **Inmediatamente después** se recargan y re-embeddan también todos los chunk de `_dynamic.jsonl`
Por lo tanto el conocimiento añadido por HTTP es **duradero** y no se pierde al reiniciar.
---
### 13.4 Seguridad en hilos (thread safety)
Todas las API usan un `threading.RLock()` dentro de [RAGRetriever](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py):
- ✅ Varias llamadas `/search` pueden ejecutarse en paralelo durante la conversación
- ✅ Una `add_document`/`remove` bloquea brevemente el índice solo para las operaciones numpy de vstack/take — el retrieval no se pierde, espera
- ✅ La persistencia en disco (NPZ + JSONL) ocurre fuera del lock, por lo que no ralentiza la llamada de la conversación
---
## 14. Hoja de ruta / posibles mejoras
- [ ] **Soporte PDF**: añadir `pypdf` o `pdfplumber` en un grupo opcional
- [ ] **MMR reranking** en lugar del solo top-k por coseno (mejor diversidad entre chunk)
- [ ] **HyDE** (el LLM genera una consulta hipotética + embedding de la misma, para consultas coloquiales)
- [ ] **Cross-encoder reranker** de 2 etapas para KB grandes (>10k chunk)
- [ ] **Hook sobre LanguageModelHandler local** (mlx-lm / transformers) — actualmente el RAG solo está enganchado para los backends `chat-completions` y `responses-api`
- [ ] **Watchdog KB**: auto-rebuild del índice cuando cambian los archivos (inotify / watchdog)

676
docs/RAG_SERVER_SIDE.md Normal file
View File

@ -0,0 +1,676 @@
# RAG Server-Side — Guida Ufficiale
> **Feature**: Retrieval Augmented Generation integrato nella pipeline speech-to-speech.
> **Approccio**: 2 — Injection trasparente lato server (nessuna modifica lato client).
> **Versione minima speech-to-speech**: `0.2.11`
---
## 1. Panoramica
L'RAG server-side arricchisce **automaticamente** ogni risposta dell'LLM con brani rilevanti estratti da una knowledge base locale. L'intero ciclo avviene in modo invisibile al client Realtime:
```
Utente parla → STT → 🟡 RAG RETRIEVAL (hook interno) → LLM → TTS → Audio all'utente
↓
kb/*.md, kb/*.txt, kb/*.jsonl
↓
top-k chunk iniettati nel prompt
```
### Vantaggi
- ✅ Zero modifiche lato client — funziona con qualunque SDK Realtime
- ✅ Riuso del singleton globale (1 sola copia degli embeddings per N pipeline parallele)
- ✅ Persistenza dell'indice NPZ — ricostruzione solo se i documenti cambiano
- ✅ 100% compatibile con i log già esistenti (`LLM REQUEST PROMPT` include i chunk iniettati)
- ✅ Supporto multilingue (IT/ES/EN configurabile)
---
## 2. Installazione dipendenze
Le dipendenze RAG sono opzionali (gruppo `rag` in [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94)):
```bash
cd /home/azurian/speech-to-speech
uv pip install -e ".[rag]"
```
**Contenuto del gruppo**:
- `sentence-transformers>=3.0.0` (porta automaticamente `torch`, `numpy`, `transformers` già presenti)
> ✅ **Su DGX Spark GB10**: il modello embedding viene caricato nativamente su CUDA (detected da `--rag_device auto`, default).
---
## 3. Struttura Knowledge Base
La KB risiede nella cartella configurata dal parametro `--rag_kb_path` (default: `./kb`).
```
kb/
├── README.md ← istruzioni scaffold (auto-generato)
├── 01_faq_producto.md ← esempio spagnolo incluso
├── 02_politicas_internas.md ← esempio spagnolo incluso
│
├── manual/ ← sotto-cartelle supportate
│ ├── 01_instalacion.md
│ └── 02_facturacion.txt
│
├── datos/
│ └── clientes.jsonl ← formato pre-chunkizzato
│
├── _index.npz ← ⚙️ indice generato (NON modificare)
└── _chunks.jsonl ← ⚙️ catalogo chunk (NON modificare)
```
### Formati supportati
#### A. File Markdown / TXT (raccomandato, zero sforzo)
Qualsiasi `*.md` o `*.txt` nella cartella o sottocartelle viene:
1. Letto in UTF-8 (fallback latin-1 con sostituzione errori)
2. Splittato automaticamente in chunk:
- **chunk size** default: `512` caratteri (parametro `--rag_chunk_size`)
- **overlap** default: `64` caratteri (parametro `--rag_chunk_overlap`)
- Algoritmo: splitter ricorsivo per separatori `\n\n → \n → . ? ! ; , → spazio → carattere`
3. Ogni chunk riceve `source = path_relativo#chunk_index`
#### B. JSONL pre-chunkizzato (controllo fine)
Se preferisci gestire chunk e metadata manualmente (es. da estrazione PDF strutturata), crea `*.jsonl` con **una riga per chunk**:
```jsonl
{"text": "Horario soporte lun-vier 09 a 18h", "source": "faq_horarios", "chunk_index": 0, "metadata": {"categoria": "soporte", "pagina": 12}}
{"text": "Devolución 14 días naturales", "source": "faq_compras", "chunk_index": 0, "metadata": {"categoria": "ventas"}}
```
Campi supportati:
| Campo | Obbligatorio | Note |
|---|---|---|
| `text` | ✅ | Corpo del chunk (stringa) |
| `source` | ❌ | Default: `nome_file.jsonl#rigaN` |
| `chunk_index` | ❌ | Default: numero riga 0-based |
| qualsiasi altro | ❌ | Salvato in `chunk.metadata` e riportato nei log |
---
## 4. Persistenza indice NPZ
Per evitare di ricalcolare centinaia/migliaia di embeddings ad ogni avvio:
### Primo avvio
```
file md/txt/jsonl → chunking → embedding → salva:
kb/_index.npz (matrice numpy float32 N × embedding_dim)
kb/_chunks.jsonl (testo, source, metadata per ogni riga)
```
Tempo stimato: ~500 chunk/s su GB10 con MiniLM-L12-v2.
### Avvii successivi
```
Se esiste _index.npz E _chunks.jsonl E le dimensioni coincidono:
→ carica direttamente da disco (<1 secondo)
Altrimenti:
→ ricostruisce da zero
```
### Forzare ricostruzione
Usa **uno** di questi metodi quando aggiungi/modifichi documenti:
1. **Flag CLI**: aggiungi `--rag_force_rebuild` all'avvio (consigliato)
2. **Manuale**: cancella `kb/_index.npz` e `kb/_chunks.jsonl`
---
## 5. Configurazione (CLI / JSON)
Tutti i parametri sono definiti in [rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) e sono accessibili:
- Via **flag CLI** nel comando `speech-to-speech ... --rag_enabled`
- Via **file JSON** passato come unico argomento
- Via `HfArgumentParser` (pattern standard del progetto)
### Tabella parametri
| Flag CLI | Default | Descrizione |
|---|---|---|
| `--rag_enabled` | `False` | **Master switch**. Senza questo flag, tutto il resto viene ignorato. |
| `--rag_kb_path` | `./kb` | Path assoluto/relativo della cartella KB. |
| `--rag_embedding_model` | `paraphrase-multilingual-MiniLM-L12-v2` | Modello HuggingFace `sentence-transformers`. *Multilingue*, ottimo compromesso qualità/velocità. |
| `--rag_device` | `auto` | `auto` / `cuda` / `cpu` / `mps`. |
| `--rag_top_k` | `3` | Numero massimo di chunk iniettati per turno. **Non superare 5-6 se TTS output lungo** (crescono i token di contesto). |
| `--rag_threshold` | `0.25` | Soglia minima similarità coseno. Range `[0,1]`.<br>• `0.15`–`0.25` → recall alto, recuperi quasi tutto<br>• `0.35`–`0.45` → precisione alta, solo match certi<br>• `0.5+` → molto restrittivo |
| `--rag_language` | `es` | Lingua header injection: `es`, `it`, `en`. Cambia l'intestazione del blocco RAG passato all'LLM. |
| `--rag_inject_as` | `system` | Dove iniettare i risultati:<br>• `system` → concatenato alle istruzioni system (**consigliato**)<br>• `user` → accodato all'ultimo messaggio utente |
| `--rag_chunk_size` | `512` | Caratteri massimi per chunk (solo md/txt). |
| `--rag_chunk_overlap` | `64` | Overlap caratteri tra chunk adiacenti. |
| `--rag_embedding_batch_size` | `32` | Batch size embedding durante build indice. |
| `--rag_force_rebuild` | `False` | Se `True`, ignora l'indice NPZ e rigenera tutto. |
### Modelli embedding consigliati
| Modello | Lingue | Dim embedding | Velocità GB10 | Migliore per |
|---|---|---|---|---|
| `paraphrase-multilingual-MiniLM-L12-v2` (default) | 50+ | 384 | ⚡⚡⚡ | IT/ES/EN mix, KB generiche |
| `hiiamsid/sentence_similarity_spanish_es` | Solo ES | 768 | ⚡⚡ | KB solo spagnola |
| `BAAI/bge-m3` | Multilingue | 1024 | ⚡ | KB grandi, query complesse |
| `intfloat/multilingual-e5-large-instruct` | Multilingue | 1024 | ⚡ | Query complesse, istruzioni |
---
## 6. Integrazione in start_pipeline.sh
Aggiungi al tuo [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) le righe seguenti (alla fine del comando `speech-to-speech`):
```bash
#!/bin/bash
# ... (flags esistenti: llm_backend, model_name, base_url, ecc.)
# ========== RAG SERVER-SIDE ==========
--rag_enabled \
--rag_kb_path ./kb \
--rag_top_k 3 \
--rag_threshold 0.30 \
--rag_language es \
--rag_inject_as system \
--rag_device auto
# --rag_force_rebuild # ⚠️ COMMENTA dopo aver ricostruito l'indice!
```
**Esempio start completo** (con i parametri che già usi):
```bash
uv run speech-to-speech \
--llm_backend chat-completions \
--chat_completions_handler_base_url "$LLM_BASE_URL" \
--chat_completions_handler_model_name "$LLM_MODEL" \
--chat_completions_handler_api_key "$LLM_API_KEY" \
--mode realtime \
--realtime_host 0.0.0.0 --realtime_port 8000 \
--tts qwen3 --qwen3_tts_model_name "$TTS_MODEL" \
--stt whisper --whisper_stt_model_name "$STT_MODEL" \
\
--rag_enabled \
--rag_kb_path ./kb \
--rag_top_k 3 \
--rag_threshold 0.30 \
--rag_language es \
--rag_inject_as system
```
---
## 7. Architettura interna — Sequenza operazioni
### 7.1 Setup (una tantum ad avvio)
Avviene in [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py#L1029-L1059) dentro `main()`:
```
main()
├─► if rag_enabled == True:
│ ├─► RAGRetriever(...) → carica modello sentence-transformers
│ ├─► .build_index() → NPZ o rebuild da zero
│ │ ├─► _load_index() se esiste → carica immediato
│ │ └─► _load_text_files() + _load_jsonl() + encode() + _save_index()
│ └─► set_global_rag(rag) → registra singleton globale
└─► build_pipeline(...)
└─► ogni LLM handler, nel suo setup(): self.rag = get_global_rag()
```
### 7.2 Per ogni turno utente (runtime)
Hook in [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py#L609-L613), riga **610**:
```
process(request: LLMIn)
├─► build active_chat
├─► _apply_config(instructions, wants_audio) → system message con le istruzioni
├─► _inject_rag_context(active_chat, turn_id, turn_revision) ← 🔴 NOSTRO HOOK
│ ├─► (1) query = estrai ultimo testo utente da .buffer
│ ├─► (2) results = rag.search(query, top_k, threshold)
│ │ ├─► embedding query (1 vettore)
│ │ ├─► @ (matrice embeddings @ query.T) → coseno similarity
│ │ ├─► argpartition top-k + argsort score
│ │ └─► filtro per threshold
│ ├─► (3) se results vuoti → log debug, NIENTE injection
│ ├─► (4) altrimenti formatta in blocco testuale:
│ │ "Fragmentos relevantes recuperados..."
│ │ "[1] (source=xxx.md, score=0.672)\n<testo>"
│ ├─► (5a) inject_as=system → aggiungi come system message
│ └─► (5b) inject_as=user → aggiungi come user message finale
├─► resolve_auto_language(lang prompt)
├─► _generate() →
│ ├─► LOG: LLM REQUEST PROMPT ← include i chunk RAG!
│ ├─► stream LLM
│ └─► LOG: LLM RESPONSE ← risposta finale generata col contesto
└─► yield chunks → TTS
```
---
## 8. Messaggi di log
Tutti i log RAG usano lo stesso format della pipeline (prefix `pipeline X`, livello INFO o DEBUG).
### Setup OK
```
RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/kb
RAG: Indice caricato da disco: 27 chunk (shape=(27, 384)).
RAG: Attivo. kb=/home/kb top_k=3 soglia=0.300 inject_as=system lingua=es chunk=512
RAG hook attivo in BaseOpenAICompatibleHandler (inject_as=system, lingua=es)
```
### Setup: nessun documento
```
RAG: Nessun documento trovato in /home/kb. Indice vuoto (retrieval non inietterà niente).
```
### Retrieval riuscito
```
RAG: iniettati 2 chunk per turn=turn_91a75 rev=1 — 01_faq_producto.md#c1(0.67); 02_politicas_internas.md#c0(0.52)
```
### Nessun match
```
RAG: nessun chunk rilevante per turn=turn_91a75 (query='Hola, ¿cómo te llamas?' soglia=0.300)
```
### Fallimento search (non blocca la pipeline!)
```
RAG search fallita per turn=turn_91a75 rev=1: <eccezione>
```
---
## 9. Debug & troubleshooting
### Sintomo 1: "RAG non inietta niente, anche se so che il documento c'è"
1. Controlla i log — che messaggio compare? `iniettati X` o `nessun chunk`?
2. Se `nessun chunk` → abbassa la soglia: `--rag_threshold 0.20` o anche `0.15`.
3. Aumenta top_k: `--rag_top_k 5`.
4. Ricostruisci indice: aggiungi `--rag_force_rebuild` (magari hai modificato i file dopo il primo avvio).
5. Testa direttamente: confronta il testo della domanda vs il testo del chunk — i modelli MiniLM possono avere scarsi risultati con frasi troppo colloquiali. **Soluzione**: aumenta overlap o scrivi chunk con domande+risposte esplicite.
### Sintomo 2: "Primo avvio lentissimo / crash CUDA OOM"
1. Riduci batch size embedding: `--rag_embedding_batch_size 4` (default 32).
2. Se hai migliaia di chunk, prendi in considerazione `paraphrase-multilingual-MiniLM-L12-v2` (384 dim, metà memoria di bge-m3).
### Sintomo 3: "L'LLM ignora i chunk e inventa fatti"
1. Rinforza le `instructions` di sistema (via session.update) con un prompt tipo:
> *"Responde SOLO con información contenuta nei Fragmentos relevantes recuperados. Si no hay información, di 'No tengo información sobre eso'."*
2. Passa a `--rag_inject_as user` (alcuni modelli rispettano di più i blocchi injection dentro i messaggi utente).
3. Aumenta la **soglia** (riduci falsi positivi): `--rag_threshold 0.40`.
4. Verifica che i chunk abbiano senso (elimina intestazioni Markdown, metadati non pertinenti).
### Sintomo 4: "Il log LLM REQUEST PROMPT non mostra i chunk"
Verifica:
- `--rag_enabled` è effettivamente passato (log `RAG: Attivo` in avvio?)
- L'handler LLM è una sottoclasse di `BaseOpenAICompatibleHandler` (chat-completions e responses-api lo sono; mlx-lm locale e transformers no — usano [language_model_handler.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/language_model_handler.py) che NON condivide la base; per questi ultimi il RAG al momento non è agganciato).
---
## 10. Esempi di sessioni reali
### Caso: FAQ prodotto in spagnolo
- **Utente audio**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"*
- **STT**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"*
- **RAG retrieval**:
```
Score 0.71 → 01_faq_producto.md#c3: "¿Ofrecéis reembolsos? Sí. Todos los planes tienen un periodo de devolución de 14 días naturales..."
Score 0.23 → 02_politicas_internas.md#c1: "Retención de registros: datos usuario dado de baja 6 meses..." ← scartato da threshold=0.30
```
- **Testo iniettato nel system**:
```
Fragmentos relevantes recuperados de la base de conocimientos:
[1] (source=01_faq_producto.md, score=0.712)
¿Ofrecéis reembolsos?
Sí. Todos los planes tienen un periodo de devolución de **14 días naturales**
desde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos
se evalúan caso por caso por el equipo de soporte.
```
- **Risposta LLM**: *"Tienes 14 días naturales desde la compra para solicitar un reembolso sin necesidad de dar justificaciones. Pasado ese plazo, el equipo de soporte lo evalúa caso por caso."*
- **TTS**: l'audio viene generato e spedito al client.
---
## 11. File del progetto coinvolti
| File | Ruolo |
|---|---|
| [RAG/\_\_init\_\_.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/__init__.py) | Export pubblici (singleton + dataclass) |
| [RAG/retriever.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py) | Core: chunking, embedding, persistenza NPZ, search |
| [arguments_classes/rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) | Dataclass HfArgumentParser 12 parametri |
| [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py) (`main()`, righe 1029-1059) | Setup singleton globale + build index |
| [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py) | Import singleton (riga 41); setup hook (riga 163-166); injection (riga 610); helper `_extract_last_user_text` + `_inject_rag_context` (righe 274-347) |
| [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94) | Gruppo optional-deps `rag` |
| [kb/README.md](file:///home/azurian/speech-to-speech/kb/README.md) | Scaffold iniziale KB |
| [kb/01_faq_producto.md](file:///home/azurian/speech-to-speech/kb/01_faq_producto.md) | Esempio FAQ spagnolo |
| [kb/02_politicas_internas.md](file:///home/azurian/speech-to-speech/kb/02_politicas_internas.md) | Esempio politiche spagnolo |
| [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) | Aggiungere flag --rag_* al comando (vedi §6) |
| [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) | Script preconfigurato (variabili d'ambiente + tutti i flag RAG) |
| [RAG/router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/router.py) | **NUOVO** — Router FastAPI `/v1/rag/*` (7 endpoint HTTP) per amministrazione dinamica |
| [api/openai_realtime/websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py#L511-L526) | Montaggio condizionale del router RAG nello stesso FastAPI del websocket |
---
## 13. 🆕 API HTTP dinamiche (popola la KB a caldo)
La pipeline espone **7 endpoint JSON** sullo stesso server che serve il websocket
(`ws://host:12345` → HTTP su `http://host:12345`).
Base path comune: `/v1/rag`.
> ⚠️ Se RAG è disattivato (no `--rag_enabled`) tutti gli endpoint rispondono
> **503 Service Unavailable** con `{"code": "RAG_NOT_ENABLED", ...}`.
### 13.1 Endpoint disponibili
| Metodo | Path | Descrizione |
|---|---|---|
| `GET` | `/v1/rag/status` | Stato: numero chunk, embedding dim, sorgenti, device, soglie |
| `GET` | `/v1/rag/sources` | **LIST** — sorgenti uniche + chunk count per ognuna |
| `GET` | `/v1/rag/chunks` | **LIST** con query params: filtri, paginazione, ordinamento per rilevanza |
| `POST` | `/v1/rag/chunks/list` | Stesso listing, ma body JSON (consigliato per query complesse) |
| `POST` | `/v1/rag/chunks/update` | **UPDATE** singolo chunk (testo / metadata / sorgente) |
| `POST` | `/v1/rag/upsert/document` | **UPSERT atomico** di un documento intero (delete old + insert new). ✅ Uso più comune per CRUD |
| `POST` | `/v1/rag/search` | Cerca chunk con lo stesso algoritmo usato per l'iniezione (debug/test) |
| `POST` | `/v1/rag/add/document` | INSERT puro di testo libero (chunking auto) |
| `POST` | `/v1/rag/add/chunks` | INSERT puro di chunk preformattati |
| `POST` | `/v1/rag/remove` | **REMOVE**: per prefisso o exact match |
| `POST` | `/v1/rag/reload` | Ricostruzione totale indice da file in `kb/` |
> 💡 **Pattern CRUD consigliato**:
> - CREATE → `/upsert/document` (idempotente: se la source non esiste la crea)
> - READ → `/sources` (lista sorgenti) + `/chunks?source_exact=X` (dettaglio)
> - UPDATE → `/upsert/document` (stessa source, testo nuovo → sostituzione atomica)
> - DELETE → `/upsert/document` con `text: ""` oppure `/remove` con `exact: true`
---
### 13.2 Esempi curl
> Base URL: `http://127.0.0.1:12345/v1/rag` (cambia porta e host se hai modificato lo script).
#### 📤 Aggiungere un documento (testo libero, chunking automatico)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/document \
-H 'Content-Type: application/json' \
-d '{
"source": "crm/cliente_456_nota_20260826",
"text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual, renovación 15/02/2027. Contacto: +34 600 111 222, javier@empresa.es. Notas: prefiere que le llamen por la mañana antes de las 11h. Ha reportado un incidente con la factura número 2026-08-114 el 20/08/2026 que ya fue resuelto con un descuento del 15% aplicado en la siguiente factura.",
"metadata": {"cliente_id": 456, "pais": "es", "categoria": "crm"},
"persist": true,
"dynamic": true
}' | jq
```
**Risposta:**
```json
{
"added": 1,
"sources": ["crm/cliente_456_nota_20260826"],
"total_after": 10
}
```
> 💡 **`persist: true`** → il chunk viene scritto anche in `kb/_dynamic.jsonl`,
> quindi **al prossimo riavvio** della pipeline sarà ancora presente.
> Imposta a `false` per chunk temporanei (solo sessione corrente).
#### 📤 Aggiungere N chunk pre-formattati (es. da DB)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/chunks \
-H 'Content-Type: application/json' \
-d '{
"items": [
{"text": "Pedido 8942 — 25/08/2026. Productos: Teclado RGB Pro x1, Ratón Ergonómico x1. Estado: enviado, tracking 1Z999AA10123456784. Entrega estimada: 27/08/2026.",
"source": "pedidos/cliente_456/8942",
"metadata": {"pedido_id": 8942, "estado": "enviado"}},
{"text": "Dirección de envío habitual del cliente 456: Av. Diagonal 444, puerta 3, 08013 Barcelona, España. Contacto entrega: +34 600 111 222.",
"source": "direcciones/cliente_456/principal",
"metadata": {"tipo": "envio", "predeterminada": true}}
],
"persist": true,
"dynamic": true
}' | jq
```
#### 🔎 Provare il retrieval da HTTP (debug, no audio)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/search \
-H 'Content-Type: application/json' \
-d '{
"query": "¿Qué pedido tiene Javier García y cuándo llega?",
"top_k": 3,
"threshold": 0.25
}' | jq
```
Risposta con i chunk recuperati e lo score:
```json
{
"query": "¿Qué pedido tiene Javier García y cuándo llega?",
"count": 2,
"results": [
{"text": "Pedido 8942 — 25/08/2026...",
"source": "pedidos/cliente_456/8942", "score": 0.813},
{"text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual...",
"source": "crm/cliente_456_nota_20260826", "score": 0.742}
]
}
```
#### 📊 Ver stato generale
```bash
curl -sS http://127.0.0.1:12345/v1/rag/status | jq
```
#### 🗑️ Eliminare tutti i chunk di un cliente
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \
-H 'Content-Type: application/json' \
-d '{"source_prefix": "crm/cliente_456"}' | jq
```
Suggerimento: usa prefissi intelligenti per organizzare la KB:
```
source_prefix = "crm/" → cancella tutta la rubrica
source_prefix = "pedidos/" → cancella tutti i dati ordini
source_prefix = "dynamic#" → cancella tutti i chunk temporanei auto-numerati
```
#### 🛡️ Remove con exact match (solo source ESATTA)
```bash
# Cancella SOLO il documento 'crm/cliente_456_nota_20260826'
# (non cancella 'crm/cliente_456_nota_20260827')
curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \
-H 'Content-Type: application/json' \
-d '{"source_prefix": "crm/cliente_456_nota_20260826", "exact": true}' | jq
```
#### 📋 Lista sorgenti (index inventory)
```bash
# Tutte le sorgenti nell'indice, con conteggio chunk
curl -sS http://127.0.0.1:12345/v1/rag/sources | jq
```
Esempio output:
```json
{
"count": 5,
"items": [
{"source": "01_faq_producto.md", "chunk_count": 5, "has_dynamic": false},
{"source": "crm/cliente_456_nota_20260826", "chunk_count": 1, "has_dynamic": true},
{"source": "pedidos/cliente_456/8942", "chunk_count": 1, "has_dynamic": true}
]
}
```
#### 🔍 Elencare chunk con filtri e paginazione
```bash
# GET con query params: solo chunk di cliente_456, paged
curl -sS 'http://127.0.0.1:12345/v1/rag/chunks?source_prefix=crm/cliente_456&limit=10&offset=0' | jq
# POST con body JSON: ORDINA per rilevanza su una query + filtro
curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/list \
-H 'Content-Type: application/json' \
-d '{
"source_prefix": "crm/",
"query": "descuento aplicado en factura",
"min_score": 0.20,
"limit": 10,
"include_text": true
}' | jq
```
Risposta:
```json
{
"total": 2,
"offset": 0,
"limit": 10,
"query": "descuento aplicado en factura",
"items": [
{
"index": 9,
"source": "crm/cliente_456_nota_20260826",
"chunk_index": 1000009,
"score": 0.731,
"metadata": {"cliente_id": 456, "categoria": "crm"},
"is_dynamic": true,
"text": "Cliente: Javier García... 15% descuento..."
}
]
}
```
Campi speciali:
- `index`: la posizione numerica nel vettore di chunk — passala direttamente a `/chunks/update?index=...`
- `score`: `null` senza query, float 0-1 con ordinamento per rilevanza
#### 🆙 Aggiornare un singolo chunk
Usa `/chunks/update` quando devi modificare **solo un pezzetto specifico** (es. correggere un errore di battitura, cambiare metadata). Per aggiornare **un intero documento** usa invece `/upsert/document`.
```bash
# Opzione A: aggiorna tramite `index` (da list_chunks → campo "index": 9)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \
-H 'Content-Type: application/json' \
-d '{
"index": 9,
"new_text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2028 (2 años). Descuento 20% acordado en la llamada del 10/08.",
"new_metadata": {"cliente_id": 456, "categoria": "crm", "ultima_revision": "2026-08-26"}
}' | jq
# Opzione B: aggiorna tramite (source, chunk_index)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \
-H 'Content-Type: application/json' \
-d '{
"source": "crm/cliente_456_nota_20260826",
"chunk_index": 1000009,
"new_source": "crm/cliente_456/nota_principal"
}' | jq
```
Risposta successo `200`:
```json
{
"updated": 1,
"chunk": {
"index": 9,
"source": "crm/cliente_456/nota_principal",
"chunk_index": 1000009,
"text": "Cliente: Javier García...",
"metadata": {"cliente_id": 456, "...": "..."},
"is_dynamic": true
}
}
```
Errori HTTP possibili:
- `404 CHUNK_NOT_FOUND`
- `409 AMBIGUOUS_SOURCE` → hai passato `source` da sola ma esistono N chunk con stessa source; aggiungi `chunk_index` oppure usa `index`
#### 🔄 Upsert documento (CRUD consigliato)
**Questo è l'endpoint che userai nel 90% dei casi.**
Idempotente sul campo `source`:
- Se la `source` non esiste → CREATE
- Se la `source` esiste → DELETE tutti i chunk vecchi + split del nuovo testo + INSERT
- Se mandi `text: ""` → DELETE atomico (nessun re-insert)
```bash
# CREATE/UPDATE di un documento intero (es. nota cliente)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \
-H 'Content-Type: application/json' \
-d '{
"source": "crm/cliente_456/perfil",
"text": "Javier García — ID 456. Plan Premium Anual, renovación automática 15/02/2028. Precio: 24€/mes. Email javier@empresa.es. Tel +34 600 111 222. Dirección envío habitual: Av. Diagonal 444, puerta 3, 08013 Barcelona. Notas: siempre prefiere atención por la mañana, antes de las 11h. Caso abierto 2026-08-0124: seguimiento calidad postventa, cerrado con valoración 5/5.",
"metadata": {"cliente_id": 456, "pais": "es", "actualizado": "2026-08-26"},
"dynamic": true,
"persist": true
}' | jq
```
Risposta:
```json
{
"removed": 2,
"added": 1,
"sources": ["crm/cliente_456/perfil"],
"total_after": 10,
"mode": "upsert"
}
```
DELETE tramite upsert:
```bash
# Elimina del tutto tutti i chunk associati a questa source (atomico)
curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \
-H 'Content-Type: application/json' \
-d '{"source": "crm/cliente_456/perfil", "text": ""}' | jq
```
#### 🔄 Ricaricare tutta la KB da file (dopo aver copiato nuovi md/txt)
```bash
curl -sS -X POST http://127.0.0.1:12345/v1/rag/reload \
-H 'Content-Type: application/json' \
-d '{"force_rebuild": true}' | jq
```
---
### 13.3 Persistenza cross-riavvio: `kb/_dynamic.jsonl`
Tutti i chunk aggiunti via API con `persist: true` vengono appesi a `kb/_dynamic.jsonl` (un JSON per riga). Al prossimo avvio della pipeline:
1. Viene ricostruito l'indice statico da file md/txt/jsonl
2. **Subito dopo** vengono ricaricati e ri-embeddati anche tutti i chunk da `_dynamic.jsonl`
Quindi la conoscenza aggiunta via HTTP è **durevole** e non si perde riavviando.
---
### 13.4 Thread safety
Tutte le API usano un `threading.RLock()` all'interno di [RAGRetriever](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py):
- ✅ Più chiamate `/search` possono andare in parallelo durante la conversazione
- ✅ Una `add_document`/`remove` blocca brevemente l'indice solo per le operazioni di vstack/take numpy — retrieval non si perde, viene atteso
- ✅ Persistenza su disco (NPZ + JSONL) avviene fuori dal lock, quindi non rallenta la chiamata conversazionale
---
## 14. Roadmap / miglioramenti possibili
- [ ] **Supporto PDF**: aggiungere `pypdf` o `pdfplumber` in gruppo opzionale
- [ ] **MMR reranking** invece del solo top-k coseno (migliore diversità chunk)
- [ ] **HyDE** (LLM genera query ipotetica + embedding di quella, per query colloquiali)
- [ ] **Cross-encoder reranker** 2-stage per KB grandi (>10k chunk)
- [ ] **Hook su LanguageModelHandler locale** (mlx-lm / transformers) — attualmente l'RAG è attivo solo per backends `chat-completions` e `responses-api`
- [ ] **Watchdog KB**: auto-rebuild indice quando i file cambiano (inotify / watchdog)

BIN
docs/assets/endpoint-swap-dark.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 473 KiB

BIN
docs/assets/endpoint-swap-light.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 464 KiB

BIN
female_short.wav Normal file
View File

Binary file not shown.

33
kb/01_faq_producto.md Normal file
View File

@ -0,0 +1,33 @@
# Preguntas Frecuentes - Producto (Ejemplo)
> Este es un archivo de ejemplo. Sustitúyelo con tus documentos reales.
## ¿Cómo puedo restablecer mi contraseña?
Para restablecer tu contraseña ve a la página de inicio de sesión y haz clic en
*"Olvidé mi contraseña"*. Recibirás un correo electrónico con un enlace válido
durante 15 minutos. Si no recibes el correo, revisa la carpeta de spam o ponte en
contacto con soporte en soporte@empresa.es.
## ¿Cuál es el horario de atención al cliente?
El horario de atención telefónica y por chat es de **lunes a viernes de 09:00 a 18:00**
(hora peninsular española). Los fines de semana solo hay soporte por email con
respuesta máxima en 24 horas.
## ¿Qué métodos de pago aceptáis?
Aceptamos Visa, Mastercard, American Express, PayPal, transferencia bancaria y
Bizum. Para clientes empresariales disponemos de pago a 30 días tras aprobación
del crédito.
## ¿Ofrecéis reembolsos?
Sí. Todos los planes tienen un periodo de devolución de **14 días naturales**
desde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos
se evalúan caso por caso por el equipo de soporte.
## ¿En qué idiomas está disponible la plataforma?
La interfaz web está disponible en español, inglés, francés, italiano y portugués.
El soporte por chat se ofrece únicamente en español e inglés de momento.

31
kb/02_politicas_internas.md Normal file
View File

@ -0,0 +1,31 @@
# Políticas Internas - Ejemplo
> Este es un archivo de ejemplo. Elíminalo o sustitúyelo por tus políticas reales.
## Política de privacidad de datos de clientes
Todos los datos de los clientes se almacenan cifrados en repositorios UE (Irlanda
y Frankfurt) según el Reglamento General de Protección de Datos (RGPD). Los datos
de pago nunca se almacenan en nuestros servidores; son procesados directamente por
nuestro proveedor PCI-DSS certificado (Stripe).
## Retención de registros
- Datos de usuario activo: mientras dure la relación contractual + 1 mes.
- Datos de usuario dado de baja: 6 meses (obligación fiscal).
- Registros de auditoría y acceso: 2 años.
- Grabaciones de llamadas de soporte: 90 días, salvo controversia judicial.
## Niveles de servicio (SLA)
El servicio tiene un **SLA de disponibilidad del 99,9% mensual** para clientes del
plan Pro y Enterprise. Si no se cumple, se aplica un crédito proporcional del 10%
sobre la cuota mensual. Para incidentes de severidad 1 (servicio caído) el tiempo
medio de respuesta es inferior a 1 hora en horario comercial.
## Seguridad: autenticación y sesiones
- Todos los usuarios deben usar autenticación en dos pasos (2FA) obligatoria.
- Las sesiones web expiran a los 60 minutos de inactividad.
- Contraseñas: mínimo 12 caracteres, 1 mayúscula, 1 número, 1 símbolo. Rotación
recomendada cada 90 días.

19
kb/README.md Normal file
View File

@ -0,0 +1,19 @@
# Base de Conocimiento - Knowledge Base (Scaffold de ejemplo)
Coloca aquí tus documentos en formato **Markdown (.md)** o **Texto plano (.txt)**.
Se prefieres pre-chunkizar, puedes usar archivos **JSONL** (una línea por chunk):
```jsonl
{"text": "Texto del chunk 1...", "source": "manual_usuario.pdf", "chunk_index": 0, "metadata": {"pagina": 12}}
{"text": "Texto del chunk 2...", "source": "manual_usuario.pdf", "chunk_index": 1, "metadata": {"pagina": 13}}
```
## Primera ejecución
1. Añade tus documentos a esta carpeta (subcarpetas permitidas).
2. Arranca la pipeline con la flag `--rag_enabled`.
3. Se generarán automáticamente:
- `_index.npz` → matriz embeddings (numpy, persistente)
- `_chunks.jsonl` → lista chunks con metadata
Para **reindexar** después de añadir documentos nuevos: usa `--rag_force_rebuild` o borra manualmente `_index.npz` y `_chunks.jsonl`.

11
kb/_chunks.jsonl Normal file
View File

@ -0,0 +1,11 @@
{"text": "# Preguntas Frecuentes - Producto (Ejemplo)\n\n> Este es un archivo de ejemplo. Sustitúyelo con tus documentos reales.\n\n## ¿Cómo puedo restablecer mi contraseña?\n\nPara restablecer tu contraseña ve a la página de inicio de sesión y haz clic en\n*\"Olvidé mi contraseña\"*. Recibirás un correo electrónico con un enlace válido\ndurante 15 minutos. Si no recibes el correo, revisa la carpeta de spam o ponte en\ncontacto con soporte en soporte@empresa.es.\n\n## ¿Cuál es el horario de atención al cliente?", "source": "01_faq_producto.md", "chunk_index": 0, "metadata": {"splitted": true}}
{"text": "orte@empresa.es.\n\n## ¿Cuál es el horario de atención al cliente?El horario de atención telefónica y por chat es de **lunes a viernes de 09:00 a 18:00**\n(hora peninsular española). Los fines de semana solo hay soporte por email con\nrespuesta máxima en 24 horas.\n\n## ¿Qué métodos de pago aceptáis?\n\nAceptamos Visa, Mastercard, American Express, PayPal, transferencia bancaria y\nBizum. Para clientes empresariales disponemos de pago a 30 días tras aprobación\ndel crédito.\n\n## ¿Ofrecéis reembolsos?", "source": "01_faq_producto.md", "chunk_index": 1, "metadata": {"splitted": true}}
{"text": "a 30 días tras aprobación\ndel crédito.\n\n## ¿Ofrecéis reembolsos?Sí. Todos los planes tienen un periodo de devolución de **14 días naturales**\ndesde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos\nse evalúan caso por caso por el equipo de soporte.\n\n## ¿En qué idiomas está disponible la plataforma?\n\nLa interfaz web está disponible en español, inglés, francés, italiano y portugués.\nEl soporte por chat se ofrece únicamente en español e inglés de momento.", "source": "01_faq_producto.md", "chunk_index": 2, "metadata": {"splitted": true}}
{"text": "# Políticas Internas - Ejemplo\n\n> Este es un archivo de ejemplo. Elíminalo o sustitúyelo por tus políticas reales.\n\n## Política de privacidad de datos de clientes\n\nTodos los datos de los clientes se almacenan cifrados en repositorios UE (Irlanda\ny Frankfurt) según el Reglamento General de Protección de Datos (RGPD). Los datos\nde pago nunca se almacenan en nuestros servidores; son procesados directamente por\nnuestro proveedor PCI-DSS certificado (Stripe).\n\n## Retención de registros", "source": "02_politicas_internas.md", "chunk_index": 0, "metadata": {"splitted": true}}
{"text": "oveedor PCI-DSS certificado (Stripe).\n\n## Retención de registros- Datos de usuario activo: mientras dure la relación contractual + 1 mes.\n- Datos de usuario dado de baja: 6 meses (obligación fiscal).\n- Registros de auditoría y acceso: 2 años.\n- Grabaciones de llamadas de soporte: 90 días, salvo controversia judicial.\n\n## Niveles de servicio (SLA)", "source": "02_politicas_internas.md", "chunk_index": 1, "metadata": {"splitted": true}}
{"text": "días, salvo controversia judicial.\n\n## Niveles de servicio (SLA)El servicio tiene un **SLA de disponibilidad del 99,9% mensual** para clientes del\nplan Pro y Enterprise. Si no se cumple, se aplica un crédito proporcional del 10%\nsobre la cuota mensual. Para incidentes de severidad 1 (servicio caído) el tiempo\nmedio de respuesta es inferior a 1 hora en horario comercial.\n\n## Seguridad: autenticación y sesiones", "source": "02_politicas_internas.md", "chunk_index": 2, "metadata": {"splitted": true}}
{"text": "ra en horario comercial.\n\n## Seguridad: autenticación y sesiones- Todos los usuarios deben usar autenticación en dos pasos (2FA) obligatoria.\n- Las sesiones web expiran a los 60 minutos de inactividad.\n- Contraseñas: mínimo 12 caracteres, 1 mayúscula, 1 número, 1 símbolo. Rotación\n recomendada cada 90 días.", "source": "02_politicas_internas.md", "chunk_index": 3, "metadata": {"splitted": true}}
{"text": "# Base de Conocimiento - Knowledge Base (Scaffold de ejemplo)\n\nColoca aquí tus documentos en formato **Markdown (.md)** o **Texto plano (.txt)**.\nSe prefieres pre-chunkizar, puedes usar archivos **JSONL** (una línea por chunk):\n\n```jsonl\n{\"text\": \"Texto del chunk 1...\", \"source\": \"manual_usuario.pdf\", \"chunk_index\": 0, \"metadata\": {\"pagina\": 12}}\n{\"text\": \"Texto del chunk 2...\", \"source\": \"manual_usuario.pdf\", \"chunk_index\": 1, \"metadata\": {\"pagina\": 13}}\n```\n\n## Primera ejecución", "source": "README.md", "chunk_index": 0, "metadata": {"splitted": true}}
{"text": "index\": 1, \"metadata\": {\"pagina\": 13}}\n```\n\n## Primera ejecución1. Añade tus documentos a esta carpeta (subcarpetas permitidas).\n2. Arranca la pipeline con la flag `--rag_enabled`.\n3. Se generarán automáticamente:\n - `_index.npz` → matriz embeddings (numpy, persistente)\n - `_chunks.jsonl` → lista chunks con metadata\n\nPara **reindexar** después de añadir documentos nuevos: usa `--rag_force_rebuild` o borra manualmente `_index.npz` y `_chunks.jsonl`.", "source": "README.md", "chunk_index": 1, "metadata": {"splitted": true}}
{"text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2027. Tel +34 600 111 222. Incidente factura 2026-08-114 resuelto con 15% descuento.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000009, "metadata": {"cliente_id": 456, "categoria": "crm", "_dynamic": true}}
{"text": "Cliente: Pippo Pappo, id=123. Falta pago rata mensual.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000010, "metadata": {"cliente_id": 123, "categoria": "crm", "_dynamic": true}}

2
kb/_dynamic.jsonl Normal file
View File

@ -0,0 +1,2 @@
{"text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2027. Tel +34 600 111 222. Incidente factura 2026-08-114 resuelto con 15% descuento.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000009, "metadata": {"cliente_id": 456, "categoria": "crm", "_dynamic": true}}
{"text": "Cliente: Pippo Pappo, id=123. Falta pago rata mensual.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000010, "metadata": {"cliente_id": 123, "categoria": "crm", "_dynamic": true}}

BIN
kb/_index.npz Normal file
View File

Binary file not shown.

BIN
logo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 65 KiB

2
mypy.ini Normal file
View File

@ -0,0 +1,2 @@
[mypy]
ignore_missing_imports = True

146
pyproject.toml Normal file
View File

@ -0,0 +1,146 @@
[build-system]
requires = ["setuptools>=77.0.3", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "speech-to-speech"
version = "0.2.11"
description = "Low-latency speech-to-speech pipeline"
readme = "README.md"
requires-python = ">=3.10"
license = "Apache-2.0"
license-files = ["LICENSE"]
authors = [
{ name = "Hugging Face" },
]
keywords = ["speech-to-speech", "voice-agents", "speech-recognition", "text-to-speech", "openai"]
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Topic :: Multimedia :: Sound/Audio :: Speech",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
]
dependencies = [
"fastapi>=0.115.0",
"httpx>=0.28.0",
"nltk==3.9.4",
"numpy>=1.26.0,<2.4.4; platform_system == 'Darwin'",
"numpy>=1.26.0; platform_system != 'Darwin'",
"openai==2.28.0",
"pillow>=10.0.0",
"pydantic>=2.0",
"rich>=13.0",
"scipy>=1.10.0",
"sounddevice==0.5.3; platform_system == 'Darwin'",
"sounddevice>=0.5.0; platform_system != 'Darwin'",
"soundfile>=0.13.0; platform_system == 'Darwin'",
"torch==2.11.0; platform_system == 'Darwin'",
"torch>=2.4.0; platform_system != 'Darwin'",
"torchaudio==2.11.0; platform_system == 'Darwin'",
"torchaudio>=2.4.0; platform_system != 'Darwin'",
"transformers==5.6.2; platform_system == 'Darwin'",
"transformers>=4.57.0; platform_system != 'Darwin'",
"uvicorn>=0.30.0",
"websockets>=12.0",
"nano-parakeet>=0.2.0; platform_system != 'Darwin'",
"faster-qwen3-tts[ggml]>=0.3.2; platform_system != 'Darwin' and platform_system != 'Windows'",
"faster-qwen3-tts>=0.3.2; platform_system == 'Windows'",
"lingua-language-detector>=2.0.2",
"miniaudio==1.61; platform_system == 'Darwin'",
"mlx==0.31.1; platform_system == 'Darwin'",
"mlx-audio==0.4.2; platform_system == 'Darwin'",
"mlx-lm==0.31.1; platform_system == 'Darwin'",
"mlx-metal==0.31.1; platform_system == 'Darwin'",
"misaki>=0.9.4; platform_system == 'Darwin'",
"espeakng-loader>=0.2.4; platform_system == 'Darwin'",
"spacy>=3.8.4; platform_system == 'Darwin'",
"phonemizer-fork>=3.3.2; platform_system == 'Darwin'",
]
[project.optional-dependencies]
chattts = [
"ChatTTS>=0.1.1",
]
facebook-mms = [
"transformers>=4.57.0",
]
faster-whisper = [
"faster-whisper>=1.0.3",
]
kokoro = [
"kokoro>=0.9.2; platform_system != 'Darwin'",
]
language-detection = [
"lingua-language-detector>=2.0.2",
]
mlx-lm = [
"mlx-lm==0.31.1; platform_system == 'Darwin'",
"mlx-vlm==0.4.1; platform_system == 'Darwin'",
]
paraformer = [
"funasr>=1.1.6",
"modelscope>=1.17.1",
"onnxruntime<1.24; python_version < '3.11'",
]
pocket = [
"pocket-tts>=0.1.0",
]
rag = [
"sentence-transformers>=3.0.0",
]
websocket = [
"websockets>=12.0",
]
whisper-mlx = [
"lightning-whisper-mlx>=0.0.10; platform_system == 'Darwin'",
]
[project.urls]
Homepage = "https://github.com/huggingface/speech-to-speech"
Repository = "https://github.com/huggingface/speech-to-speech"
Issues = "https://github.com/huggingface/speech-to-speech/issues"
[project.scripts]
speech-to-speech = "speech_to_speech.s2s_pipeline:main"
[dependency-groups]
dev = [
"ruff",
"mypy",
"pytest",
"pytest-asyncio",
"websockets>=12.0",
]
[tool.pytest.ini_options]
asyncio_mode = "auto"
[tool.uv]
package = true
prerelease = "allow"
[tool.setuptools.packages.find]
where = ["src"]
include = ["speech_to_speech*"]
exclude = ["tests*"]
[tool.setuptools.package-data]
speech_to_speech = ["TTS/*.wav"]
[tool.ruff]
src = ["src"]
line-length = 120
[tool.ruff.lint]
select = ["E", "F", "I", "W"]
ignore = ["E501"]
[tool.mypy]
python_version = "3.10"
ignore_missing_imports = true
warn_unused_configs = true
check_untyped_defs = false

385
scripts/benchmark_stt.py Normal file
View File

@ -0,0 +1,385 @@
"""
STT Benchmarking Script
This script benchmarks different Speech-to-Text (STT) handlers to compare their performance.
Measures: inference time, warmup time, memory usage, and transcription quality.
Usage:
python benchmark_stt.py --audio_file path/to/audio.wav --iterations 10
python benchmark_stt.py --audio_file path/to/audio.wav --handlers whisper mlx-audio-whisper
"""
import argparse
import json
import logging
import time
from pathlib import Path
from queue import Queue
from threading import Event
from typing import Any, Dict, List, Optional
import numpy as np
import soundfile as sf
from speech_to_speech.pipeline.messages import VADAudio
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)
class BenchmarkResult:
"""Stores benchmark results for a single STT handler."""
def __init__(self, handler_name: str):
self.handler_name = handler_name
self.warmup_time = 0.0
self.inference_times: list[float] = []
self.time_to_first_token: list[float] = []
self.transcriptions: list[str] = []
self.errors: list[str] = []
def add_inference(self, time_taken: float, transcription: Any, ttft: Optional[float] = None):
self.inference_times.append(time_taken)
self.transcriptions.append(transcription)
if ttft is not None:
self.time_to_first_token.append(ttft)
def add_error(self, error: str):
self.errors.append(error)
def get_stats(self) -> Dict[str, Any]:
"""Calculate statistics from benchmark results."""
if not self.inference_times:
return {
"handler": self.handler_name,
"status": "failed",
"errors": self.errors,
}
stats = {
"handler": self.handler_name,
"warmup_time": self.warmup_time,
"avg_inference_time": np.mean(self.inference_times),
"min_inference_time": np.min(self.inference_times),
"max_inference_time": np.max(self.inference_times),
"std_inference_time": np.std(self.inference_times),
"total_iterations": len(self.inference_times),
"errors": self.errors,
"sample_transcription": self.transcriptions[0] if self.transcriptions else None,
}
# Add time to first token stats if available
if self.time_to_first_token:
stats["avg_time_to_first_token"] = np.mean(self.time_to_first_token)
stats["min_time_to_first_token"] = np.min(self.time_to_first_token)
stats["max_time_to_first_token"] = np.max(self.time_to_first_token)
stats["std_time_to_first_token"] = np.std(self.time_to_first_token)
return stats
def load_audio(audio_path: str) -> np.ndarray:
"""Load audio file and return as numpy array."""
logger.info(f"Loading audio from: {audio_path}")
audio, sample_rate = sf.read(audio_path)
# Convert to mono if stereo
if len(audio.shape) > 1:
audio = audio.mean(axis=1)
# Resample to 16kHz if needed (most whisper models expect 16kHz)
if sample_rate != 16000:
logger.warning(f"Audio sample rate is {sample_rate}Hz, resampling to 16000Hz")
try:
import librosa
audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000)
except ImportError:
logger.error("librosa not installed. Please install it with: pip install librosa")
logger.error("Attempting scipy resampling as fallback...")
from scipy import signal
# Calculate resampling ratio
num_samples = int(len(audio) * 16000 / sample_rate)
audio = signal.resample(audio, num_samples)
return audio.astype(np.float32)
def benchmark_handler(
handler_name: str,
audio: np.ndarray,
iterations: int,
handler_kwargs: Optional[Dict[str, Any]] = None
) -> BenchmarkResult:
"""Benchmark a single STT handler."""
logger.info(f"Benchmarking {handler_name}...")
result = BenchmarkResult(handler_name)
try:
# Create queues and events for handler
stop_event = Event()
queue_in: Queue[Any] = Queue()
queue_out: Queue[Any] = Queue()
handler: Any = None
if handler_name == "whisper":
from speech_to_speech.STT.whisper_stt_handler import WhisperSTTHandler
setup_kwargs = handler_kwargs or {
"model_name": "distil-whisper/distil-large-v3",
"device": "cuda",
"torch_dtype": "float16",
}
handler = WhisperSTTHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_kwargs=setup_kwargs
)
elif handler_name == "whisper-mlx":
from speech_to_speech.STT.lightning_whisper_mlx_handler import LightningWhisperSTTHandler
setup_kwargs = handler_kwargs or {
"model_name": "large-v3",
"device": "mps",
}
handler = LightningWhisperSTTHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_kwargs=setup_kwargs
)
elif handler_name == "mlx-audio-whisper":
from speech_to_speech.STT.mlx_audio_whisper_handler import MLXAudioWhisperSTTHandler
setup_kwargs = handler_kwargs or {
"model_name": "mlx-community/whisper-large-v3-turbo",
}
handler = MLXAudioWhisperSTTHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_kwargs=setup_kwargs
)
elif handler_name == "faster-whisper":
from speech_to_speech.STT.faster_whisper_handler import FasterWhisperSTTHandler
setup_kwargs = handler_kwargs or {
"model_name": "large-v3",
"device": "auto",
"compute_type": "float16",
}
handler = FasterWhisperSTTHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_kwargs=setup_kwargs
)
elif handler_name == "moonshine":
from archive.STT.moonshine_handler import MoonshineSTTHandler
handler = MoonshineSTTHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
)
elif handler_name == "parakeet-tdt":
from speech_to_speech.STT.parakeet_tdt_handler import ParakeetTDTSTTHandler
setup_kwargs = handler_kwargs or {
"device": "mps",
"enable_live_transcription": False,
}
handler = ParakeetTDTSTTHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_kwargs=setup_kwargs
)
elif handler_name == "parakeet-tdt-progressive":
from speech_to_speech.STT.parakeet_tdt_handler import ParakeetTDTSTTHandler
setup_kwargs = handler_kwargs or {
"device": "mps",
"enable_live_transcription": True,
"live_transcription_update_interval": 0.25,
}
handler = ParakeetTDTSTTHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_kwargs=setup_kwargs
)
else:
raise ValueError(f"Unknown handler: {handler_name}")
# Warmup is done in handler setup
logger.info(f"Handler {handler_name} initialized and warmed up")
# Additional warmup on the actual audio (excluded from timings)
for _ in handler.process(VADAudio(audio=audio)):
pass
# Run benchmark iterations
for i in range(iterations):
logger.info(f"Iteration {i+1}/{iterations} for {handler_name}")
start_time = time.perf_counter()
# Process audio
transcription = None
time_to_first_token = None
first_output = True
for output in handler.process(VADAudio(audio=audio)):
# Measure time to first token
if first_output:
time_to_first_token = time.perf_counter() - start_time
first_output = False
if isinstance(output, tuple):
transcription = output[0] # (text, language)
else:
transcription = output
end_time = time.perf_counter()
time_taken = end_time - start_time
result.add_inference(time_taken, transcription, time_to_first_token)
ttft_str = f", TTFT: {time_to_first_token:.4f}s" if time_to_first_token else ""
text_preview = str(transcription)[:50] if transcription is not None else "(none)"
logger.info(f" Time: {time_taken:.4f}s{ttft_str}, Text: {text_preview}...")
# Cleanup
handler.cleanup()
stop_event.set()
except Exception as e:
logger.error(f"Error benchmarking {handler_name}: {e}", exc_info=True)
result.add_error(str(e))
return result
def print_results(results: List[BenchmarkResult]):
"""Print benchmark results in a formatted table."""
print("\n" + "="*80)
print("BENCHMARK RESULTS")
print("="*80)
for result in results:
stats = result.get_stats()
print(f"\nHandler: {stats['handler']}")
print("-" * 80)
if stats.get("status") == "failed":
print(" Status: FAILED")
print(f" Errors: {stats['errors']}")
continue
print(f" Warmup Time: {stats['warmup_time']:.4f}s")
print(f" Avg Inference Time: {stats['avg_inference_time']:.4f}s")
print(f" Min Inference Time: {stats['min_inference_time']:.4f}s")
print(f" Max Inference Time: {stats['max_inference_time']:.4f}s")
print(f" Std Deviation: {stats['std_inference_time']:.4f}s")
# Print time to first token stats if available
if 'avg_time_to_first_token' in stats:
print("\n Time to First Token:")
print(f" Avg TTFT: {stats['avg_time_to_first_token']:.4f}s")
print(f" Min TTFT: {stats['min_time_to_first_token']:.4f}s")
print(f" Max TTFT: {stats['max_time_to_first_token']:.4f}s")
print(f" Std TTFT: {stats['std_time_to_first_token']:.4f}s")
print(f"\n Total Iterations: {stats['total_iterations']}")
print(f" Sample Transcription: {stats['sample_transcription']}")
if stats['errors']:
print(f" Errors: {stats['errors']}")
# Comparison table
print("\n" + "="*80)
print("COMPARISON (Average Inference Time)")
print("="*80)
successful_results = [r for r in results if r.inference_times]
if successful_results:
sorted_results = sorted(successful_results, key=lambda x: np.mean(x.inference_times))
fastest = sorted_results[0]
fastest_time = np.mean(fastest.inference_times)
for result in sorted_results:
avg_time = np.mean(result.inference_times)
speedup = avg_time / fastest_time
print(f" {result.handler_name:25s}: {avg_time:.4f}s ({speedup:.2f}x slower than fastest)")
def save_results(results: List[BenchmarkResult], output_file: str):
"""Save benchmark results to JSON file."""
data = {
"results": [r.get_stats() for r in results],
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
}
with open(output_file, 'w') as f:
json.dump(data, f, indent=2)
logger.info(f"Results saved to: {output_file}")
def main():
parser = argparse.ArgumentParser(description="Benchmark STT handlers")
parser.add_argument(
"--audio_file",
type=str,
required=True,
help="Path to audio file for benchmarking"
)
parser.add_argument(
"--handlers",
nargs="+",
default=["whisper", "whisper-mlx", "mlx-audio-whisper", "faster-whisper", "parakeet-tdt", "parakeet-tdt-progressive"],
help="List of handlers to benchmark (default: all)"
)
parser.add_argument(
"--iterations",
type=int,
default=5,
help="Number of iterations per handler (default: 5)"
)
parser.add_argument(
"--output",
type=str,
default="stt_benchmark_results.json",
help="Output JSON file for results (default: stt_benchmark_results.json)"
)
args = parser.parse_args()
# Validate audio file exists
if not Path(args.audio_file).exists():
logger.error(f"Audio file not found: {args.audio_file}")
return
# Load audio
audio = load_audio(args.audio_file)
logger.info(f"Audio loaded: {len(audio)} samples, {len(audio)/16000:.2f}s duration")
# Run benchmarks
results = []
for handler_name in args.handlers:
result = benchmark_handler(handler_name, audio, args.iterations)
results.append(result)
# Print and save results
print_results(results)
save_results(results, args.output)
logger.info("Benchmarking complete!")
if __name__ == "__main__":
main()

395
scripts/benchmark_tts.py Normal file
View File

@ -0,0 +1,395 @@
"""
TTS Benchmarking Script
Benchmarks Text-to-Speech (TTS) handlers to compare performance.
Measures: warmup time, inference time, time-to-first-chunk, audio duration, and RTF.
Usage:
python benchmark_tts.py --text "Hello world" --iterations 3
python benchmark_tts.py --handlers kokoro qwen3 pocket_tts
"""
import argparse
import json
import logging
import time
from queue import Queue
from threading import Event
from typing import Any, Dict, List, Optional
import numpy as np
from speech_to_speech.pipeline.messages import TTSInput
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)
DEFAULT_SAMPLE_RATE = 16000
VALID_QWEN3_MLX_QUANTIZATIONS = ("bf16", "4bit", "6bit", "8bit")
class BenchmarkResult:
"""Stores benchmark results for a single TTS handler."""
def __init__(self, handler_name: str):
self.handler_name = handler_name
self.warmup_time = 0.0
self.inference_times: list[float] = []
self.time_to_first_chunk: list[float] = []
self.audio_durations: list[float] = []
self.errors: list[str] = []
def add_inference(self, time_taken: float, audio_duration: float, ttfc: Optional[float] = None):
self.inference_times.append(time_taken)
self.audio_durations.append(audio_duration)
if ttfc is not None:
self.time_to_first_chunk.append(ttfc)
def add_error(self, error: str):
self.errors.append(error)
def get_stats(self) -> Dict[str, Any]:
if not self.inference_times:
return {
"handler": self.handler_name,
"status": "failed",
"errors": self.errors,
}
avg_time = float(np.mean(self.inference_times))
avg_audio = float(np.mean(self.audio_durations))
avg_rtf = avg_audio / avg_time if avg_time > 0 else 0.0
stats = {
"handler": self.handler_name,
"warmup_time": self.warmup_time,
"avg_inference_time": avg_time,
"min_inference_time": float(np.min(self.inference_times)),
"max_inference_time": float(np.max(self.inference_times)),
"std_inference_time": float(np.std(self.inference_times)),
"avg_audio_duration": avg_audio,
"min_audio_duration": float(np.min(self.audio_durations)),
"max_audio_duration": float(np.max(self.audio_durations)),
"std_audio_duration": float(np.std(self.audio_durations)),
"avg_rtf": avg_rtf,
"total_iterations": len(self.inference_times),
"errors": self.errors,
}
if self.time_to_first_chunk:
stats["avg_time_to_first_chunk"] = float(np.mean(self.time_to_first_chunk))
stats["min_time_to_first_chunk"] = float(np.min(self.time_to_first_chunk))
stats["max_time_to_first_chunk"] = float(np.max(self.time_to_first_chunk))
stats["std_time_to_first_chunk"] = float(np.std(self.time_to_first_chunk))
return stats
def benchmark_handler(
handler_name: str,
text: str,
iterations: int,
handler_kwargs: Optional[Dict[str, Any]] = None,
language_code: Optional[str] = "en",
) -> BenchmarkResult:
logger.info(f"Benchmarking {handler_name}...")
result = BenchmarkResult(handler_name)
try:
stop_event = Event()
should_listen = Event()
queue_in: Queue[Any] = Queue()
queue_out: Queue[Any] = Queue()
handler: Any = None
setup_kwargs = handler_kwargs or {}
start_setup = time.perf_counter()
if handler_name == "kokoro":
from speech_to_speech.TTS.kokoro_handler import KokoroTTSHandler
setup_kwargs = {"device": "auto", **setup_kwargs}
handler = KokoroTTSHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_args=(should_listen,),
setup_kwargs=setup_kwargs,
)
elif handler_name == "pocket_tts":
from speech_to_speech.TTS.pocket_tts_handler import PocketTTSHandler
setup_kwargs = {"device": "cpu", **setup_kwargs}
handler = PocketTTSHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_args=(should_listen,),
setup_kwargs=setup_kwargs,
)
elif handler_name == "qwen3":
from speech_to_speech.TTS.qwen3_tts_handler import Qwen3TTSHandler
setup_kwargs = {
"device": "cuda",
"model_name": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
"ref_audio": "TTS/ref_audio.wav",
**setup_kwargs,
}
handler = Qwen3TTSHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_args=(should_listen,),
setup_kwargs=setup_kwargs,
)
elif handler_name == "chatTTS":
from speech_to_speech.TTS.chatTTS_handler import ChatTTSHandler
setup_kwargs = {"device": "cuda", **setup_kwargs}
handler = ChatTTSHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_args=(should_listen,),
setup_kwargs=setup_kwargs,
)
elif handler_name == "facebookMMS":
from speech_to_speech.TTS.facebookmms_handler import FacebookMMSTTSHandler
setup_kwargs = {"device": "cuda", "language": "en", **setup_kwargs}
handler = FacebookMMSTTSHandler(
stop_event,
queue_in=queue_in,
queue_out=queue_out,
setup_args=(should_listen,),
setup_kwargs=setup_kwargs,
)
else:
raise ValueError(f"Unknown handler: {handler_name}")
result.warmup_time = time.perf_counter() - start_setup
logger.info(f"Handler {handler_name} initialized and warmed up in {result.warmup_time:.3f}s")
for i in range(iterations):
logger.info(f"Iteration {i+1}/{iterations} for {handler_name}")
start_time = time.perf_counter()
time_to_first_chunk = None
first_output = True
total_samples = 0
tts_input = TTSInput(text=text, language_code=language_code)
for chunk in handler.process(tts_input):
if first_output:
time_to_first_chunk = time.perf_counter() - start_time
first_output = False
if chunk is None:
continue
try:
total_samples += len(chunk)
except Exception:
pass
end_time = time.perf_counter()
time_taken = end_time - start_time
audio_duration = total_samples / DEFAULT_SAMPLE_RATE if total_samples > 0 else 0.0
result.add_inference(time_taken, audio_duration, time_to_first_chunk)
ttfc_str = f", TTFC: {time_to_first_chunk:.4f}s" if time_to_first_chunk else ""
logger.info(
f" Time: {time_taken:.4f}s{ttfc_str}, Audio: {audio_duration:.2f}s, RTF: {audio_duration / time_taken if time_taken > 0 else 0:.2f}"
)
handler.cleanup()
stop_event.set()
except Exception as e:
logger.error(f"Error benchmarking {handler_name}: {e}", exc_info=True)
result.add_error(str(e))
return result
def normalize_qwen3_mlx_quantizations(values: List[str] | None) -> List[str]:
if not values:
return []
normalized = []
seen = set()
for value in values:
quantization = str(value).strip().lower()
if quantization in ("default", "none", ""):
quantization = "bf16"
if quantization not in VALID_QWEN3_MLX_QUANTIZATIONS:
raise ValueError(
"Unsupported qwen3 MLX quantization "
f"{value!r}. Supported values: {', '.join(VALID_QWEN3_MLX_QUANTIZATIONS)}"
)
if quantization in seen:
continue
seen.add(quantization)
normalized.append(quantization)
return normalized
def build_benchmark_targets(args) -> List[tuple[str, str, Dict[str, Any]]]:
targets = []
qwen3_quantizations = normalize_qwen3_mlx_quantizations(args.qwen3_mlx_quantizations)
for handler_name in args.handlers:
if handler_name == "qwen3" and qwen3_quantizations:
for quantization in qwen3_quantizations:
targets.append(
(
f"qwen3[{quantization}]",
"qwen3",
{"mlx_quantization": quantization},
)
)
continue
targets.append((handler_name, handler_name, {}))
return targets
def print_results(results: List[BenchmarkResult]):
print("\n" + "=" * 80)
print("TTS BENCHMARK RESULTS")
print("=" * 80)
for result in results:
stats = result.get_stats()
print(f"\nHandler: {stats['handler']}")
print("-" * 80)
if stats.get("status") == "failed":
print(" Status: FAILED")
print(f" Errors: {stats['errors']}")
continue
print(f" Warmup Time: {stats['warmup_time']:.4f}s")
print(f" Avg Inference Time: {stats['avg_inference_time']:.4f}s")
print(f" Min Inference Time: {stats['min_inference_time']:.4f}s")
print(f" Max Inference Time: {stats['max_inference_time']:.4f}s")
print(f" Std Deviation: {stats['std_inference_time']:.4f}s")
print(f" Avg Audio Duration: {stats['avg_audio_duration']:.2f}s")
print(f" Min Audio Duration: {stats['min_audio_duration']:.2f}s")
print(f" Max Audio Duration: {stats['max_audio_duration']:.2f}s")
print(f" Std Audio Duration: {stats['std_audio_duration']:.4f}s")
print(f" Avg RTF: {stats['avg_rtf']:.2f}")
if "avg_time_to_first_chunk" in stats:
print("\n Time to First Chunk:")
print(f" Avg TTFC: {stats['avg_time_to_first_chunk']:.4f}s")
print(f" Min TTFC: {stats['min_time_to_first_chunk']:.4f}s")
print(f" Max TTFC: {stats['max_time_to_first_chunk']:.4f}s")
print(f" Std TTFC: {stats['std_time_to_first_chunk']:.4f}s")
print(f"\n Total Iterations: {stats['total_iterations']}")
if stats["errors"]:
print(f" Errors: {stats['errors']}")
print("\n" + "=" * 80)
print("COMPARISON (Average Inference Time)")
print("=" * 80)
successful_results = [r for r in results if r.inference_times]
if successful_results:
sorted_results = sorted(successful_results, key=lambda x: np.mean(x.inference_times))
fastest = sorted_results[0]
fastest_time = np.mean(fastest.inference_times)
for result in sorted_results:
avg_time = np.mean(result.inference_times)
speedup = avg_time / fastest_time
print(f" {result.handler_name:25s}: {avg_time:.4f}s ({speedup:.2f}x slower than fastest)")
def save_results(results: List[BenchmarkResult], output_file: str):
data = {
"results": [r.get_stats() for r in results],
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
}
with open(output_file, "w") as f:
json.dump(data, f, indent=2)
logger.info(f"Results saved to: {output_file}")
def main():
parser = argparse.ArgumentParser(description="Benchmark TTS handlers")
parser.add_argument(
"--text",
type=str,
default="Hello from the speech to speech benchmark. This is a latency test.",
help="Text to synthesize",
)
parser.add_argument(
"--handlers",
nargs="+",
default=["kokoro", "qwen3", "pocket_tts"],
help="List of handlers to benchmark (kokoro, qwen3, pocket_tts, chatTTS, facebookMMS)",
)
parser.add_argument(
"--iterations",
type=int,
default=3,
help="Number of iterations per handler (default: 3)",
)
parser.add_argument(
"--output",
type=str,
default="tts_benchmark_results.json",
help="Output JSON file for results (default: tts_benchmark_results.json)",
)
parser.add_argument(
"--language_code",
type=str,
default="en",
help="Language code to pass to TTS handlers (default: en)",
)
parser.add_argument(
"--qwen3_mlx_quantizations",
nargs="+",
default=None,
help=(
"Optional list of Apple Silicon MLX Qwen3-TTS quantizations to benchmark "
"as separate variants. Supported values: bf16, 4bit, 6bit, 8bit."
),
)
args = parser.parse_args()
if not args.handlers:
logger.error("No handlers provided")
return
try:
targets = build_benchmark_targets(args)
except ValueError as e:
logger.error(str(e))
return
results = []
for result_name, handler_name, handler_kwargs in targets:
result = benchmark_handler(
handler_name,
args.text,
args.iterations,
handler_kwargs=handler_kwargs,
language_code=args.language_code,
)
result.handler_name = result_name
results.append(result)
print_results(results)
save_results(results, args.output)
logger.info("TTS benchmarking complete!")
if __name__ == "__main__":
main()

136
scripts/listen_and_play.py Normal file
View File

@ -0,0 +1,136 @@
import socket
import threading
from dataclasses import dataclass, field
from queue import Queue
import numpy as np
import sounddevice as sd
from transformers import HfArgumentParser
@dataclass
class ListenAndPlayArguments:
send_rate: int = field(default=16000, metadata={"help": "In Hz. Default is 16000."})
recv_rate: int = field(default=16000, metadata={"help": "In Hz. Default is 16000."})
list_play_chunk_size: int = field(
default=1024,
metadata={"help": "The size of data chunks (in bytes). Default is 1024."},
)
host: str = field(
default="localhost",
metadata={
"help": "The hostname or IP address for listening and playing. Default is 'localhost'."
},
)
send_port: int = field(
default=12345,
metadata={"help": "The network port for sending data. Default is 12345."},
)
recv_port: int = field(
default=12346,
metadata={"help": "The network port for receiving data. Default is 12346."},
)
def listen_and_play(
send_rate=16000,
recv_rate=44100,
list_play_chunk_size=1024,
host="localhost",
send_port=12345,
recv_port=12346,
):
send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
send_socket.connect((host, send_port))
recv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
recv_socket.connect((host, recv_port))
print("Recording and streaming...")
stop_event = threading.Event()
recv_queue = Queue()
send_queue = Queue()
# Pre-generate a static dither buffer (±1 LSB, -96 dB) to keep the audio
# sink active without calling numpy inside the real-time audio callback.
dither_bytes = np.random.randint(
-1, 2, size=list_play_chunk_size, dtype=np.int16
).tobytes()
def callback_recv(outdata, frames, time, status):
if not recv_queue.empty():
data = recv_queue.get()
outdata[: len(data)] = data
outdata[len(data) :] = b"\x00" * (len(outdata) - len(data))
else:
outdata[:] = dither_bytes
def callback_send(indata, frames, time, status):
if recv_queue.empty():
data = bytes(indata)
send_queue.put(data)
def send(stop_event, send_queue):
while not stop_event.is_set():
data = send_queue.get()
send_socket.sendall(data)
def recv(stop_event, recv_queue):
def receive_full_chunk(conn, chunk_size):
data = b""
while len(data) < chunk_size:
packet = conn.recv(chunk_size - len(data))
if not packet:
return None # Connection has been closed
data += packet
return data
while not stop_event.is_set():
data = receive_full_chunk(recv_socket, list_play_chunk_size * 2)
if data:
recv_queue.put(data)
try:
send_stream = sd.RawInputStream(
samplerate=send_rate,
channels=1,
dtype="int16",
blocksize=list_play_chunk_size,
callback=callback_send,
)
recv_stream = sd.RawOutputStream(
samplerate=recv_rate,
channels=1,
dtype="int16",
blocksize=list_play_chunk_size,
callback=callback_recv,
)
threading.Thread(target=send_stream.start).start()
threading.Thread(target=recv_stream.start).start()
send_thread = threading.Thread(target=send, args=(stop_event, send_queue))
send_thread.start()
recv_thread = threading.Thread(target=recv, args=(stop_event, recv_queue))
recv_thread.start()
input("Press Enter to stop...")
except KeyboardInterrupt:
print("Finished streaming.")
finally:
stop_event.set()
# Given that socket::recv is blocking in receive_data_chunk, shut it down to allow the thread to continue.
recv_socket.shutdown(socket.SHUT_RDWR)
recv_thread.join()
send_thread.join()
send_socket.close()
recv_socket.close()
print("Connection closed.")
if __name__ == "__main__":
parser = HfArgumentParser((ListenAndPlayArguments,)) # type: ignore[arg-type]
(listen_and_play_kwargs,) = parser.parse_args_into_dataclasses()
listen_and_play(**vars(listen_and_play_kwargs))

385
scripts/listen_and_play_realtime.py Normal file
View File

@ -0,0 +1,385 @@
import argparse
import asyncio
import base64
import time
from dataclasses import dataclass, field
from queue import Empty, Queue
from threading import Event, Lock
from typing import Any, Optional
from openai import AsyncOpenAI
@dataclass
class ListenAndPlayRealtimeArguments:
host: str = field(
default="127.0.0.1",
metadata={"help": "Realtime server host. Default is 127.0.0.1."},
)
port: int = field(
default=8765,
metadata={"help": "Realtime server port. Default is 8765."},
)
model: str = field(
default="local",
metadata={"help": "Model name sent to the OpenAI-compatible realtime client."},
)
api_key: str = field(
default="test-key",
metadata={"help": "API key for the OpenAI SDK client. Local server ignores it."},
)
base_url: Optional[str] = field(
default=None,
metadata={"help": "Optional HTTP base URL, e.g. http://127.0.0.1:8765/v1"},
)
websocket_base_url: Optional[str] = field(
default=None,
metadata={"help": "Optional WS base URL, e.g. ws://127.0.0.1:8765/v1"},
)
send_rate: int = field(
default=16000,
metadata={"help": "Microphone sample rate in Hz. Default is 16000."},
)
recv_rate: int = field(
default=16000,
metadata={"help": "Speaker sample rate in Hz. Default is 16000."},
)
chunk_size: int = field(
default=1024,
metadata={"help": "Audio callback block size in samples. Default is 1024."},
)
input_device: Optional[int] = field(
default=None,
metadata={"help": "Optional sounddevice input device index."},
)
output_device: Optional[int] = field(
default=None,
metadata={"help": "Optional sounddevice output device index."},
)
instructions: Optional[str] = field(
default=None,
metadata={"help": "Optional session instructions to apply on connect."},
)
voice: Optional[str] = field(
default=None,
metadata={
"help": (
"TTS voice sent as session.audio.output.voice. "
"Local Kokoro: e.g. bm_fable, af_heart, am_adam. "
"OpenAI Realtime: e.g. marin, cedar, alloy."
),
},
)
print_json: bool = field(
default=False,
metadata={"help": "Print raw event payloads in addition to friendly logs."},
)
block_mic_during_playback: bool = field(
default=False,
metadata={
"help": "If set, pause microphone capture while speaker audio is playing. Disabled by default so barge-in works."
},
)
def _make_client(args: ListenAndPlayRealtimeArguments) -> AsyncOpenAI:
base_url = args.base_url or f"http://{args.host}:{args.port}/v1"
websocket_base_url = args.websocket_base_url or f"ws://{args.host}:{args.port}/v1"
return AsyncOpenAI(
api_key=args.api_key,
base_url=base_url,
websocket_base_url=websocket_base_url,
)
def _build_session_update(args: ListenAndPlayRealtimeArguments) -> dict:
def maybe_pcm_format(rate: int) -> Optional[dict]:
# The OpenAI realtime Pydantic models only validate audio/pcm at 24 kHz.
# Our local pipeline defaults to 16 kHz internally when format is omitted,
# so omit the field for the common local case instead of sending an
# invalid 16 kHz declaration.
if rate == 16000:
return None
if rate == 24000:
return {"type": "audio/pcm", "rate": 24000}
raise ValueError(
f"Unsupported rate {rate}. Use 16000 for the local pipeline default "
f"or 24000 to match the OpenAI realtime audio format schema."
)
input_cfg = {
"turn_detection": {"type": "server_vad", "interrupt_response": True},
}
output_cfg: dict[str, Any] = {}
input_format = maybe_pcm_format(args.send_rate)
output_format = maybe_pcm_format(args.recv_rate)
if input_format is not None:
input_cfg["format"] = input_format
if output_format is not None:
output_cfg["format"] = output_format
if args.voice:
output_cfg["voice"] = args.voice
session = {
"type": "realtime",
"audio": {
"input": input_cfg,
"output": output_cfg,
},
}
if args.instructions:
session["instructions"] = args.instructions
return {"type": "session.update", "session": session}
async def listen_and_play_realtime(args: ListenAndPlayRealtimeArguments) -> None:
import sounddevice as sd
client = _make_client(args)
mic_queue: Queue[bytes] = Queue(maxsize=128)
stop_event = Event()
playback_buffer = bytearray()
playback_lock = Lock()
speaker_active_until = [0.0]
partial_user_text = ""
live_user_width = 0
saw_user_speech = False
def render_live_user_text(text: str, final: bool = False) -> None:
nonlocal live_user_width
line = f"USER: {text}"
padded = line
if live_user_width > len(line):
padded += " " * (live_user_width - len(line))
if final:
print(f"\r{padded}", flush=True)
live_user_width = 0
return
print(f"\r{padded}", end="", flush=True)
live_user_width = len(line)
def clear_live_user_text() -> None:
nonlocal live_user_width
if live_user_width == 0:
return
print("\r" + (" " * live_user_width) + "\r", end="", flush=True)
live_user_width = 0
def clear_playback_buffer() -> None:
speaker_active_until[0] = 0.0
with playback_lock:
playback_buffer.clear()
def callback_recv(outdata, _frames, _time_info, status):
if status:
print(f"Speaker status: {status}", flush=True)
needed = len(outdata)
with playback_lock:
available = min(needed, len(playback_buffer))
if available:
outdata[:available] = playback_buffer[:available]
del playback_buffer[:available]
if available < needed:
outdata[available:] = b"\x00" * (needed - available)
def callback_send(indata, _frames, _time_info, status):
if status:
print(f"Mic status: {status}", flush=True)
if args.block_mic_during_playback:
with playback_lock:
speaker_active = bool(playback_buffer)
if speaker_active or time.monotonic() < speaker_active_until[0]:
return
try:
mic_queue.put_nowait(bytes(indata))
except Exception:
pass
async def send_audio(conn):
while not stop_event.is_set():
try:
chunk = await asyncio.to_thread(mic_queue.get, True, 0.1)
except Empty:
continue
await conn.send(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(chunk).decode("ascii"),
}
)
async def receive_events(conn):
nonlocal partial_user_text, saw_user_speech
while not stop_event.is_set():
event = await conn.recv()
if args.print_json:
try:
print(f"EVENT: {event.model_dump_json()}", flush=True)
except Exception:
print(f"EVENT: {event}", flush=True)
if event.type == "session.created":
print("Connected.", flush=True)
elif event.type == "input_audio_buffer.speech_started":
clear_playback_buffer()
partial_user_text = ""
if saw_user_speech:
print("", flush=True)
saw_user_speech = True
elif event.type == "input_audio_buffer.speech_stopped":
pass
elif event.type == "conversation.item.input_audio_transcription.delta":
# This server currently sends the latest partial hypothesis in
# each "delta" event rather than a token-level suffix, so render
# the newest snapshot instead of concatenating repeated text.
partial_user_text = event.delta.strip()
if partial_user_text:
render_live_user_text(partial_user_text)
elif event.type == "conversation.item.input_audio_transcription.completed":
partial_user_text = ""
render_live_user_text(event.transcript.strip(), final=True)
elif event.type == "response.created":
clear_live_user_text()
print("ASSISTANT: <response started>", flush=True)
elif event.type == "response.output_audio.delta":
audio = base64.b64decode(event.delta)
with playback_lock:
playback_buffer.extend(audio)
speaker_active_until[0] = time.monotonic() + max(0.15, len(audio) / (2 * args.recv_rate))
elif event.type == "response.output_audio.done":
print("ASSISTANT: <audio done>", flush=True)
elif event.type == "response.output_audio_transcript.done":
print(f"ASSISTANT: {event.transcript}", flush=True)
elif event.type == "response.function_call_arguments.done":
print(
f"TOOL: {event.name} call_id={event.call_id} arguments={event.arguments}",
flush=True,
)
elif event.type == "response.done":
if event.response.status == "cancelled":
clear_playback_buffer()
print(f"ASSISTANT: <response {event.response.status}>", flush=True)
elif event.type == "error":
clear_live_user_text()
print(f"ERROR: {event.error.type}: {event.error.message}", flush=True)
else:
clear_live_user_text()
print(f"EVENT: {event.type}", flush=True)
async def wait_for_stop():
await asyncio.to_thread(input, "Press Enter to stop...\n")
stop_event.set()
input_stream = sd.RawInputStream(
samplerate=args.send_rate,
channels=1,
dtype="int16",
blocksize=args.chunk_size,
callback=callback_send,
device=args.input_device,
)
output_stream = sd.RawOutputStream(
samplerate=args.recv_rate,
channels=1,
dtype="int16",
blocksize=args.chunk_size,
callback=callback_recv,
device=args.output_device,
)
input_stream.start()
output_stream.start()
try:
async with client.realtime.connect(model=args.model) as conn:
await conn.send(_build_session_update(args)) # type: ignore[arg-type]
sender_task = asyncio.create_task(send_audio(conn))
receiver_task = asyncio.create_task(receive_events(conn))
stopper_task = asyncio.create_task(wait_for_stop())
done, pending = await asyncio.wait(
{sender_task, receiver_task, stopper_task},
return_when=asyncio.FIRST_COMPLETED,
)
stop_event.set()
for task in pending:
task.cancel()
await asyncio.gather(*pending, return_exceptions=True)
for task in done:
exc = task.exception()
if exc is not None:
raise exc
finally:
stop_event.set()
clear_live_user_text()
input_stream.stop()
output_stream.stop()
input_stream.close()
output_stream.close()
def main() -> None:
parser = argparse.ArgumentParser(description="Talk to the local OpenAI-compatible realtime speech pipeline.")
defaults = ListenAndPlayRealtimeArguments()
parser.add_argument("--host", default=defaults.host)
parser.add_argument("--port", type=int, default=defaults.port)
parser.add_argument("--model", default=defaults.model)
parser.add_argument("--api-key", default=defaults.api_key)
parser.add_argument("--base-url", default=defaults.base_url)
parser.add_argument("--websocket-base-url", default=defaults.websocket_base_url)
parser.add_argument("--send-rate", type=int, default=defaults.send_rate)
parser.add_argument("--recv-rate", type=int, default=defaults.recv_rate)
parser.add_argument("--chunk-size", type=int, default=defaults.chunk_size)
parser.add_argument("--input-device", type=int, default=defaults.input_device)
parser.add_argument("--output-device", type=int, default=defaults.output_device)
parser.add_argument("--instructions", default=defaults.instructions)
parser.add_argument(
"--voice",
default=defaults.voice,
help=("session.audio.output.voice (Kokoro id like bm_fable, or OpenAI name like marin)."),
)
parser.add_argument("--print-json", action="store_true", default=defaults.print_json)
parser.add_argument(
"--block-mic-during-playback",
action="store_true",
default=defaults.block_mic_during_playback,
)
namespace = parser.parse_args()
args = ListenAndPlayRealtimeArguments(
host=namespace.host,
port=namespace.port,
model=namespace.model,
api_key=namespace.api_key,
base_url=namespace.base_url,
websocket_base_url=namespace.websocket_base_url,
send_rate=namespace.send_rate,
recv_rate=namespace.recv_rate,
chunk_size=namespace.chunk_size,
input_device=namespace.input_device,
output_device=namespace.output_device,
instructions=namespace.instructions,
voice=namespace.voice,
print_json=namespace.print_json,
block_mic_during_playback=namespace.block_mic_during_playback,
)
try:
asyncio.run(listen_and_play_realtime(args))
except KeyboardInterrupt:
pass
if __name__ == "__main__":
main()

569
scripts/synthetic_conversation_realtime_client.py Normal file
View File

@ -0,0 +1,569 @@
"""Synthetic realtime client(s): single or parallel, single-turn or multi-turn.
Opens one (or N) websocket connection(s) to /v1/realtime and cycles through
``--turns`` prompts per client at a fixed cadence (``--interval`` seconds
between turn starts). Each turn:
1. Synthesize the prompt with macOS ``say`` (cached on disk after first run).
2. Stream the prompt audio + trailing silence as input_audio_buffer.append.
3. Wait for response.done.
4. Sleep until --interval has elapsed since the turn started.
This single script subsumes two earlier ones:
* Parallel pool / capacity test:
python scripts/synthetic_conversation_realtime_client.py --clients 3 --turns 1
→ 3 clients connect simultaneously, each sends one prompt. Surplus clients
beyond pool size receive ``session_limit_reached`` and exit cleanly.
* Single-client soak / multi-turn:
python scripts/synthetic_conversation_realtime_client.py --turns 60 --interval 10
→ 1 client, ~10 minute conversation, 60 sequential turns.
* Both at once (soak the pool):
python scripts/synthetic_conversation_realtime_client.py --clients 2 --turns 60
* Soak a Hugging Face Inference Endpoint via its load balancer (10 min, 2 clients):
export HF_TOKEN=hf_...
python scripts/synthetic_conversation_realtime_client.py \
--lb-url https://<your-lb>.us-east-1.aws.endpoints.huggingface.cloud \
--clients 2 \
--turns 60 \
--interval 10 \
--log-dir /tmp/hf_endpoint_soak
Each client uses a per-client prompt offset (coprime shift) so concurrent
clients send distinct prompts at each turn — making cross-session leaks
trivially detectable in the per-client transcript logs.
Outputs land under --log-dir:
* prompts/prompt_NNN.wav — shared cache of say-synthesized prompts
* client_NNN/conversation.txt — per-client transcript with timestamps
* client_NNN/conversation.wav — per-client assistant audio concatenated
"""
import argparse
import asyncio
import base64
import json
import logging
import os
import shutil
import subprocess
import sys
import time
import wave
from pathlib import Path
import httpx
import numpy as np
import soundfile as sf
import websockets
from scipy.signal import resample_poly
logger = logging.getLogger("synthetic_client")
SAMPLE_RATE_HZ = 16000
CHUNK_MS = 20
BYTES_PER_SAMPLE = 2 # PCM16
CHUNK_BYTES = SAMPLE_RATE_HZ * BYTES_PER_SAMPLE * CHUNK_MS // 1000 # 640
# Trailing silence after each prompt so server-side VAD detects speech_stopped
# and auto-commits. The local realtime service only supports server VAD.
TRAILING_SILENCE_MS = 1500
# Per-client prompt offset shift. 7 is coprime with len(PROMPTS)=60, so each
# client visits a unique permutation of the prompt list.
PROMPT_SHIFT_PER_CLIENT = 7
# 60 varied prompts — short enough to fit a 10-second turn budget, diverse
# enough that responses don't pattern-match. Cycled if --turns > len(PROMPTS).
PROMPTS: list[str] = [
"What is the capital of France?",
"Tell me a joke about robots.",
"How does photosynthesis work?",
"What is two plus two?",
"Who painted the Mona Lisa?",
"What is the largest ocean on Earth?",
"Recommend a simple pasta recipe.",
"What is the speed of light?",
"Who wrote Romeo and Juliet?",
"What is the boiling point of water in Celsius?",
"Tell me one fact about the planet Mars.",
"What is the difference between weather and climate?",
"How many continents are there?",
"What is a haiku?",
"What is the chemical formula for water?",
"Who was the first president of the United States?",
"What is gravity?",
"Tell me a fun fact about dolphins.",
"What is the tallest mountain on Earth?",
"How do magnets work?",
"What is the meaning of life in one sentence?",
"Recommend a short book to read.",
"What is the population of Tokyo, roughly?",
"How does a refrigerator stay cold?",
"Who invented the telephone?",
"What is the smallest country in the world?",
"Explain the theory of relativity simply.",
"What is the difference between a virus and bacteria?",
"Who wrote War and Peace?",
"What is the deepest part of the ocean called?",
"How does the human heart work?",
"What is the most spoken language in the world?",
"Tell me one fact about black holes.",
"What is the longest river on Earth?",
"How do you make a paper airplane?",
"What is the chemical symbol for gold?",
"Who composed the Fifth Symphony?",
"What is the difference between an alligator and a crocodile?",
"How many bones are in the human body?",
"What is the smallest planet in our solar system?",
"Suggest a beginner workout routine.",
"What is the busiest airport in the world?",
"How does Wi-Fi work in one sentence?",
"What is the oldest known written language?",
"Tell me about the Great Wall of China briefly.",
"What is a neutron star?",
"How does sound travel through space?",
"What is the most popular sport globally?",
"Who painted Starry Night?",
"What happens when you mix baking soda and vinegar?",
"How fast can a cheetah run?",
"What is the difference between fiction and non-fiction?",
"Why is the sky blue?",
"What is the largest desert in the world?",
"How many planets are in our solar system?",
"Who discovered penicillin?",
"What is a leap year?",
"Tell me how rain forms.",
"What is the longest-living animal?",
"Say goodbye for now.",
]
# ---------------------------------------------------------------------------
# Audio helpers
# ---------------------------------------------------------------------------
def synthesize_with_say(text: str, out_path: Path) -> None:
"""Render *text* to a 16kHz mono PCM16 WAV using macOS ``say``."""
subprocess.run(
[
"say",
text,
"-o",
str(out_path),
"--file-format=WAVE",
"--data-format=LEI16@16000",
],
check=True,
capture_output=True,
)
def load_pcm16_mono_16k(path: Path) -> bytes:
"""Load any WAV / AIFF and return 16kHz mono PCM16 little-endian bytes."""
data, src_rate = sf.read(str(path), always_2d=True)
mono = data.mean(axis=1) if data.shape[1] > 1 else data[:, 0]
if src_rate != SAMPLE_RATE_HZ:
mono = resample_poly(mono, SAMPLE_RATE_HZ, src_rate)
mono = np.clip(mono, -1.0, 1.0)
pcm16 = (mono * 32767.0).astype(np.int16)
return pcm16.tobytes()
def write_wav(path: Path, pcm16_bytes: bytes, rate: int = SAMPLE_RATE_HZ) -> None:
with wave.open(str(path), "wb") as w:
w.setnchannels(1)
w.setsampwidth(BYTES_PER_SAMPLE)
w.setframerate(rate)
w.writeframes(pcm16_bytes)
# ---------------------------------------------------------------------------
# Per-turn ws flow
# ---------------------------------------------------------------------------
async def stream_prompt(ws, audio_pcm: bytes) -> None:
"""Send prompt chunks at real-time pacing, then trailing silence."""
for i in range(0, len(audio_pcm), CHUNK_BYTES):
chunk = audio_pcm[i : i + CHUNK_BYTES]
if len(chunk) < CHUNK_BYTES:
chunk = chunk + b"\x00" * (CHUNK_BYTES - len(chunk))
await ws.send(
json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(chunk).decode("ascii")})
)
await asyncio.sleep(CHUNK_MS / 1000.0)
silence_chunk = b"\x00" * CHUNK_BYTES
silence_payload = base64.b64encode(silence_chunk).decode("ascii")
for _ in range(TRAILING_SILENCE_MS // CHUNK_MS):
await ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": silence_payload}))
await asyncio.sleep(CHUNK_MS / 1000.0)
async def consume_until_response_done(
ws,
response_audio_out: bytearray,
response_timeout_s: float,
) -> dict:
"""Read server events until response.done (or timeout). Returns turn summary."""
info: dict = {
"transcript_in": "",
"transcript_out": "",
"error": None,
}
deadline = time.monotonic() + response_timeout_s
while time.monotonic() < deadline:
try:
raw = await asyncio.wait_for(ws.recv(), timeout=max(0.1, deadline - time.monotonic()))
except asyncio.TimeoutError:
info["error"] = "response_timeout"
return info
event = json.loads(raw)
t = event.get("type", "")
if t == "conversation.item.input_audio_transcription.completed":
info["transcript_in"] += event.get("transcript", "")
elif t in ("response.audio.delta", "response.output_audio.delta"):
delta = event.get("delta", "")
if delta:
response_audio_out.extend(base64.b64decode(delta))
elif t in ("response.audio_transcript.delta", "response.output_audio_transcript.delta"):
info["transcript_out"] += event.get("delta", "")
elif t in ("response.audio_transcript.done", "response.output_audio_transcript.done"):
transcript = event.get("transcript")
if transcript:
info["transcript_out"] = transcript
elif t == "response.done":
return info
elif t == "error":
info["error"] = event.get("error", {}).get("message", "error event")
return info
info["error"] = "response_timeout"
return info
def _truncate_transcript(s: str, max_len: int = 20) -> str:
"""Quote *s*; if longer than *max_len*, slice + ellipsis + total char count."""
if len(s) <= max_len:
return repr(s)
return f"{(s[:max_len] + '...')!r} ({len(s)} chars)"
# ---------------------------------------------------------------------------
# Client driver (one per client)
# ---------------------------------------------------------------------------
async def _lb_allocate_session(
lb_url: str,
auth_headers: dict[str, str],
http: httpx.AsyncClient,
) -> dict:
"""POST {lb_url}/session → returns dict with connect_url, session_id, session_token."""
resp = await http.post(f"{lb_url}/session", headers=auth_headers, timeout=30.0)
resp.raise_for_status()
return resp.json()
async def _lb_send_event(
lb_url: str,
session_id: str,
session_token: str,
event: str,
http: httpx.AsyncClient,
) -> None:
"""POST {lb_url}/internal/sessions/{id}/event with {session_token, event}."""
url = f"{lb_url}/internal/sessions/{session_id}/event"
resp = await http.post(url, json={"session_token": session_token, "event": event}, timeout=30.0)
resp.raise_for_status()
async def run_client(
client_id: int,
args: argparse.Namespace,
ws_url: str,
extra_headers: list[tuple[str, str]],
audio_files: list[tuple[str, Path]],
) -> dict:
"""Run one client's multi-turn conversation. Returns per-client summary.
If --lb-url is set, hits the load balancer first to allocate a slot on a
compute node, then connects to the returned connect_url. Otherwise connects
directly to ws_url. The LB callback events ("connected"/"disconnected") are
sent around the websocket session so the LB doesn't reclaim the slot early.
"""
summary: dict = {
"client_id": client_id,
"connected": False,
"rejected": False,
"completed": 0,
"errors": 0,
"error_msg": None,
}
log_dir = Path(args.log_dir) / f"client_{client_id:03d}"
log_dir.mkdir(parents=True, exist_ok=True)
transcript_log = log_dir / "conversation.txt"
response_audio = bytearray()
prefix = f"[c{client_id}]"
auth_headers = {k: v for k, v in extra_headers}
lb_session_id: str | None = None
lb_session_token: str | None = None
async with httpx.AsyncClient() as http:
# Step 1: allocate a slot via the LB (if configured).
if args.lb_url:
try:
alloc = await _lb_allocate_session(args.lb_url, auth_headers, http)
except Exception as e: # noqa: BLE001 — diagnostic path
summary["error_msg"] = f"LB /session failed: {type(e).__name__}: {e}"
logger.info(f"{prefix} {summary['error_msg']}")
return summary
connect_url = alloc["connect_url"]
lb_session_id = alloc["session_id"]
lb_session_token = alloc["session_token"]
logger.info(
f"{prefix} LB allocated session_id={lb_session_id} "
f"ws_url={alloc.get('websocket_url', '?')}"
)
else:
connect_url = ws_url
try:
async with websockets.connect(
connect_url,
max_size=2**24,
additional_headers=extra_headers or None,
) as ws:
summary["connected"] = True
# Step 3: tell the LB the client actually connected (clears the
# pending_timeout_s reaper). Best-effort — failure here logs but
# doesn't abort the session.
if lb_session_id and lb_session_token:
try:
await _lb_send_event(args.lb_url, lb_session_id, lb_session_token, "connected", http)
except Exception as e: # noqa: BLE001
logger.warning(f"{prefix} LB 'connected' callback failed: {e}")
first = json.loads(await asyncio.wait_for(ws.recv(), timeout=10.0))
if first.get("type") == "error" and "session_limit_reached" in str(first):
summary["rejected"] = True
summary["error_msg"] = first.get("error", {}).get("message", "limit reached")
logger.warning(f"{prefix} REJECTED: {summary['error_msg']}")
elif first.get("type") != "session.created":
summary["error_msg"] = f"unexpected first event: {first.get('type')}"
logger.error(f"{prefix} ERROR: {summary['error_msg']}")
else:
sess = first.get("session") or {}
session_id = sess.get("id") or first.get("event_id") or "?"
logger.info(f"{prefix} connected, session={session_id}")
with transcript_log.open("w") as log_f:
log_f.write(f"# Synthetic conversation, client={client_id}, session={session_id}\n")
log_f.write(f"# Target turns={args.turns}, interval={args.interval}s\n\n")
for turn_idx in range(args.turns):
# Per-client prompt offset (coprime shift) so concurrent
# clients send distinct prompts at every turn.
prompt_idx = (turn_idx + client_id * PROMPT_SHIFT_PER_CLIENT) % len(audio_files)
text, wav_path = audio_files[prompt_idx]
turn_start = time.monotonic()
turn_audio = load_pcm16_mono_16k(wav_path)
logger.info(f"{prefix} turn {turn_idx + 1}/{args.turns} USER: {text!r}")
log_f.write(f"[turn {turn_idx + 1}/{args.turns}] USER: {text}\n")
try:
await stream_prompt(ws, turn_audio)
except websockets.exceptions.ConnectionClosed as e:
logger.info(
f"{prefix} turn {turn_idx + 1}/{args.turns} "
f"connection closed during send: {e}"
)
log_f.write(f"[turn {turn_idx + 1}/{args.turns}] CONNECTION_CLOSED: {e}\n")
summary["error_msg"] = f"connection_closed: {e}"
break
info = await consume_until_response_done(
ws, response_audio, args.response_timeout
)
turn_elapsed = time.monotonic() - turn_start
if info["error"]:
logger.info(
f"{prefix} turn {turn_idx + 1}/{args.turns} ERROR: {info['error']}"
)
log_f.write(f"[turn {turn_idx + 1}/{args.turns}] ERROR: {info['error']}\n\n")
summary["errors"] += 1
else:
summary["completed"] += 1
logger.info(
f"{prefix} turn {turn_idx + 1}/{args.turns} "
f"ASSISTANT: {_truncate_transcript(info['transcript_out'])}"
)
log_f.write(f"[turn {turn_idx + 1}/{args.turns}] STT: {info['transcript_in']}\n")
log_f.write(
f"[turn {turn_idx + 1}/{args.turns}] ASSISTANT: {info['transcript_out']}\n\n"
)
log_f.flush()
remaining = args.interval - turn_elapsed
if remaining > 0:
await asyncio.sleep(remaining)
except Exception as e: # noqa: BLE001 — diagnostic path
summary["error_msg"] = f"{type(e).__name__}: {e}"
logger.error(f"{prefix} EXCEPTION: {summary['error_msg']}")
finally:
# Step 5: release the LB slot, regardless of how the ws session ended.
if lb_session_id and lb_session_token:
try:
await _lb_send_event(
args.lb_url, lb_session_id, lb_session_token, "disconnected", http
)
logger.info(f"{prefix} LB notified: disconnected")
except Exception as e: # noqa: BLE001
logger.warning(f"{prefix} LB 'disconnected' callback failed: {e}")
if response_audio:
wav_out = log_dir / "conversation.wav"
write_wav(wav_out, bytes(response_audio))
logger.info(f"{prefix} wrote {len(response_audio)} bytes -> {wav_out}")
return summary
# ---------------------------------------------------------------------------
# Top-level: synthesize prompts, spawn clients, summarize
# ---------------------------------------------------------------------------
async def run_all(args: argparse.Namespace) -> None:
log_dir = Path(args.log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
prompts_dir = log_dir / "prompts"
prompts_dir.mkdir(parents=True, exist_ok=True)
logger.info(f"Synthesizing {len(PROMPTS)} prompts with `say` (cached in {prompts_dir})...")
audio_files: list[tuple[str, Path]] = []
for i, text in enumerate(PROMPTS):
wav_path = prompts_dir / f"prompt_{i:03d}.wav"
if not wav_path.exists():
synthesize_with_say(text, wav_path)
audio_files.append((text, wav_path))
ws_url = args.url or f"ws://{args.host}:{args.port}/v1/realtime"
token = os.environ.get("HF_TOKEN")
if not token:
raise SystemExit(
"HF_TOKEN env var is not set. Export it before running this script "
"(e.g. `export HF_TOKEN=hf_...`)."
)
extra_headers: list[tuple[str, str]] = [("Authorization", f"Bearer {token}")]
logger.info("Auth: Bearer token attached from HF_TOKEN env")
if args.lb_url:
target = f"LB {args.lb_url}"
else:
target = ws_url
logger.info(
f"Spawning {args.clients} client(s) against {target}, "
f"{args.turns} turns each @ {args.interval:.1f}s interval"
)
summaries = await asyncio.gather(
*(
run_client(
client_id=i,
args=args,
ws_url=ws_url,
extra_headers=extra_headers,
audio_files=audio_files,
)
for i in range(args.clients)
)
)
logger.info("\n=== summary ===")
for s in summaries:
status = "rejected" if s["rejected"] else ("error" if s["error_msg"] else "ok")
logger.info(
f" c{s['client_id']}: {status:8s} completed={s['completed']}/{args.turns} "
f"errors={s['errors']} err={s['error_msg']}"
)
n_ok = sum(1 for s in summaries if s["connected"] and not s["rejected"] and not s["error_msg"])
n_rej = sum(1 for s in summaries if s["rejected"])
n_err = sum(1 for s in summaries if s["error_msg"] and not s["rejected"])
total_turns = sum(s["completed"] for s in summaries)
logger.info(f"=> {n_ok} successful clients, {n_rej} rejected, {n_err} errored")
logger.info(f"=> {total_turns} total turns completed across pool")
def main() -> None:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
stream=sys.stdout,
)
if not shutil.which("say"):
raise SystemExit("This script requires macOS `say` (not found in PATH).")
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--lb-url",
default=None,
help=(
"Base URL of the load balancer (e.g. https://lb.example.com). When set, the script "
"POSTs /session to allocate a slot on a compute node, connects to the returned "
"connect_url, and sends connected/disconnected events around the session. Takes "
"precedence over --url / --host / --port."
),
)
parser.add_argument(
"--url",
default=None,
help=(
"Full ws:// or wss:// URL to a compute-node realtime endpoint, e.g. "
"wss://endpoint.example.com/v1/realtime. Used when --lb-url is not set. "
"Overrides --host/--port."
),
)
parser.add_argument("--host", default="127.0.0.1", help="Used only when --url and --lb-url are unset.")
parser.add_argument("--port", type=int, default=8765, help="Used only when --url and --lb-url are unset.")
parser.add_argument("--clients", type=int, default=1, help="Number of parallel clients (default 1).")
parser.add_argument("--turns", type=int, default=60, help="Turns per client (default 60).")
parser.add_argument(
"--interval",
type=float,
default=10.0,
help="Seconds between turn starts per client. Total runtime ≈ turns × interval. Default 10.",
)
parser.add_argument(
"--response-timeout",
type=float,
default=30.0,
help="Per-turn wait for response.done before marking the turn errored.",
)
parser.add_argument(
"--log-dir",
default="/tmp/synthetic_conversation",
help="Top-level directory for prompt cache and per-client logs.",
)
args = parser.parse_args()
if args.clients < 1:
parser.error("--clients must be >= 1")
if args.turns < 1:
parser.error("--turns must be >= 1")
asyncio.run(run_all(args))
if __name__ == "__main__":
main()

BIN
speech.mp3 Normal file
View File

Binary file not shown.

139
src/speech_to_speech/LLM/README.md Normal file
View File

@ -0,0 +1,139 @@
# LLM Summary
## Available LLM backends (`--llm_backend`)
Runtime-supported values in `s2s_pipeline.py`:
- `transformers` → `language_model.py` (Transformers backend)
- `mlx-lm` → `language_model.py` (MLX backend)
- `responses-api` → `responses_api_language_model.py`
## Usage
### 1) Transformers (`--llm_backend transformers`)
- Handler: `LanguageModelHandler`
- Typical use: local GPU/CPU inference using Hugging Face Transformers
- Backend-specific args prefix: `--llm_*`
- Shared args (from base): `--model_name`, `--chat_size`, `--init_chat_prompt`, `--enable_lang_prompt`
```bash
python s2s_pipeline.py \
--llm_backend transformers \
--model_name Qwen/Qwen3-4B-Instruct-2507 \
--llm_device cuda \
--llm_torch_dtype float16 \
--llm_gen_max_new_tokens 128
```
Common options:
- `--llm_gen_min_new_tokens`
- `--llm_gen_temperature`
- `--llm_gen_do_sample`
- `--chat_size`
- `--init_chat_prompt`
### 2) MLX-LM (`--llm_backend mlx-lm`)
- Handler: `LanguageModelHandler`
- Typical use: Apple Silicon local inference
- Backend-specific args prefix: same as Transformers (`--llm_*`)
```bash
python s2s_pipeline.py \
--llm_backend mlx-lm \
--model_name mlx-community/Qwen3-4B-Instruct-2507-bf16 \
--llm_device mps \
--llm_gen_max_new_tokens 128
```
Common options:
- `--llm_gen_temperature`
- `--llm_gen_do_sample`
- `--chat_size`
- `--init_chat_prompt`
### 3) OpenAI-compatible API (`--llm_backend responses-api`)
- Handler: `ResponsesApiModelHandler`
- Typical use: remote model serving via OpenAI-compatible endpoints
- Backend-specific args prefix: `--responses_api_*`
- Shared args (from base): `--model_name`, `--chat_size`, `--init_chat_prompt`, `--enable_lang_prompt`
```bash
python s2s_pipeline.py \
--llm_backend responses-api \
--model_name gpt-5.4-mini \
--responses_api_api_key YOUR_API_KEY \
--responses_api_base_url https://api.example.com/v1 \
--responses_api_stream true
```
Common options:
- `--chat_size`
- `--init_chat_prompt`
- `--user_role`
## LLM Behavior
When STT is set to language auto-detection (`--language auto`), LLM handlers can receive `(text, language_code)` and prepend a language control instruction like:
- `Please reply to my message in <language>.`
This helps the assistant respond in the detected language. The behavior is opt-in via `--enable_lang_prompt` (shared across all backends); it defaults to `False`.
## Setup
### CUDA setup
```bash
python s2s_pipeline.py \
--llm_backend transformers \
--model_name microsoft/Phi-3-mini-4k-instruct
```
### Local Mac setup
```bash
python s2s_pipeline.py \
--local_mac_optimal_settings \
--model_name mlx-community/Qwen3-4B-Instruct-2507-bf16
```
`--local_mac_optimal_settings` already sets `--llm_backend mlx-lm` and will default the model to `mlx-community/Qwen3-4B-Instruct-2507-bf16` if not overridden.
### Realtime (OpenAI-compatible) setup
Run the server in realtime mode, then connect with the realtime client:
```bash
# 1. Start the pipeline in realtime mode
python s2s_pipeline.py \
--mode realtime \
--llm_backend mlx-lm \
--model_name mlx-community/Qwen3-4B-Instruct-2507-bf16 \
--ws_host 0.0.0.0 \
--ws_port 8765
# 2. Connect with the realtime client
python listen_and_play_realtime.py --host 127.0.0.1 --port 8765
```
Or with `--local_mac_optimal_settings` on Apple Silicon:
```bash
python s2s_pipeline.py \
--local_mac_optimal_settings \
--mode realtime \
--ws_host 0.0.0.0 \
--ws_port 8765
```
### Remote API setup
```bash
python s2s_pipeline.py \
--llm_backend responses-api \
--model_name gpt-5.4-mini \
--responses_api_api_key YOUR_API_KEY
```

0
src/speech_to_speech/LLM/__init__.py Normal file
View File

714
src/speech_to_speech/LLM/base_openai_compatible_language_model.py Normal file
View File

@ -0,0 +1,714 @@
from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from collections.abc import Iterator
from typing import Any, Optional
import httpx
from nltk import sent_tokenize
from openai import OpenAI
from openai.types.realtime.conversation_item import (
RealtimeConversationItemAssistantMessage,
RealtimeConversationItemFunctionCall,
)
from openai.types.realtime.realtime_conversation_item_assistant_message import (
Content as AssistantContent,
)
from openai.types.responses import ResponseFunctionToolCall
from pydantic import BaseModel, ConfigDict, Field
from speech_to_speech.baseHandler import BaseHandler
from speech_to_speech.LLM.chat import (
Chat,
ChatItemError,
SupportedItem,
build_active_chat,
make_system_message,
make_user_message,
)
from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn, build_compactor
from speech_to_speech.LLM.text_prompt import build_text_system_prompt
from speech_to_speech.LLM.utils import remove_unspeechable, resolve_auto_language
from speech_to_speech.LLM.voice_prompt import build_voice_system_prompt
from speech_to_speech.pipeline.cancel_scope import CancelScope
from speech_to_speech.pipeline.handler_types import LLMIn, LLMOut
from speech_to_speech.pipeline.messages import (
EndOfResponse,
LLMResponseChunk,
TokenUsage,
)
from speech_to_speech.RAG import RAGRetriever, SearchResult, get_global_rag
from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
from speech_to_speech.utils.utils import is_out_of_band, response_wants_audio
logger = logging.getLogger(__name__)
# About 18–24 seconds of default SDK backoff before warmup fails.
WARMUP_MAX_RETRIES = 6
# ── Normalised provider events ────────────────────────────────────────────────
# Each backend's stream/response is mapped to this small vocabulary so the shared
# speech-pipeline logic (sentence batching, cancellation, history, token usage)
# lives in one place. Subclasses differ only in how they produce these events.
class TextDelta(BaseModel):
"""Incremental assistant text. Always RAW (unfiltered); the base applies
``remove_unspeechable`` for the audio path."""
text: str
class AssistantMessage(BaseModel):
"""A complete assistant turn to write back to history."""
content: list[AssistantContent]
class ToolCall(BaseModel):
"""A complete function tool call (``call_id`` / ``id`` already regenerated)."""
item: ResponseFunctionToolCall
class Usage(BaseModel):
"""Token accounting for the turn."""
input_tokens: int
output_tokens: int
ProviderEvent = TextDelta | AssistantMessage | ToolCall | Usage
class _Turn(BaseModel):
"""Per-request context threaded through generation (immutable for the turn)."""
model_config = ConfigDict(arbitrary_types_allowed=True)
language_code: Optional[str]
gen: int | None
runtime_config: Any
response: Any
turn_id: str | None
turn_revision: int | None
speech_stopped_at_s: float | None
wants_audio: bool
class _GenState(BaseModel):
"""Mutable accumulators collected while consuming a turn's events."""
model_config = ConfigDict(arbitrary_types_allowed=True)
tools: list[ResponseFunctionToolCall] = Field(default_factory=list)
pending: list[SupportedItem] = Field(default_factory=list)
clean_text: str = "" # filtered text, kept only for the debug log
input_tokens: int = 0
output_tokens: int = 0
class BaseOpenAICompatibleHandler(BaseHandler[LLMIn, LLMOut], ABC):
"""Shared lifecycle for OpenAI-compatible LLM backends (Responses & Chat
Completions).
Subclasses implement four hooks — :meth:`warmup`,
:meth:`_build_compaction_generate_fn`, :meth:`_serialize`, :meth:`_request`,
:meth:`_iter_events` and :meth:`_build_optional_kwargs` — and inherit the
request/response orchestration: speculative-turn gating, cancellation,
sentence batching, text-only vs audio handling, history write-back, token
usage, out-of-band handling and error termination.
"""
# ── setup ─────────────────────────────────────────────────────────────────
def setup(
self,
model_name: str = "gpt-5.4-mini",
device: str = "cuda",
gen_kwargs: dict[str, Any] = {},
base_url: Optional[str] = None,
api_key: Optional[str] = None,
stream: bool = True,
user_role: str = "user",
cancel_scope: CancelScope | None = None,
speculative_turns: SpeculativeTurnTracker | None = None,
disable_thinking: bool = True,
reasoning_effort: Optional[str] = None,
request_timeout_s: float = 20.0,
stream_batch_sentences: int = 3,
enable_lang_prompt: bool = False,
compact_history: bool = False,
**_kwargs: Any,
) -> None:
self.cancel_scope = cancel_scope
self.speculative_turns = speculative_turns
self.model_name = model_name
self.stream = stream
self.stream_batch_sentences = max(1, stream_batch_sentences)
self.enable_lang_prompt = enable_lang_prompt
self.gen_kwargs = dict(gen_kwargs)
self.request_timeout_s = float(request_timeout_s)
self.request_timeout = httpx.Timeout(
self.request_timeout_s,
connect=min(10.0, self.request_timeout_s),
)
self.user_role = user_role
self.client = OpenAI(api_key=api_key, base_url=base_url)
self._extra_body = self._build_extra_body(base_url, disable_thinking, reasoning_effort)
self.compactor = build_compactor(self._build_compaction_generate_fn()) if compact_history else None
self.rag = get_global_rag()
if self.rag is not None:
logger.info("RAG hook attivo in BaseOpenAICompatibleHandler (inject_as=%s, lingua=%s)", self.rag.inject_as, self.rag.language)
self.warmup()
@staticmethod
def _is_official_openai(base_url: Optional[str]) -> bool:
"""Whether ``base_url`` points at the official OpenAI server.
Normalises a trailing slash so ``https://api.openai.com/v1/`` is also
recognised; the official server rejects the provider-specific extra_body
keys we send to vLLM / the HF router.
"""
if base_url is None:
return False
return base_url.rstrip("/") == "https://api.openai.com/v1"
@classmethod
def _build_extra_body(
cls,
base_url: Optional[str],
disable_thinking: bool,
reasoning_effort: Optional[str],
) -> Optional[dict[str, Any]]:
"""Build the provider-specific ``extra_body`` used to disable reasoning.
Providers differ in how reasoning is turned off: vLLM/Qwen honour
``chat_template_kwargs.enable_thinking=false``, while others (e.g. GLM via
the HF router) ignore that and require ``reasoning_effort='none'``. A
non-empty ``reasoning_effort`` therefore takes precedence; otherwise we fall
back to the chat-template flag. None of this applies to the official
OpenAI server, which rejects unknown extra_body keys.
"""
if base_url is None or cls._is_official_openai(base_url):
return None
if reasoning_effort:
return {"reasoning_effort": reasoning_effort}
if disable_thinking:
return {"chat_template_kwargs": {"enable_thinking": False}}
return None
# ── subclass hooks ──────────────────────────────────────────────────────--
@abstractmethod
def warmup(self) -> None:
"""Issue a cheap request so the model/connection is ready before serving."""
...
@abstractmethod
def _build_compaction_generate_fn(self) -> CompactGenerateFn:
"""Return a ``(system, user) -> text`` fn used to compact long histories."""
...
@abstractmethod
def _serialize(self, active_chat: Chat) -> Any:
"""Serialise the chat to the backend's request payload (input/messages)."""
...
@abstractmethod
def _request(self, api_input: Any, optional_kwargs: dict[str, Any]) -> Any:
"""Issue the create() call and return the response or stream."""
...
@abstractmethod
def _iter_stream_events(self, api_response: Any) -> Iterator[ProviderEvent]:
"""Map a streaming response to normalised :data:`ProviderEvent`s."""
...
@abstractmethod
def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
"""Map a non-streaming response to normalised :data:`ProviderEvent`s."""
...
def _iter_events(self, api_response: Any) -> Iterator[ProviderEvent]:
"""Dispatch to the stream/non-stream mapper. ``self.stream`` is the single
source of truth (it set the request's ``stream=`` flag), so the response
type always matches it."""
if self.stream:
yield from self._iter_stream_events(api_response)
else:
yield from self._iter_response_events(api_response)
@abstractmethod
def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
"""Build the per-request tools/tool_choice kwargs in the backend's shape."""
...
# ── speculative-turn / cancellation gating ─────────────────────────────────
def _turn_is_latest(self, turn_id: str | None, turn_revision: int | None) -> bool:
return self.speculative_turns is None or self.speculative_turns.is_latest(turn_id, turn_revision)
def _generation_is_stale(self, gen: int | None) -> bool:
return gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen)
def _turn_output_allowed(self, turn_id: str | None, turn_revision: int | None) -> bool:
if self.speculative_turns is None:
return True
return self.speculative_turns.is_latest_after_reopen_grace(turn_id, turn_revision)
def _apply_config(
self,
chat: Chat,
instructions: Optional[str],
wants_audio: bool = True,
) -> None:
if instructions:
builder = build_voice_system_prompt if wants_audio else build_text_system_prompt
full_instructions = builder(instructions)
chat.add_item(make_system_message(full_instructions))
# ── RAG helpers ─────────────────────────────────────────────────────────────
@staticmethod
def _extract_last_user_text(chat: Chat) -> str:
from openai.types.realtime.conversation_item import RealtimeConversationItemUserMessage
last = ""
for entry in reversed(chat.buffer):
if isinstance(entry, RealtimeConversationItemUserMessage):
parts: list[str] = []
for c in entry.content or []:
if getattr(c, "type", None) == "input_text" and getattr(c, "text", None):
parts.append(c.text)
if parts:
last = "\n".join(parts)
break
return last.strip()
def _inject_rag_context(
self,
chat: Chat,
turn_id: Optional[str],
turn_revision: Optional[int],
) -> None:
if self.rag is None:
return
query = self._extract_last_user_text(chat)
if not query or len(query) < 3:
return
try:
results: list[SearchResult] = self.rag.search(
query,
top_k=self.rag.top_k,
threshold=self.rag.threshold,
)
except Exception as exc:
logger.warning(
"RAG search fallita per turn=%s rev=%s: %s",
turn_id,
turn_revision,
exc,
)
return
if not results:
logger.debug(
"RAG: nessun chunk rilevante per turn=%s (query=%r soglia=%.3f)",
turn_id,
query[:80],
self.rag.threshold,
)
return
formatted = RAGRetriever.format_results(results, language=self.rag.language)
sources_summary = "; ".join(
f"{r.chunk.source}#c{r.chunk.chunk_index}({r.score:.2f})" for r in results
)
logger.info(
"RAG: iniettati %d chunk per turn=%s rev=%s — %s",
len(results),
turn_id,
turn_revision,
sources_summary,
)
if self.rag.inject_as == "system":
current = chat.init_chat_message
existing_texts: list[str] = []
if current is not None and current.content:
for c in current.content:
if getattr(c, "type", None) == "text" and getattr(c, "text", None):
existing_texts.append(c.text)
base = "\n\n".join(existing_texts).rstrip()
merged = (base + "\n\n" + formatted).lstrip()
chat.add_item(make_system_message(merged))
else:
chat.add_item(make_user_message(formatted))
# ── output helpers ──────────────────────────────────────────────────────--
def _chunk(
self,
turn: _Turn,
*,
text: str = "",
tools: list[ResponseFunctionToolCall] | None = None,
language_code: Optional[str] = None,
) -> LLMResponseChunk:
return LLMResponseChunk(
text=text,
language_code=language_code if language_code is not None else turn.language_code,
tools=tools or [],
runtime_config=turn.runtime_config,
response=turn.response,
turn_id=turn.turn_id,
turn_revision=turn.turn_revision,
speech_stopped_at_s=turn.speech_stopped_at_s,
cancel_generation=turn.gen,
)
def _record_tool_call(self, state: _GenState, turn: _Turn, item: ResponseFunctionToolCall) -> Iterator[LLMOut]:
"""Emit a tool call, persisting it (and any assistant text seen so far)
to history *before* it is forwarded to the client.
The function_call must already exist in the conversation by the time the
client returns its ``function_call_output``; otherwise a fast client
races ahead of the deferred end-of-turn write-back and the output is
rejected ("No function_call with call_id ... found"), which makes the
model re-issue the same tool call. The call lands in ``_pending_tool_calls``
(not serialized until its output pairs it), so eager recording is safe.
Out-of-band turns never touch the default conversation, and a stale turn
records nothing (it is not forwarded to the client either)."""
state.tools.append(item)
fc_item = RealtimeConversationItemFunctionCall(
type="function_call",
name=item.name,
arguments=item.arguments,
call_id=item.call_id,
id=item.id,
status=item.status,
)
if self._generation_is_stale(turn.gen) or not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
logger.info("LLM generation cancelled (stale speculative turn)")
return
if not is_out_of_band(turn.response):
# Flush assistant text accumulated before this call first (so history
# order matches what the client received), then persist the call —
# all before the chunk leaves for the client.
chat = turn.runtime_config.chat
for pending_item in state.pending:
chat.add_item(pending_item)
state.pending.clear()
chat.add_item(fc_item)
yield self._chunk(turn, tools=[item])
# ── consumption ─────────────────────────────────────────────────────────--
def _consume_streaming(self, events: Iterator[ProviderEvent], state: _GenState, turn: _Turn) -> Iterator[LLMOut]:
cancelled = False
printable_text = ""
sentence_batch: list[str] = []
def _flush(batch: list[str]) -> Iterator[LLMOut]:
if not batch:
return
if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
logger.info("LLM generation cancelled (stale speculative turn)")
return
yield self._chunk(turn, text=" ".join(batch))
for event in events:
if self._generation_is_stale(turn.gen) or not self._turn_is_latest(turn.turn_id, turn.turn_revision):
logger.info("LLM generation cancelled (interruption)")
cancelled = True
break
if isinstance(event, Usage):
state.input_tokens = event.input_tokens
state.output_tokens = event.output_tokens
elif isinstance(event, AssistantMessage):
state.pending.append(
RealtimeConversationItemAssistantMessage(type="message", role="assistant", content=event.content)
)
elif isinstance(event, ToolCall):
# Flush any pending spoken text before emitting the tool call.
if printable_text.strip():
sentence_batch.append(printable_text.strip())
printable_text = ""
if sentence_batch:
if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
logger.info("LLM generation cancelled (stale speculative turn)")
cancelled = True
break
yield from _flush(sentence_batch)
sentence_batch = []
yield from self._record_tool_call(state, turn, event.item)
elif isinstance(event, TextDelta):
if not turn.wants_audio:
# Text-only: forward verbatim. Keep every character (no
# remove_unspeechable, which strips TTS-unfriendly symbols) and
# don't sentence-split (sent_tokenize collapses newlines/markdown).
state.clean_text += event.text
if event.text:
if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
logger.info("LLM generation cancelled (stale speculative turn)")
cancelled = True
break
yield self._chunk(turn, text=event.text)
continue
new_text = remove_unspeechable(event.text)
state.clean_text += new_text
printable_text += new_text
sentences = sent_tokenize(printable_text)
if len(sentences) > 1:
for s in sentences[:-1]:
sentence_batch.append(s)
if len(sentence_batch) >= self.stream_batch_sentences:
if not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
logger.info("LLM generation cancelled (stale speculative turn)")
cancelled = True
break
yield from _flush(sentence_batch)
sentence_batch = []
if cancelled:
break
printable_text = sentences[-1]
if not cancelled:
if printable_text.strip():
sentence_batch.append(printable_text.strip())
if sentence_batch:
if self._generation_is_stale(turn.gen):
logger.info("LLM generation cancelled (interruption)")
else:
logger.debug(f"Clean text: {state.clean_text}")
yield from _flush(sentence_batch)
logger.info(f"Tools: {state.tools}")
def _consume_nonstreaming(self, events: Iterator[ProviderEvent], state: _GenState, turn: _Turn) -> Iterator[LLMOut]:
if self._generation_is_stale(turn.gen) or not self._turn_is_latest(turn.turn_id, turn.turn_revision):
logger.info("LLM generation cancelled (interruption)")
return
for event in events:
if isinstance(event, Usage):
state.input_tokens = event.input_tokens
state.output_tokens = event.output_tokens
elif isinstance(event, AssistantMessage):
state.pending.append(
RealtimeConversationItemAssistantMessage(type="message", role="assistant", content=event.content)
)
elif isinstance(event, ToolCall):
yield from self._record_tool_call(state, turn, event.item)
elif isinstance(event, TextDelta):
# Text-only keeps every character verbatim; audio strips
# TTS-unfriendly symbols via remove_unspeechable.
spoken = event.text if not turn.wants_audio else remove_unspeechable(event.text)
state.clean_text += spoken
out = spoken if not turn.wants_audio else spoken.strip()
if (
out
and not self._generation_is_stale(turn.gen)
and self._turn_output_allowed(turn.turn_id, turn.turn_revision)
):
yield self._chunk(turn, text=out)
logger.debug(f"Clean text: {state.clean_text}")
logger.info(f"Tools: {state.tools}")
# ── orchestration ─────────────────────────────────────────────────────────
def _generate(
self,
active_chat: Chat,
original_chat: Chat,
turn: _Turn,
optional_kwargs: dict[str, Any],
) -> Iterator[LLMOut]:
api_response: Any = None
state = _GenState()
error_message: str | None = None
api_input = self._serialize(active_chat)
# Images the model actually sees this turn; only these are stripped on
# write-back, so an image a fast client injects mid-generation for the
# next turn survives (it is not in this serialized snapshot).
consumed_image_ids = active_chat.image_message_ids()
if not api_input:
# Nothing to send: empty `instructions` and no `input` (in the response,
# the default conversation, or the out-of-band context). The provider
# would reject this; fail with a clear message instead of an opaque error.
error_message = "Cannot generate a response: no instructions and no input were provided."
try:
if error_message is None:
try:
import json as _json
prompt_dump = _json.dumps(api_input, ensure_ascii=False, default=str)
except Exception:
prompt_dump = repr(api_input)
logger.info(
"LLM REQUEST — model=%s turn=%s rev=%s stream=%s\n>>> PROMPT:\n%s",
self.model_name,
turn.turn_id,
turn.turn_revision,
self.stream,
prompt_dump,
)
api_response = self._request(api_input, optional_kwargs)
if api_response is not None:
events = self._iter_events(api_response)
if self.stream:
yield from self._consume_streaming(events, state, turn)
else:
yield from self._consume_nonstreaming(events, state, turn)
except httpx.ReadTimeout:
logger.warning(
"OpenAI API read timed out after %.1fs; ending the current response",
self.request_timeout_s,
)
if not self._generation_is_stale(turn.gen) and self._turn_output_allowed(turn.turn_id, turn.turn_revision):
# Canned apology carries no language_code (mirrors the prior handlers).
yield LLMResponseChunk(
text="Wow I'm a bit slow today, could you repeat that?",
runtime_config=turn.runtime_config,
response=turn.response,
turn_id=turn.turn_id,
turn_revision=turn.turn_revision,
speech_stopped_at_s=turn.speech_stopped_at_s,
cancel_generation=turn.gen,
)
except Exception as exc:
# Any other generation failure must still terminate the response: record
# the error and fall through to the EndOfResponse below. Without this the
# exception would escape process() and no EndOfResponse would be emitted,
# leaving st.in_response stuck and locking every subsequent response.
logger.exception("LLM generation failed; ending the current response")
if error_message is None:
error_message = f"Language model generation failed: {exc}"
finally:
if api_response is not None and hasattr(api_response, "close"):
try:
api_response.close()
except Exception:
pass
if error_message is not None:
_status = f"error={error_message}"
elif self._generation_is_stale(turn.gen):
_status = "cancelled(stale_gen)"
elif not self._turn_output_allowed(turn.turn_id, turn.turn_revision):
_status = "dropped(not_latest_turn)"
else:
_status = "ok"
try:
import json as _json2
_tools_dump = _json2.dumps(
[
{"name": tc.name, "call_id": tc.call_id, "arguments": tc.arguments}
for tc in state.tools
],
ensure_ascii=False,
)
except Exception:
_tools_dump = repr(state.tools)
logger.info(
"LLM RESPONSE — model=%s turn=%s rev=%s status=%s input_tokens=%d output_tokens=%d\n"
"<<< TEXT:\n%s\n"
"<<< TOOLS:\n%s",
self.model_name,
turn.turn_id,
turn.turn_revision,
_status,
state.input_tokens,
state.output_tokens,
state.clean_text,
_tools_dump,
)
if (
error_message is None
and not self._generation_is_stale(turn.gen)
and self._turn_output_allowed(turn.turn_id, turn.turn_revision)
):
# Out-of-band responses emit output and usage but never write back to the
# default conversation (their context was a throwaway chat).
if not is_out_of_band(turn.response):
# Tool calls (and any assistant text preceding them) were already
# written eagerly in _record_tool_call; only trailing items remain.
for item in state.pending:
original_chat.add_item(item)
original_chat.strip_images(consumed_image_ids)
original_chat.trim_if_needed(self.compactor)
if state.input_tokens or state.output_tokens:
yield TokenUsage(
input_tokens=state.input_tokens,
output_tokens=state.output_tokens,
turn_id=turn.turn_id,
turn_revision=turn.turn_revision,
)
yield EndOfResponse(
turn_id=turn.turn_id, turn_revision=turn.turn_revision, cancel_generation=turn.gen, error=error_message
)
def process(self, request: LLMIn) -> Iterator[LLMOut]:
"""Process a language model request and yield LLMResponseChunks."""
runtime_config = request.runtime_config
response = request.response
turn_id = request.turn_id
turn_revision = request.turn_revision
speech_stopped_at_s = request.speech_stopped_at_s
if not self._turn_is_latest(turn_id, turn_revision):
logger.info("Skipping stale LLM request for turn=%s rev=%s", turn_id, turn_revision)
yield EndOfResponse(turn_id=turn_id, turn_revision=turn_revision)
return
original_chat = runtime_config.chat
if is_out_of_band(response):
try:
active_chat = build_active_chat(original_chat, response)
except ChatItemError as exc:
logger.info("Out-of-band response rejected: %s", exc)
yield EndOfResponse(turn_id=turn_id, turn_revision=turn_revision, error=str(exc))
return
else:
active_chat = original_chat.copy()
language_code = request.language_code
instructions = (
response.instructions if response and response.instructions else runtime_config.session.instructions
) or ""
req_tools = response.tools if response and response.tools else runtime_config.session.tools
req_tool_choice = (
response.tool_choice if response and response.tool_choice else runtime_config.session.tool_choice
)
wants_audio = response_wants_audio(response)
self._apply_config(active_chat, instructions, wants_audio)
self._inject_rag_context(active_chat, turn_id, turn_revision)
language_code, lang_name = resolve_auto_language(language_code)
if lang_name and self.enable_lang_prompt:
active_chat.add_item(make_user_message(f"Please reply to my message in {lang_name}."))
optional_kwargs = self._build_optional_kwargs(req_tools, req_tool_choice)
# CancelScope.is_stale(gen) is checked when the stream iterator advances; a
# blocked read inside httpx cannot be aborted by cancel_scope.cancel() from
# the websocket router. Mitigations: request_timeout_s / ReadTimeout.
gen = self.cancel_scope.generation if self.cancel_scope else None
turn = _Turn(
language_code=language_code,
gen=gen,
runtime_config=runtime_config,
response=response,
turn_id=turn_id,
turn_revision=turn_revision,
speech_stopped_at_s=speech_stopped_at_s,
wants_audio=wants_audio,
)
yield from self._generate(active_chat, original_chat, turn, optional_kwargs)
@property
def timing_log_level(self) -> int:
return logging.INFO
def should_log_timing(self, output: LLMOut) -> bool:
return isinstance(output, LLMResponseChunk) and self.last_time > self.min_time_to_debug

760
src/speech_to_speech/LLM/chat.py Normal file
View File

@ -0,0 +1,760 @@
from __future__ import annotations
import json
import logging
import threading
from collections.abc import Callable
from typing import Any, Literal, Union
from openai.types.realtime import ConversationItem
from openai.types.realtime.conversation_item import (
RealtimeConversationItemAssistantMessage,
RealtimeConversationItemFunctionCall,
RealtimeConversationItemFunctionCallOutput,
RealtimeConversationItemSystemMessage,
RealtimeConversationItemUserMessage,
)
from openai.types.realtime.realtime_conversation_item_assistant_message import (
Content as AssistantContent,
)
from openai.types.realtime.realtime_conversation_item_system_message import Content as SystemContent
from openai.types.realtime.realtime_conversation_item_user_message import Content as UserContent
from openai.types.realtime.realtime_response_create_params import RealtimeResponseCreateParams
from openai.types.responses.response_input_image_param import ResponseInputImageParam
from openai.types.responses.response_input_message_content_list_param import (
ResponseInputMessageContentListParam,
)
from openai.types.responses.response_input_param import (
FunctionCallOutput,
ResponseFunctionToolCallParam,
ResponseInputItemParam,
ResponseInputParam,
ResponseOutputMessageParam,
)
from openai.types.responses.response_input_param import (
Message as ResponseMessage,
)
from openai.types.responses.response_input_text_param import ResponseInputTextParam
from openai.types.responses.response_output_text_param import ResponseOutputTextParam
from pydantic import BaseModel
from speech_to_speech.utils.utils import _generate_id
logger = logging.getLogger(__name__)
class ChatItemError(Exception):
"""Raised when a conversation item fails validation in :meth:`Chat.add_item`."""
class CompactionResult(BaseModel):
"""Output of a :data:`CompactFn` summarization run."""
user_summary: str
assistant_summary: str
def _ensure_id(value: str | None, prefix: str) -> str:
if value is None:
return _generate_id(prefix)
if not value.startswith(f"{prefix}_"):
raise ChatItemError(f"ID must start with '{prefix}_', got {value!r}")
return value
SupportedItem = Union[
RealtimeConversationItemSystemMessage,
RealtimeConversationItemUserMessage,
RealtimeConversationItemAssistantMessage,
RealtimeConversationItemFunctionCall,
RealtimeConversationItemFunctionCallOutput,
]
CompactFn = Callable[[ResponseInputParam], CompactionResult]
class Chat:
"""Manages conversation history with bounded size to avoid OOM issues.
The buffer stores ``ConversationItem`` objects (user messages, assistant
messages, function calls, function call outputs). System messages are
stored separately in ``init_chat_message`` and never placed in the buffer.
History bounding is decided per ``add_item`` call via the ``compactor``
argument:
- ``compactor=None``: when the user-turn count exceeds ``size`` the oldest
complete turn is evicted in place. Synchronous, lossy, no LLM involvement.
- ``compactor=<fn>``: when ``size`` is exceeded, ``fn`` is invoked in a
background thread to summarize older turns into a single user/assistant
pair (with pending function calls preserved). Single-flight: while a
compaction is running, additional triggers are silently bypassed.
"""
def __init__(self, size: int) -> None:
self.size = size
self.init_chat_message: RealtimeConversationItemSystemMessage | None = None
# ``size`` is the number of user turns to keep. When exceeded the
# oldest complete turn (everything up to the next user message)
# is evicted -- or, with a compactor, summarized in the background.
self.buffer: list[SupportedItem] = []
self._pending_tool_calls: dict[str, RealtimeConversationItemFunctionCall] = {}
self._user_turn_count: int = 0
# All state mutations and serializations go through _lock. Public methods
# acquire it once; internal callers that already hold it use the
# ``_locked`` helpers, so no reentry is needed (regular Lock is safe).
self._lock = threading.Lock()
self._compact_in_flight: bool = False
self._compact_thread: threading.Thread | None = None
self._shutdown = threading.Event()
self._gen_counter = 0
# ── Internal mutators (caller holds _lock) ─────────────────
def _evict_oldest_turn(self) -> None:
"""Remove items from the front until the next user message boundary."""
if not self.buffer:
return
first = self.buffer.pop(0)
if isinstance(first, RealtimeConversationItemUserMessage):
self._user_turn_count -= 1
while self.buffer and not isinstance(self.buffer[0], RealtimeConversationItemUserMessage):
self.buffer.pop(0)
def _has_call_id_in_buffer(self, call_id: str) -> bool:
for entry in self.buffer:
if isinstance(entry, RealtimeConversationItemFunctionCall) and entry.call_id == call_id:
return True
return False
def _mark_call_completed(
self, call_id: str, status: Literal["completed", "incomplete", "in_progress"] | None = None
) -> None:
"""Set ``status`` to ``"completed"`` on the matching function_call."""
for entry in self.buffer:
if isinstance(entry, RealtimeConversationItemFunctionCall) and entry.call_id == call_id:
entry.status = "completed" if status is None else status
return
def append_tool_output(self, call_id: str, output_item: RealtimeConversationItemFunctionCallOutput) -> None:
"""Append a ``function_call_output``, re-injecting its ``function_call`` if evicted.
Also marks the paired ``function_call`` as ``"completed"`` if its
status was ``None``.
Raises :class:`ChatItemError` if *call_id* is unknown.
"""
with self._lock:
self._append_tool_output_locked(call_id, output_item)
def _append_tool_output_locked(self, call_id: str, output_item: RealtimeConversationItemFunctionCallOutput) -> None:
"""Body of :meth:`append_tool_output`. Caller must hold ``_lock``."""
if self._has_call_id_in_buffer(call_id):
self._pending_tool_calls.pop(call_id, None)
self._mark_call_completed(call_id, output_item.status)
self.buffer.append(output_item)
return
if call_id in self._pending_tool_calls:
logger.info("Re-injecting evicted function_call for call_id=%s", call_id)
fc = self._pending_tool_calls.pop(call_id)
fc.status = "completed" if output_item.status is None else output_item.status
self.buffer.append(fc)
self.buffer.append(output_item)
return
raise ChatItemError(f"No function_call with call_id '{call_id}' found in conversation history.")
def init_chat(self, message: RealtimeConversationItemSystemMessage) -> None:
with self._lock:
self.init_chat_message = message
def add_item(self, item: SupportedItem) -> SupportedItem:
"""Validate and route a conversation item into the chat buffer.
Does not enforce the soft size limit — call :meth:`trim_if_needed`
explicitly after each successful generation to evict or compact old
turns. A hard upper bound at ``2 * size`` is enforced inline as a
runaway-client safety net: if the user-turn count exceeds it, the
oldest complete turn is evicted (lossy, no compaction).
Raises :class:`ChatItemError` if the item fails validation.
"""
with self._lock:
if isinstance(item, RealtimeConversationItemSystemMessage):
item.id = _ensure_id(item.id, "sys")
self.init_chat_message = item
logger.debug("Set system message via conversation item")
elif isinstance(item, RealtimeConversationItemUserMessage):
item.id = _ensure_id(item.id, "msg")
item.content = [
p
for p in item.content
if (p.type == "input_text" and p.text) or (p.type == "input_image" and p.image_url)
]
if not item.content:
raise ChatItemError(
"Message has no supported content. Supported modalities: input_text, input_image."
)
self.buffer.append(item)
self._user_turn_count += 1
logger.debug("Added user message to chat (%d parts)", len(item.content))
elif isinstance(item, RealtimeConversationItemAssistantMessage):
item.id = _ensure_id(item.id, "msg")
item.content = [p for p in item.content if p.type == "output_text" and p.text]
if not item.content:
return item
self.buffer.append(item)
logger.debug("Added assistant message to chat (%d parts)", len(item.content))
elif isinstance(item, RealtimeConversationItemFunctionCall):
item.id = _ensure_id(item.id, "fc")
item.call_id = _ensure_id(item.call_id, "call")
self._pending_tool_calls[item.call_id] = item
logger.debug("Added function_call to chat (call_id=%s)", item.call_id)
elif isinstance(item, RealtimeConversationItemFunctionCallOutput):
item.id = _ensure_id(item.id, "fco")
self._append_tool_output_locked(item.call_id, item)
logger.debug("Added function_call_output to chat (call_id=%s)", item.call_id)
else:
raise ChatItemError(f"Unsupported item type: {getattr(item, 'type', None)}")
if self.size > 0 and self._user_turn_count > 2 * self.size:
logger.warning(
"Chat buffer exceeded hard cap (%d > 2 * size=%d); evicting oldest turn",
self._user_turn_count,
self.size,
)
while self._user_turn_count > 2 * self.size:
self._evict_oldest_turn()
return item
def trim_if_needed(self, compactor: CompactFn | None = None) -> None:
"""Enforce the size limit after a generation completes. Fires when
``user_turn_count > size``.
- ``compactor=None``: synchronous eviction of the oldest complete turn.
- ``compactor=<fn>``: launch a background compaction (single-flight).
Call once after each successful generation, not inside :meth:`add_item`.
"""
with self._lock:
if self._user_turn_count <= self.size:
return
if compactor is not None:
self._maybe_trigger_compaction(compactor)
else:
while self._user_turn_count > self.size:
self._evict_oldest_turn()
def replace_user_message_text(self, item_id: str, text: str) -> bool:
"""Replace the text content of an existing user message.
Used by speculative turn revisions: the conversation turn remains the
same, but the STT transcript is superseded by a transcription of a
longer raw-audio buffer.
"""
with self._lock:
for item in self.buffer:
if not isinstance(item, RealtimeConversationItemUserMessage) or item.id != item_id:
continue
item.content = [UserContent(type="input_text", text=text)]
logger.debug("Replaced speculative user message %s", item_id)
return True
return False
def remove_user_message(self, item_id: str) -> bool:
"""Remove an existing user message from the bounded chat buffer."""
with self._lock:
for index, item in enumerate(self.buffer):
if not isinstance(item, RealtimeConversationItemUserMessage) or item.id != item_id:
continue
del self.buffer[index]
self._user_turn_count -= 1
logger.debug("Removed speculative user message %s", item_id)
return True
return False
def to_responses_api_chat(self, items: list[SupportedItem] | None = None) -> ResponseInputParam:
"""Serialize the chat (system prompt + buffer) for the OpenAI Responses API.
If *items* is provided, serialize that slice instead of the live buffer
(used by the compaction snapshot).
"""
with self._lock:
return self._to_responses_api_chat_locked(items if items is not None else self.buffer)
def _to_responses_api_chat_locked(self, items: list[SupportedItem]) -> ResponseInputParam:
"""Body of :meth:`to_responses_api_chat`. Caller must hold ``_lock``."""
buffer_items = list(items)
result: list[ResponseInputItemParam] = []
if self.init_chat_message:
result.append(
ResponseMessage(
content=[
ResponseInputTextParam(text=p.text or "A helpful AI assistant.", type="input_text")
for p in self.init_chat_message.content
],
role="system",
type="message",
)
)
for item in buffer_items:
assert item.id is not None and item.id != "", f"item.id is {item.id}"
if isinstance(item, RealtimeConversationItemUserMessage):
content: ResponseInputMessageContentListParam = []
for user_part in item.content:
if user_part.type == "input_text" and user_part.text is not None:
content.append(ResponseInputTextParam(text=user_part.text or "", type="input_text"))
elif user_part.type == "input_image" and user_part.image_url is not None:
img = ResponseInputImageParam(type="input_image", detail=user_part.detail or "auto")
if user_part.image_url is not None:
img["image_url"] = user_part.image_url
content.append(img)
if content:
result.append(ResponseMessage(content=content, role="user", type="message"))
elif isinstance(item, RealtimeConversationItemAssistantMessage):
assistant_content: list[ResponseOutputTextParam] = []
for assistant_part in item.content:
if assistant_part.type == "output_text" and assistant_part.text is not None:
assistant_content.append(
ResponseOutputTextParam(text=assistant_part.text, type="output_text", annotations=[])
)
if assistant_content:
result.append(
ResponseOutputMessageParam(
id=item.id,
content=assistant_content,
role="assistant",
status=item.status or "completed",
type="message",
)
)
elif isinstance(item, RealtimeConversationItemFunctionCall) and item.call_id is not None:
assert item.call_id is not None and item.call_id != ""
function_call = ResponseFunctionToolCallParam(
arguments=item.arguments,
call_id=item.call_id,
name=item.name,
type="function_call",
id=item.id,
)
if item.id is not None:
function_call["id"] = item.id
if item.status is not None:
function_call["status"] = item.status
result.append(function_call)
elif isinstance(item, RealtimeConversationItemFunctionCallOutput):
function_call_output = FunctionCallOutput(
call_id=item.call_id,
output=item.output,
type="function_call_output",
)
if item.id is not None:
function_call_output["id"] = item.id
if item.status is not None:
function_call_output["status"] = item.status
result.append(function_call_output)
return result
def to_transformers_chat(self) -> list[dict[str, Any]]:
"""Serialize the full chat for HuggingFace transformers ``apply_chat_template``.
User messages with only text produce a plain string ``content`` value.
User messages containing images keep ``content`` as a list of dicts so
VLM pipelines can process them.
"""
with self._lock:
messages: list[TransformersChatMessage] = []
if self.init_chat_message:
text = " ".join(p.text for p in self.init_chat_message.content if p.text)
messages.append(TransformersSystemMessage(content=text))
for item in self.buffer:
if isinstance(item, RealtimeConversationItemUserMessage):
has_images = any(p.type == "input_image" for p in item.content)
if has_images:
messages.append(
TransformersUserMessage(content=[p.model_dump(exclude_none=True) for p in item.content])
)
else:
text = " ".join(p.text for p in item.content if p.type == "input_text" and p.text)
messages.append(TransformersUserMessage(content=text))
elif isinstance(item, RealtimeConversationItemAssistantMessage):
text = " ".join(p.text for p in item.content if p.text)
messages.append(TransformersAssistantMessage(content=text))
elif isinstance(item, RealtimeConversationItemFunctionCall):
assert item.call_id is not None and item.call_id != ""
args: Any = item.arguments
try:
args = json.loads(args) if isinstance(args, str) else args
except (json.JSONDecodeError, TypeError):
args = {}
messages.append(
TransformersFunctionCallMessage(
tool_calls=[
TransformersToolCall(
id=item.call_id,
function=TransformersToolCallFunction(name=item.name, arguments=args),
)
]
)
)
elif isinstance(item, RealtimeConversationItemFunctionCallOutput):
name = ""
for prev in reversed(messages):
if isinstance(prev, TransformersFunctionCallMessage):
for tc in prev.tool_calls:
if tc.id == item.call_id:
name = tc.function.name
break
if name:
break
messages.append(
TransformersToolMessage(
tool_call_id=item.call_id,
name=name,
content=item.output,
)
)
return [m.model_dump() for m in messages]
def copy(self) -> Chat:
"""Return a shallow snapshot safe for concurrent read access."""
with self._lock:
clone = Chat(self.size)
clone.init_chat_message = self.init_chat_message
clone.buffer = list(self.buffer)
clone._pending_tool_calls = dict(self._pending_tool_calls)
clone._user_turn_count = self._user_turn_count
return clone
def reset(self) -> None:
"""Clear all conversation state. Cancels any in-flight compaction splice."""
with self._lock:
self._gen_counter += 1
self._compact_in_flight = False
self.buffer = []
self.init_chat_message = None
self._pending_tool_calls = {}
self._user_turn_count = 0
def close(self) -> None:
"""Permanently shut down the chat. In-flight compaction splice is suppressed.
The compaction worker (a daemon thread) is not joined: it may be blocked
in an LLM call. Process exit reaps it.
"""
self._shutdown.set()
with self._lock:
self._gen_counter += 1
self._compact_in_flight = False
def image_message_ids(self) -> set[str]:
"""IDs of user messages currently carrying ``input_image`` content."""
with self._lock:
return {
item.id
for item in self.buffer
if isinstance(item, RealtimeConversationItemUserMessage)
and item.id is not None
and any(p.type == "input_image" for p in item.content)
}
def strip_images(self, only_ids: set[str] | None = None) -> None:
"""Remove image content parts from user messages in the buffer.
Called after appending the assistant response so images don't persist
across turns. With *only_ids*, strip only those message IDs — the images
the just-completed response actually consumed (captured before the
request was sent). This leaves intact an image a fast client injected
mid-generation for the *next* turn, which the current response never saw.
Without *only_ids*, every image is stripped.
"""
with self._lock:
for item in self.buffer:
if isinstance(item, RealtimeConversationItemUserMessage):
if only_ids is not None and item.id not in only_ids:
continue
item.content = [p for p in item.content if p.type != "input_image"]
# ── Compaction internals ──────────────────────────────────
def _snapshot_for_compaction(
self,
) -> tuple[ResponseInputParam, set[str], int]:
"""Compute the snapshot of items eligible for compaction.
Caller must hold ``_lock``. Returns
``(serialized_snapshot, marker_ids, n_turns)``. ``marker_ids``
identifies the buffer items that may be removed when the splice runs.
Always leaves the most recent user turn untouched (it may be in-flight).
Returns an empty result if there are fewer than 2 compactable turns.
"""
n_turns = max(0, self._user_turn_count - 1)
if n_turns < 2:
return [], set(), n_turns
# Slice up to (but not including) the (n_turns + 1)-th user message.
user_seen = 0
end_idx = len(self.buffer)
for i, entry in enumerate(self.buffer):
if isinstance(entry, RealtimeConversationItemUserMessage):
user_seen += 1
if user_seen == n_turns + 1:
end_idx = i
break
items_to_compact = self.buffer[:end_idx]
marker_ids = {entry.id for entry in items_to_compact if entry.id is not None}
snapshot = self._to_responses_api_chat_locked(items=items_to_compact)
# Strip image parts so the summarizer doesn't have to handle them.
for raw in snapshot:
if not isinstance(raw, dict) or raw.get("role") != "user":
continue
msg: dict[str, Any] = raw # type: ignore[assignment]
content = msg.get("content")
if isinstance(content, list):
msg["content"] = [c for c in content if not (isinstance(c, dict) and c.get("type") == "input_image")]
return snapshot, marker_ids, n_turns
def _maybe_trigger_compaction(self, compactor: CompactFn) -> None:
"""Start a background compaction worker. Bypass silently if one is running.
Caller must hold ``_lock``.
"""
if self._shutdown.is_set() or self._compact_in_flight:
return
snapshot, marker_ids, n_turns = self._snapshot_for_compaction()
if n_turns < 2 or not marker_ids:
return
gen = self._gen_counter
self._compact_in_flight = True
thread = threading.Thread(
target=self._compact_worker,
args=(compactor, snapshot, marker_ids, gen),
daemon=True,
name="chat-compact",
)
self._compact_thread = thread
logger.info(
"Chat compaction triggered: compacting %d turn(s) (%d item(s)), buffer size=%d",
n_turns,
len(marker_ids),
len(self.buffer),
)
thread.start()
def _compact_worker(
self,
compactor: CompactFn,
snapshot: ResponseInputParam,
marker_ids: set[str],
gen: int,
) -> None:
"""Worker thread entry point."""
try:
if self._shutdown.is_set() or self._gen_counter != gen:
return
try:
result = compactor(snapshot)
except Exception:
logger.exception("Chat compaction failed; chat unchanged")
return
if not isinstance(result, CompactionResult):
logger.error("Compactor must return a CompactionResult, got %r", type(result).__name__)
return
if self._shutdown.is_set() or self._gen_counter != gen:
return
self._apply_compaction(result, marker_ids, gen)
finally:
# Don't clobber the flag if reset/close has advanced the gen.
with self._lock:
if self._gen_counter == gen:
self._compact_in_flight = False
def _apply_compaction(
self,
result: CompactionResult,
marker_ids: set[str],
gen: int,
) -> None:
"""Splice the summary in front of items not consumed by compaction.
FC/FCO pairing is left entirely to :meth:`add_item` / :meth:`append_tool_output`.
Compaction only drops items; it never inserts an FC into the buffer.
Pending FCs (no FCO yet) stay in ``_pending_tool_calls`` and will be
appended adjacent to their FCO when it arrives.
"""
with self._lock:
if self._shutdown.is_set() or self._gen_counter != gen:
return
# Keep FC if its FCO is outside the compacted range -- otherwise
# the FCO in `remaining` would be orphaned.
fco_call_ids_in_range = {
x.call_id
for x in self.buffer
if isinstance(x, RealtimeConversationItemFunctionCallOutput) and x.id in marker_ids
}
fc_ids_to_keep = {
x.id
for x in self.buffer
if x.id in marker_ids
and isinstance(x, RealtimeConversationItemFunctionCall)
and x.call_id not in fco_call_ids_in_range
}
drop_ids = marker_ids - fc_ids_to_keep
remaining = [x for x in self.buffer if x.id not in drop_ids]
user_msg = make_user_message(result.user_summary)
user_msg.id = _generate_id("msg")
asst_msg = make_assistant_message(result.assistant_summary)
asst_msg.id = _generate_id("msg")
self.buffer = [user_msg, asst_msg, *remaining]
self._user_turn_count = sum(1 for x in self.buffer if isinstance(x, RealtimeConversationItemUserMessage))
logger.info(
"Chat compaction applied: buffer now %d item(s), %d user turn(s)",
len(self.buffer),
self._user_turn_count,
)
# ---------------------------------------------------------------------------
# Transformers chat message models
# ---------------------------------------------------------------------------
class TransformersToolCallFunction(BaseModel):
name: str
arguments: dict[str, Any]
class TransformersToolCall(BaseModel):
type: Literal["function"] = "function"
id: str
function: TransformersToolCallFunction
class TransformersSystemMessage(BaseModel):
role: Literal["system"] = "system"
content: str
class TransformersUserMessage(BaseModel):
role: Literal["user"] = "user"
content: str | list[dict[str, Any]]
class TransformersAssistantMessage(BaseModel):
role: Literal["assistant"] = "assistant"
content: str
class TransformersFunctionCallMessage(BaseModel):
role: Literal["assistant"] = "assistant"
tool_calls: list[TransformersToolCall]
class TransformersToolMessage(BaseModel):
role: Literal["tool"] = "tool"
tool_call_id: str
name: str
content: str
TransformersChatMessage = Union[
TransformersSystemMessage,
TransformersUserMessage,
TransformersAssistantMessage,
TransformersFunctionCallMessage,
TransformersToolMessage,
]
# ---------------------------------------------------------------------------
# Factory helpers -- hide verbose constructors behind simple calls
# ---------------------------------------------------------------------------
def make_user_message(text: str) -> RealtimeConversationItemUserMessage:
return RealtimeConversationItemUserMessage(
type="message",
role="user",
content=[UserContent(type="input_text", text=text)],
)
def make_assistant_message(text: str) -> RealtimeConversationItemAssistantMessage:
return RealtimeConversationItemAssistantMessage(
type="message",
role="assistant",
content=[AssistantContent(type="output_text", text=text)],
)
def make_system_message(text: str) -> RealtimeConversationItemSystemMessage:
return RealtimeConversationItemSystemMessage(
type="message",
role="system",
content=[SystemContent(type="input_text", text=text)],
)
def add_supported_item(chat: Chat, item: ConversationItem) -> None:
"""Narrow a protocol ``ConversationItem`` to a :data:`SupportedItem` and add it to *chat*.
Raises :class:`ChatItemError` on validation failure or unsupported type. Shared
by the conversation handler (in-band item injection) and the language-model
handlers (seeding an out-of-band response's throwaway chat from ``response.input``).
"""
# call_id on function_call items must be client-supplied: it is referenced later by
# function_call_output items, so we cannot silently generate one here.
if isinstance(item, RealtimeConversationItemFunctionCall) and (
item.call_id is None or not item.call_id.startswith("call_")
):
raise ChatItemError("function_call item is missing a call_id. The call_id should start with 'call_'.")
if isinstance(
item,
(
RealtimeConversationItemSystemMessage,
RealtimeConversationItemUserMessage,
RealtimeConversationItemAssistantMessage,
RealtimeConversationItemFunctionCall,
RealtimeConversationItemFunctionCallOutput,
),
):
chat.add_item(item)
return
raise ChatItemError(f"Unsupported item type: {getattr(item, 'type', None)}")
def build_active_chat(original_chat: Chat, response: RealtimeResponseCreateParams | None) -> Chat:
"""Build the chat an *out-of-band* response generates against (caller ensures out-of-band).
Mirrors the OpenAI realtime semantics for ``input``:
- ``input is None`` -> a read-only **copy of the default conversation** (the
out-of-band response reads history but never commits back).
- ``input == []`` -> a **fresh, empty chat** (context cleared; only the
system prompt, added later by the handler, will be present).
- ``input == [...]`` -> a **fresh chat seeded** with those items.
Raises :class:`ChatItemError` if an ``input`` item fails validation.
"""
if response is not None and response.input is not None:
fresh = Chat(original_chat.size)
for item in response.input:
add_supported_item(fresh, item)
return fresh
return original_chat.copy()

311
src/speech_to_speech/LLM/chat_completions_language_model.py Normal file
View File

@ -0,0 +1,311 @@
from __future__ import annotations
import json
import logging
import time
from collections.abc import Iterator
from typing import Any, cast
from openai import Stream
from openai.types.chat import (
ChatCompletionChunk,
ChatCompletionContentPartImageParam,
ChatCompletionContentPartParam,
ChatCompletionContentPartTextParam,
ChatCompletionNamedToolChoiceParam,
ChatCompletionToolChoiceOptionParam,
ChatCompletionToolParam,
)
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
from openai.types.chat.chat_completion_named_tool_choice_param import Function as NamedToolChoiceFunction
from openai.types.realtime.realtime_conversation_item_assistant_message import (
Content as AssistantContent,
)
from openai.types.responses import ResponseFunctionToolCall
from openai.types.shared_params import FunctionDefinition
from speech_to_speech.LLM.base_openai_compatible_language_model import (
WARMUP_MAX_RETRIES,
AssistantMessage,
BaseOpenAICompatibleHandler,
ProviderEvent,
TextDelta,
ToolCall,
Usage,
)
from speech_to_speech.LLM.chat import Chat
from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn
from speech_to_speech.utils.utils import _generate_id
logger = logging.getLogger(__name__)
def _to_chat_tools(req_tools: Any) -> list[ChatCompletionToolParam] | None:
"""Convert Responses-API function tools to Chat-Completions tool format.
Responses tools are flat ``{type:"function", name, description, parameters}``;
Chat Completions nests them under a ``function`` key. Items already in the
nested form (or non-function tools) are passed through untouched.
"""
if not req_tools:
return None
chat_tools: list[ChatCompletionToolParam] = []
for t in req_tools:
d = t if isinstance(t, dict) else t.model_dump(exclude_none=True)
if d.get("type") == "function" and "function" not in d:
fn = FunctionDefinition(name=d["name"])
if d.get("description") is not None:
fn["description"] = d["description"]
if d.get("parameters") is not None:
fn["parameters"] = d["parameters"]
chat_tools.append(ChatCompletionToolParam(type="function", function=fn))
else:
chat_tools.append(cast("ChatCompletionToolParam", d))
return chat_tools
def _to_chat_tool_choice(tool_choice: Any) -> ChatCompletionToolChoiceOptionParam:
"""Convert a Responses-API tool_choice to Chat-Completions form.
The string forms ("auto"/"required"/"none") are identical across both APIs;
only the forced-function object differs (flat ``name`` vs nested ``function``).
"""
if isinstance(tool_choice, dict) and tool_choice.get("type") == "function" and "name" in tool_choice:
return ChatCompletionNamedToolChoiceParam(
type="function", function=NamedToolChoiceFunction(name=tool_choice["name"])
)
if tool_choice is not None and not isinstance(tool_choice, (str, dict)):
d = tool_choice.model_dump(exclude_none=True)
if d.get("type") == "function" and "name" in d:
return ChatCompletionNamedToolChoiceParam(type="function", function=NamedToolChoiceFunction(name=d["name"]))
return cast("ChatCompletionToolChoiceOptionParam", d)
return cast("ChatCompletionToolChoiceOptionParam", tool_choice)
class ChatCompletionsApiModelHandler(BaseOpenAICompatibleHandler):
"""LLM handler that talks to an OpenAI-compatible ``/v1/chat/completions`` server.
Functionally mirrors :class:`ResponsesApiModelHandler` but uses the mature
Chat Completions streaming tool-call protocol (``choices[].delta.tool_calls``)
instead of ``/v1/responses``. This is the robust path for vLLM + Qwen tool
calling. The conversation is serialised with :meth:`Chat.to_transformers_chat`,
which already emits OpenAI chat messages including ``tool_calls``/``tool`` roles.
"""
def warmup(self) -> None:
logger.info(f"Warming up {self.__class__.__name__}")
start = time.time()
self.client.with_options(max_retries=WARMUP_MAX_RETRIES).chat.completions.create(
model=self.model_name,
messages=[
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello"},
],
extra_body=self._extra_body,
timeout=self.request_timeout,
)
end = time.time()
logger.info(f"{self.__class__.__name__}: warmed up! time: {(end - start):.3f} s")
def _build_compaction_generate_fn(self) -> CompactGenerateFn:
"""Return a generate fn that calls Chat Completions for compaction."""
client = self.client
model_name = self.model_name
timeout = self.request_timeout
extra_body = self._extra_body
def generate(system: str, user: str) -> str:
response = client.chat.completions.create(
model=model_name,
messages=[
{"role": "system", "content": system},
{"role": "user", "content": user},
],
extra_body=extra_body,
timeout=timeout,
)
return response.choices[0].message.content or ""
return generate
@staticmethod
def _to_chat_content_part(part: dict[str, Any]) -> ChatCompletionContentPartParam:
"""Convert one transformers content part to Chat-Completions shape.
``to_transformers_chat`` keeps Realtime-style parts (``input_text`` /
``input_image`` with a bare-string ``image_url``). The Chat Completions
HTTP API instead wants ``{type:"text", text}`` and
``{type:"image_url", image_url:{url, detail}}``. Unknown parts pass through.
"""
ptype = part.get("type")
if ptype == "input_text":
return ChatCompletionContentPartTextParam(type="text", text=part.get("text") or "")
if ptype == "input_image":
raw_url: Any = part.get("image_url")
if isinstance(raw_url, dict):
image_url = cast("ImageURL", raw_url)
else:
image_url = ImageURL(url=raw_url)
detail = part.get("detail")
if detail is not None:
image_url["detail"] = detail
return ChatCompletionContentPartImageParam(type="image_url", image_url=image_url)
return cast("ChatCompletionContentPartParam", part)
@classmethod
def _chat_messages(cls, chat: Chat) -> list[dict[str, Any]]:
"""Serialise the chat for the Chat Completions API.
``Chat.to_transformers_chat`` targets HuggingFace ``apply_chat_template``,
so two shapes need fixing up for the OpenAI Chat Completions HTTP API:
tool-call ``arguments`` must be a JSON *string* (not a parsed object), and
multimodal ``content`` parts must use the Chat Completions ``text`` /
``image_url`` shape rather than the Realtime ``input_text`` /
``input_image`` shape.
"""
messages = chat.to_transformers_chat()
for message in messages:
for tool_call in message.get("tool_calls") or []:
fn = tool_call.get("function")
if fn is not None and not isinstance(fn.get("arguments"), str):
fn["arguments"] = json.dumps(fn.get("arguments") or {}, ensure_ascii=False)
content = message.get("content")
if isinstance(content, list):
message["content"] = [cls._to_chat_content_part(p) for p in content]
if message.get("role") == "tool":
message.pop("name", None)
return messages
# ── base hooks ──────────────────────────────────────────────────────────--
def _serialize(self, active_chat: Chat) -> list[dict[str, Any]]:
return self._chat_messages(active_chat)
def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
optional_kwargs: dict[str, Any] = {}
chat_tools = _to_chat_tools(req_tools)
if chat_tools is not None:
optional_kwargs["tools"] = chat_tools
if req_tool_choice is not None:
optional_kwargs["tool_choice"] = _to_chat_tool_choice(req_tool_choice)
return optional_kwargs
def _request(self, api_input: list[dict[str, Any]], optional_kwargs: dict[str, Any]) -> Any:
create_kwargs: dict[str, Any] = dict(optional_kwargs)
if self.stream:
create_kwargs["stream_options"] = {"include_usage": True}
# Svuota i parametri dei tool per evitare il blocco 400 di NVIDIA NIM
create_kwargs.pop("tools", None)
create_kwargs.pop("tool_choice", None)
return self.client.chat.completions.create(
model=self.model_name,
messages=api_input, # type: ignore[arg-type] # runtime dicts match the Chat Completions message shape
stream=self.stream,
extra_body=self._extra_body,
timeout=self.request_timeout,
**create_kwargs,
)
# def _request(self, api_input: list[dict[str, Any]], optional_kwargs: dict[str, Any]) -> Any:
# create_kwargs: dict[str, Any] = dict(optional_kwargs)
# if self.stream:
# create_kwargs["stream_options"] = {"include_usage": True}
# return self.client.chat.completions.create(
# model=self.model_name,
# messages=api_input, # type: ignore[arg-type] # runtime dicts match the Chat Completions message shape
# stream=self.stream,
# extra_body=self._extra_body,
# timeout=self.request_timeout,
# **create_kwargs,
# )
def _iter_stream_events(self, api_response: Stream[ChatCompletionChunk]) -> Iterator[ProviderEvent]:
# Accumulate streamed tool-call deltas, keyed by their stream index, and the
# raw assistant text, then emit assistant message + tool calls + usage once
# the stream is exhausted.
tool_accum: dict[int, dict[str, str]] = {}
usage: Usage | None = None
raw_text = ""
for chunk in api_response:
# Usage-only trailing chunk (choices == []) when include_usage is set.
if chunk.usage is not None:
usage = Usage(
input_tokens=chunk.usage.prompt_tokens or 0, output_tokens=chunk.usage.completion_tokens or 0
)
if not chunk.choices:
continue
delta = chunk.choices[0].delta
if delta.tool_calls:
for tc in delta.tool_calls:
entry = tool_accum.setdefault(tc.index, {"name": "", "args": "", "id": ""})
if tc.id:
entry["id"] = tc.id
if tc.function is not None:
if tc.function.name:
entry["name"] = tc.function.name
if tc.function.arguments:
entry["args"] += tc.function.arguments
# A refusal streams as `delta.refusal` with `delta.content` None;
# surface it as assistant text so it is spoken and stored.
text_piece = delta.content or getattr(delta, "refusal", None)
if text_piece:
raw_text += text_piece
yield TextDelta(text=text_piece)
if raw_text.strip():
yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_text)])
yield from self._tool_calls_from_accum(tool_accum)
if usage is not None:
yield usage
def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
usage = api_response.usage
if usage:
yield Usage(input_tokens=usage.prompt_tokens or 0, output_tokens=usage.completion_tokens or 0)
# A valid-but-empty response (e.g. content filter) returns no choices;
# complete cleanly with no assistant text rather than raising IndexError.
message = api_response.choices[0].message if api_response.choices else None
if message is None:
return
# A refusal arrives as `message.refusal` with `message.content` None; treat
# it as assistant text so it is spoken and stored.
raw_content = message.content or getattr(message, "refusal", None)
if raw_content:
yield AssistantMessage(content=[AssistantContent(type="output_text", text=raw_content)])
yield TextDelta(text=raw_content)
tool_accum: dict[int, dict[str, str]] = {}
for tc in message.tool_calls or []:
tool_accum[len(tool_accum)] = {
"name": tc.function.name or "",
"args": tc.function.arguments or "",
"id": tc.id or "",
}
yield from self._tool_calls_from_accum(tool_accum)
@staticmethod
def _tool_calls_from_accum(tool_accum: dict[int, dict[str, str]]) -> Iterator[ToolCall]:
"""Turn accumulated tool-call deltas into ToolCall events.
IDs are regenerated (mirroring the Responses handler) so the rest of the
pipeline pairs each call_id with its function_call_output consistently.
"""
for index in sorted(tool_accum):
entry = tool_accum[index]
if not entry["name"]:
continue
yield ToolCall(
item=ResponseFunctionToolCall(
type="function_call",
name=entry["name"],
arguments=entry["args"] or "{}",
call_id=_generate_id("call"),
id=_generate_id("fc"),
status="completed",
)
)
def on_session_end(self) -> None:
logger.debug("Chat Completions API language model session state reset")

181
src/speech_to_speech/LLM/compaction_prompt.py Normal file
View File

@ -0,0 +1,181 @@
"""Prompt template and factory for the conversation compaction (history summarization) function.
Compaction reduces an unbounded conversation history to a tight user/assistant
summary pair, letting the pipeline continue indefinitely without running out of
context window. The factory :func:`build_compactor` returns a :data:`CompactFn`
compatible with :meth:`~speech_to_speech.LLM.chat.Chat.trim_if_needed`.
:data:`CompactGenerateFn` is the backend-agnostic generation interface:
``(system_prompt: str, user_prompt: str) -> response_text: str``.
Each handler wraps its own model into this interface and passes it to
:func:`build_compactor`.
"""
from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from speech_to_speech.LLM.chat import CompactFn, CompactionResult
logger = logging.getLogger(__name__)
# Callable[[system_prompt, user_prompt], response_text]
CompactGenerateFn = Callable[[str, str], str]
# ---------------------------------------------------------------------------
# Prompts
# ---------------------------------------------------------------------------
COMPACTION_SYSTEM_PROMPT = """\
You are a conversation memory compressor for a real-time voice AI assistant.
Your task: read a multi-turn conversation and produce a dense summary so the
assistant can continue naturally, as if it remembers everything that was said.
Output a single JSON object with exactly two string fields:
"user_summary" — 1–5 sentences capturing what the user has been asking
about, any preferences or constraints they have stated,
and where the conversation stands from their perspective.
"assistant_summary" — 1–5 sentences capturing what the assistant has
explained, decided, or done (including tool calls and
their results), plus any open questions or commitments.
Rules:
- Be information-dense: preserve names, numbers, file paths, error messages, and
other specifics that would be needed to continue the conversation correctly.
- Omit small-talk and filler that carries no forward context.
- Write in third person, past tense
(e.g. "The user asked about…", "The assistant provided…").
- Emit only the JSON object — no markdown, no code fences, no extra keys.\
"""
COMPACTION_USER_TEMPLATE = """\
Summarize the following conversation. Return only the JSON object.
--- CONVERSATION START ---
{conversation}
--- CONVERSATION END ---\
"""
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _render_transcript(snapshot: list[Any]) -> str:
"""Render a ResponseInputParam snapshot as a readable plain-text transcript."""
lines: list[str] = []
for item in snapshot:
if not isinstance(item, dict):
continue
item_type = item.get("type", "message")
role: str = item.get("role", "")
if role == "system":
continue
if item_type == "function_call":
name = item.get("name", "")
args = item.get("arguments", "")
lines.append(f"[Tool call: {name}({args})]")
continue
if item_type == "function_call_output":
out = item.get("output", "")
lines.append(f"[Tool result: {out}]")
continue
# Regular user / assistant message
content = item.get("content", "")
if isinstance(content, list):
text = " ".join(
c.get("text", "")
for c in content
if isinstance(c, dict) and c.get("type") in ("input_text", "output_text")
).strip()
elif isinstance(content, str):
text = content.strip()
else:
continue
if text:
label = role.capitalize() if role else "Unknown"
lines.append(f"{label}: {text}")
return "\n\n".join(lines)
_JSON_BLOCK_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
def _extract_json(text: str) -> dict[str, Any]:
"""Extract the first JSON object from *text*, stripping markdown code fences."""
try:
return json.loads(text)
except json.JSONDecodeError:
pass
m = _JSON_BLOCK_RE.search(text)
if m:
return json.loads(m.group(1))
start = text.find("{")
end = text.rfind("}")
if start != -1 and end > start:
return json.loads(text[start : end + 1])
raise ValueError(f"No JSON object found in compaction response: {text!r}")
# ---------------------------------------------------------------------------
# Factory
# ---------------------------------------------------------------------------
def build_compactor(generate_fn: CompactGenerateFn) -> CompactFn:
"""Return a :data:`CompactFn` that summarizes history using *generate_fn*.
*generate_fn* is the only model-specific dependency: it receives
``(system_prompt, user_prompt)`` and returns the model's text response.
Both :class:`~speech_to_speech.LLM.openai_api_language_model.OpenApiModelHandler`
and :class:`~speech_to_speech.LLM.language_model.BaseLanguageModelHandler`
subclasses expose a ``_build_compaction_generate_fn()`` method that wraps
their respective backend into this interface.
The returned callable is safe to call from a background thread.
Args:
generate_fn: Backend-agnostic text generation callable.
Returns:
A callable ``(snapshot: ResponseInputParam) -> CompactionResult``.
"""
def compact(snapshot: list[Any]) -> CompactionResult:
transcript = _render_transcript(snapshot)
if not transcript.strip():
logger.warning("Compaction called with an empty transcript; returning empty summaries")
return CompactionResult(user_summary="", assistant_summary="")
user_content = COMPACTION_USER_TEMPLATE.format(conversation=transcript)
raw_text = generate_fn(COMPACTION_SYSTEM_PROMPT, user_content)
data = _extract_json(raw_text)
user_summary = str(data.get("user_summary", "")).strip()
assistant_summary = str(data.get("assistant_summary", "")).strip()
if not user_summary or not assistant_summary:
raise ValueError(f"Compaction response missing required fields. Got: {data!r}")
logger.debug(
"Compaction complete. user=%d chars assistant=%d chars",
len(user_summary),
len(assistant_summary),
)
return CompactionResult(user_summary=user_summary, assistant_summary=assistant_summary)
return compact

1011
src/speech_to_speech/LLM/language_model.py Normal file
View File

File diff suppressed because it is too large Load Diff

148
src/speech_to_speech/LLM/lm_output_processor.py Normal file
View File

@ -0,0 +1,148 @@
"""
LLM Output Processor
Intercepts LLM output to:
1. Extract tool calls and send them via text_output_queue
2. Forward clean text to TTS pipeline
"""
from __future__ import annotations
import logging
from collections.abc import Iterator
from queue import Queue
from speech_to_speech.baseHandler import BaseHandler
from speech_to_speech.pipeline.events import AssistantTextEvent, ResponseFailedEvent, TokenUsageEvent
from speech_to_speech.pipeline.handler_types import LLMOut, TTSIn
from speech_to_speech.pipeline.messages import EndOfResponse, LLMResponseChunk, TokenUsage, TTSInput
from speech_to_speech.pipeline.queue_types import TextEventItem
from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
from speech_to_speech.utils.utils import response_wants_audio
logger = logging.getLogger(__name__)
class LMOutputProcessor(BaseHandler[LLMOut, TTSIn]):
"""
Processes LLM output to extract tool calls and forward clean text to TTS.
Input: :class:`LLMResponseChunk`, :class:`TokenUsage`, or :class:`EndOfResponse` from LLM
Output: :class:`TTSInput` or :class:`EndOfResponse` to TTS
Side effect: Sends :class:`AssistantTextEvent` / :class:`TokenUsageEvent` to text_output_queue
"""
def setup(
self,
text_output_queue: Queue[TextEventItem] | None = None,
speculative_turns: SpeculativeTurnTracker | None = None,
) -> None:
"""
Initialize the processor.
Args:
text_output_queue: Queue to send text messages and tool calls
"""
self.text_output_queue = text_output_queue
self.speculative_turns = speculative_turns
def _turn_output_allowed(self, turn_id: str | None, turn_revision: int | None) -> bool:
if self.speculative_turns is None:
return True
return self.speculative_turns.is_latest_after_reopen_grace(turn_id, turn_revision)
def process(self, lm_output: LLMOut) -> Iterator[TTSIn]:
"""
Process LLM output: send text/tools to WebSocket, forward clean text to TTS.
Yields:
:class:`TTSInput` or :class:`EndOfResponse` for TTS
"""
if isinstance(lm_output, TokenUsage):
if not self._turn_output_allowed(
lm_output.turn_id,
lm_output.turn_revision,
):
logger.debug(
"Dropping stale token usage for turn=%s rev=%s", lm_output.turn_id, lm_output.turn_revision
)
return
if self.text_output_queue is not None:
self.text_output_queue.put(
TokenUsageEvent(
input_tokens=lm_output.input_tokens or 0,
output_tokens=lm_output.output_tokens or 0,
turn_id=lm_output.turn_id,
turn_revision=lm_output.turn_revision,
)
)
return
if isinstance(lm_output, EndOfResponse):
if not self._turn_output_allowed(
lm_output.turn_id,
lm_output.turn_revision,
):
logger.debug(
"Dropping stale end-of-response for turn=%s rev=%s",
lm_output.turn_id,
lm_output.turn_revision,
)
return
# A failed generation (e.g. invalid out-of-band input) closes the response as
# "failed" via the text side-channel, then falls through to emit the normal
# EndOfResponse so the audio path still re-enables listening / releases the slot.
if lm_output.error and self.text_output_queue is not None:
self.text_output_queue.put(
ResponseFailedEvent(
message=lm_output.error,
turn_id=lm_output.turn_id,
turn_revision=lm_output.turn_revision,
)
)
yield EndOfResponse(
turn_id=lm_output.turn_id,
turn_revision=lm_output.turn_revision,
cancel_generation=lm_output.cancel_generation,
)
return
if not isinstance(lm_output, LLMResponseChunk):
logger.warning("LMOutputProcessor received unexpected type: %s", type(lm_output))
return
if not self._turn_output_allowed(
lm_output.turn_id,
lm_output.turn_revision,
):
logger.debug("Dropping stale LLM chunk for turn=%s rev=%s", lm_output.turn_id, lm_output.turn_revision)
return
logger.debug(f"LM processor: text='{lm_output.text}', tools={lm_output.tools}")
if self.text_output_queue is not None:
event = AssistantTextEvent(
text=lm_output.text,
turn_id=lm_output.turn_id,
turn_revision=lm_output.turn_revision,
cancel_generation=lm_output.cancel_generation,
)
if lm_output.tools:
event.tools = lm_output.tools
logger.info(f"Sending to clients: text='{lm_output.text}', tools={[t.name for t in lm_output.tools]}")
else:
logger.debug(f"Sending to clients: text='{lm_output.text}' (no tools)")
self.text_output_queue.put(event)
if lm_output.text and response_wants_audio(lm_output.response):
logger.debug(f"Forwarding to TTS: '{lm_output.text}'")
yield TTSInput(
text=lm_output.text,
language_code=lm_output.language_code,
runtime_config=lm_output.runtime_config,
response=lm_output.response,
turn_id=lm_output.turn_id,
turn_revision=lm_output.turn_revision,
speech_stopped_at_s=lm_output.speech_stopped_at_s,
cancel_generation=lm_output.cancel_generation,
)

149
src/speech_to_speech/LLM/responses_api_language_model.py Normal file
View File

@ -0,0 +1,149 @@
from __future__ import annotations
import logging
import time
from collections.abc import Iterator
from typing import Any
from openai import Stream
from openai.types.realtime.realtime_conversation_item_assistant_message import (
Content as AssistantContent,
)
from openai.types.responses import (
ResponseCompletedEvent,
ResponseFunctionToolCall,
ResponseOutputItemDoneEvent,
ResponseOutputMessage,
ResponseTextDeltaEvent,
)
from speech_to_speech.LLM.base_openai_compatible_language_model import (
WARMUP_MAX_RETRIES,
AssistantMessage,
BaseOpenAICompatibleHandler,
ProviderEvent,
TextDelta,
ToolCall,
Usage,
)
from speech_to_speech.LLM.chat import Chat
from speech_to_speech.LLM.compaction_prompt import CompactGenerateFn
from speech_to_speech.utils.utils import _generate_id
logger = logging.getLogger(__name__)
class ResponsesApiModelHandler(BaseOpenAICompatibleHandler):
"""LLM handler that talks to an OpenAI ``/v1/responses`` server."""
def warmup(self) -> None:
logger.info(f"Warming up {self.__class__.__name__}")
start = time.time()
self.client.with_options(max_retries=WARMUP_MAX_RETRIES).responses.create(
model=self.model_name,
input=[
{
"type": "message",
"role": "system",
"content": [{"type": "input_text", "text": "You are a helpful assistant"}],
},
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]},
],
timeout=self.request_timeout,
)
end = time.time()
logger.info(f"{self.__class__.__name__}: warmed up! time: {(end - start):.3f} s")
def _build_compaction_generate_fn(self) -> CompactGenerateFn:
"""Return a generate fn that calls the Responses API for compaction."""
client = self.client
model_name = self.model_name
timeout = self.request_timeout
def generate(system: str, user: str) -> str:
response = client.responses.create(
model=model_name,
input=[
{
"type": "message",
"role": "system",
"content": [{"type": "input_text", "text": system}],
},
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user}],
},
],
timeout=timeout,
)
return response.output_text
return generate
# ── base hooks ──────────────────────────────────────────────────────────--
def _serialize(self, active_chat: Chat) -> Any:
return active_chat.to_responses_api_chat()
def _build_optional_kwargs(self, req_tools: Any, req_tool_choice: Any) -> dict[str, Any]:
optional_kwargs: dict[str, Any] = {}
if req_tools is not None:
optional_kwargs["tools"] = req_tools
if req_tool_choice is not None:
optional_kwargs["tool_choice"] = req_tool_choice
return optional_kwargs
def _request(self, api_input: Any, optional_kwargs: dict[str, Any]) -> Any:
return self.client.responses.create(
model=self.model_name,
input=api_input,
stream=self.stream,
extra_body=self._extra_body,
timeout=self.request_timeout,
**optional_kwargs,
)
@staticmethod
def _assistant_content(content: Any) -> list[AssistantContent]:
return [
AssistantContent(type="output_text", text=c.text if c.type == "output_text" else c.refusal) for c in content
]
def _iter_stream_events(self, api_response: Stream) -> Iterator[ProviderEvent]:
for raw_event in api_response:
if isinstance(raw_event, ResponseTextDeltaEvent):
yield TextDelta(text=raw_event.delta)
elif isinstance(raw_event, ResponseOutputItemDoneEvent):
item = raw_event.item
if isinstance(item, ResponseFunctionToolCall):
item.call_id = _generate_id("call")
item.id = _generate_id("fc")
yield ToolCall(item=item)
elif isinstance(item, ResponseOutputMessage):
yield AssistantMessage(content=self._assistant_content(item.content))
elif isinstance(raw_event, ResponseCompletedEvent):
usage = getattr(raw_event.response, "usage", None)
if usage:
yield Usage(input_tokens=usage.input_tokens or 0, output_tokens=usage.output_tokens or 0)
def _iter_response_events(self, api_response: Any) -> Iterator[ProviderEvent]:
usage = api_response.usage
if usage:
yield Usage(input_tokens=usage.input_tokens or 0, output_tokens=usage.output_tokens or 0)
for message in api_response.output:
if isinstance(message, ResponseFunctionToolCall):
message.call_id = _generate_id("call")
message.id = _generate_id("fc")
yield ToolCall(item=message)
elif isinstance(message, ResponseOutputMessage):
yield AssistantMessage(content=self._assistant_content(message.content))
# Text-only keeps every character; the base applies remove_unspeechable
# for audio. Only output_text parts are spoken (refusals are stored).
raw = "".join(c.text for c in message.content if c.type == "output_text")
yield TextDelta(text=raw)
else:
logger.warning(f"Not supported message type: {message.type}")
def on_session_end(self) -> None:
logger.debug("OpenAI API language model session state reset")

44
src/speech_to_speech/LLM/text_prompt.py Normal file
View File

@ -0,0 +1,44 @@
"""Text-channel system prompt: lead + session prompt + tail (strongest constraints last)."""
TEXT_SYSTEM_PROMPT_LEAD = """\
You are a helpful assistant in a text conversation.
"""
TEXT_SYSTEM_PROMPT_TAIL = """\
## Text Rules
- Write clearly and directly. Match length to the request: concise for simple questions, fuller when the task genuinely needs it.
- Use markdown when it helps (lists, code blocks, tables, emphasis); don't over-format simple answers.
- This is a written channel: no spoken-style filler and no action/emote text like *laughs*.
- Use tools when they help fulfill the request. No preamble sentence is required before a tool call.
- For slow or external tools, just call the tool and use the result; you don't need to announce it.
- If unsure whether a tool is needed, just answer directly.
"""
# Skeleton for the assembled system message (placeholders filled in build_text_system_prompt).
_TEXT_SYSTEM_PROMPT_FULL = """\
{lead}
Session Prompt:
{session_prompt}{optional_tools}
{tail}
"""
def build_text_system_prompt(session_prompt: str, *, tool_section: str = "") -> str:
"""Context → session prompt → optional tool block → strongest text rules last."""
tools = tool_section.strip()
optional_tools = f"\n\n{tools}" if tools else ""
return _TEXT_SYSTEM_PROMPT_FULL.format(
lead=TEXT_SYSTEM_PROMPT_LEAD.rstrip(),
session_prompt=session_prompt.strip(),
optional_tools=optional_tools,
tail=TEXT_SYSTEM_PROMPT_TAIL.rstrip(),
)
# Full text instructions without a separate session block (legacy / rare direct use).
TEXT_SYSTEM_PROMPT = "{lead}\n\n{tail}".format(
lead=TEXT_SYSTEM_PROMPT_LEAD.rstrip(),
tail=TEXT_SYSTEM_PROMPT_TAIL.rstrip(),
)

321
src/speech_to_speech/LLM/tool_call/function_call.py Normal file
View File

@ -0,0 +1,321 @@
#!/usr/bin/env python3
"""
Function parser for extracting function names, parameter names, and values from string function calls.
Uses Python's ``tokenize`` and ``ast`` modules so that nested parentheses,
strings containing ')' characters, tuples, dicts, etc. are handled correctly.
"""
import ast
import io
import json
import logging
import re
import tokenize
from collections import OrderedDict
from typing import Any, Dict, List, Tuple
from openai.types.responses import ResponseFunctionToolCall
from pydantic import BaseModel
from speech_to_speech.LLM.tool_call.function_tool import FunctionTool
from speech_to_speech.utils.utils import _generate_id
logger = logging.getLogger(__name__)
_POSITIONAL_RE = re.compile(r"^__arg_\d+__$")
_LENIENT_CALL_RE = re.compile(
r"\b[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*\s*"
r"\((?:[^()\"']+|\"(?:\\.|[^\"])*\"|'(?:\\.|[^'])*')*\)"
)
# ── AST / tokenize helpers ───────────────────────────────────────────
def _split_top_level_calls(source: str) -> List[str]:
"""Split *source* into individual ``name(...)`` expression strings.
Uses the tokenizer to walk tokens and track parenthesis depth so that
nested parens, strings with ')' chars, etc. are handled correctly.
"""
tokens = list(tokenize.generate_tokens(io.StringIO(source).readline))
calls: List[str] = []
i = 0
while i < len(tokens):
tok = tokens[i]
if tok.type != tokenize.NAME:
i += 1
continue
start = i
j = i + 1
# Walk past dotted attribute access (e.g. ``mobile.click``)
while j + 1 < len(tokens) and tokens[j].string == "." and tokens[j + 1].type == tokenize.NAME:
j += 2
if j >= len(tokens) or tokens[j].string != "(":
i += 1
continue
# Track balanced parens
depth = 0
end = None
k = j
while k < len(tokens):
t = tokens[k]
if t.type == tokenize.OP and t.string == "(":
depth += 1
elif t.type == tokenize.OP and t.string == ")":
depth -= 1
if depth == 0:
end = k
break
k += 1
if end is None:
i += 1
continue
calls.append(tokenize.untokenize(tokens[start : end + 1]).strip())
i = end + 1
return calls
def _split_simple_calls_with_regex(source: str) -> List[str]:
"""Extract complete simple ``name(args)`` spans from malformed model output.
This fallback can recover well-formed siblings before a tokenizer error,
but not the incomplete call that caused the tokenizer error.
"""
return [match.group(0).strip() for match in _LENIENT_CALL_RE.finditer(source)]
def _parse_function_exprs(
expressions: List[str],
pattern_to_match: list[str],
*,
skip_invalid: bool = False,
) -> List["FunctionToolCall"]:
results: List[FunctionToolCall] = []
for expr in expressions:
try:
call = _parse_call_expr(expr)
except Exception:
if skip_invalid:
continue
raise
if pattern_to_match and all(pattern not in call.function_name for pattern in pattern_to_match):
continue
results.append(call)
return results
def _extract_function_name(node: ast.expr) -> str:
"""Return the dotted function name from a Call node's ``func`` attribute."""
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
base = _extract_function_name(node.value)
return f"{base}.{node.attr}" if base else node.attr
raise ValueError(f"Unsupported function target: {ast.dump(node)}")
def _literal_from_ast(node: ast.AST) -> Any:
"""Convert an AST node to a Python literal value."""
if isinstance(node, ast.Constant):
return node.value
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.List):
return [_literal_from_ast(elt) for elt in node.elts]
if isinstance(node, ast.Tuple):
return [_literal_from_ast(elt) for elt in node.elts]
if isinstance(node, ast.Dict):
return {
_literal_from_ast(key): _literal_from_ast(value)
for key, value in zip(node.keys, node.values)
if key is not None
}
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
value = _literal_from_ast(node.operand)
if not isinstance(value, (int, float)):
raise ValueError(f"Unsupported unary literal: {ast.dump(node)}")
return -value if isinstance(node.op, ast.USub) else value
raise ValueError(f"Unsupported literal: {ast.dump(node)}")
def _parse_call_expr(expr: str) -> "FunctionToolCall":
"""Parse a single ``name(args...)`` expression string into a FunctionToolCall."""
parsed = ast.parse(expr, mode="eval").body
if not isinstance(parsed, ast.Call):
raise ValueError(f"Expression is not a function call: {expr!r}")
parameters: "OrderedDict[str, Any]" = OrderedDict()
for idx, arg in enumerate(parsed.args):
parameters[f"__arg_{idx}__"] = _literal_from_ast(arg)
for kw in parsed.keywords:
if kw.arg is None:
raise ValueError("**kwargs are not supported")
parameters[kw.arg] = _literal_from_ast(kw.value)
return FunctionToolCall(
function_name=_extract_function_name(parsed.func),
parameters=parameters,
original_string=expr,
)
# ── Data model ───────────────────────────────────────────────────────
class FunctionToolCall(BaseModel):
"""Represents a parsed function call with its parameters."""
function_name: str
parameters: Dict[str, Any]
original_string: str
description: str = ""
def to_realtime_function_tool_call(
self,
function_tools: list[FunctionTool] | None = None,
) -> ResponseFunctionToolCall:
positional = {k for k in self.parameters if _POSITIONAL_RE.match(k)}
if positional:
logger.warning(
"Dropping positional arguments for '%s': %s",
self.function_name,
positional,
)
arguments = {k: v for k, v in self.parameters.items() if not _POSITIONAL_RE.match(k)}
if function_tools is not None:
tool = next(
(t for t in function_tools if t.name == self.function_name),
None,
)
if tool is None:
available = [t.name for t in function_tools]
raise ValueError(f"Function '{self.function_name}' not found in available tools: {available}")
schema = tool.parameters if isinstance(tool.parameters, dict) else {}
properties = schema.get("properties", {})
required = set(schema.get("required", []))
undeclared = {k for k in arguments if k not in properties}
if undeclared:
logger.warning(
"Dropping undeclared parameters for '%s': %s",
self.function_name,
undeclared,
)
arguments = {k: v for k, v in arguments.items() if k in properties}
missing = required - set(arguments.keys())
if missing:
raise ValueError(f"Missing required parameters for '{self.function_name}': {missing}")
return ResponseFunctionToolCall(
name=self.function_name,
arguments=json.dumps(arguments),
call_id=_generate_id("call"),
type="function_call",
id=_generate_id("fc"),
status="in_progress",
)
# ── Public API ───────────────────────────────────────────────────────
def parse_function_call(function_string: str, pattern_to_match: list[str] = []) -> List[FunctionToolCall]:
"""Parse a function call string and extract all function calls found.
Args:
function_string: String representation of function calls.
pattern_to_match: If non-empty, only calls whose function name
contains at least one of these substrings are returned.
Returns:
List of FunctionToolCall objects with parsed information.
"""
function_string = function_string.strip()
if not function_string:
return []
try:
expressions = _split_top_level_calls(function_string)
except tokenize.TokenError:
return _parse_function_exprs(
_split_simple_calls_with_regex(function_string),
pattern_to_match,
skip_invalid=True,
)
return _parse_function_exprs(expressions, pattern_to_match)
def parse_multiple_functions(function_strings: List[str]) -> List[FunctionToolCall]:
"""Parse multiple function call strings.
Args:
function_strings: List of function call strings.
Returns:
List of FunctionToolCall objects.
"""
results: List[FunctionToolCall] = []
for func_str in function_strings:
try:
results.extend(parse_function_call(func_str))
except Exception:
continue
return results
def extract_function_calls_from_text(text: str, block_regex: str = ".*") -> Tuple[str, List[FunctionToolCall]]:
"""Extract function calls from delimited code blocks inside *text*.
The LLM is prompted to wrap tool calls inside code-block delimiters
(e.g. ``<code>func(x=1)</code>``). This function finds those blocks,
parses the function calls within them, and returns the remaining text
(with blocks stripped) alongside the parsed calls.
Args:
text: Full model output potentially containing code blocks.
block_regex: Regex matching the code-block delimiters **and** their
content (e.g. ``r"<code>.*?</code>"``). Only text **inside**
matched blocks is scanned for function calls.
Returns:
``(outside_text, function_calls)`` -- the text with blocks stripped
and the parsed function calls found inside the blocks.
"""
if not block_regex:
return text, []
matches = list(re.finditer(block_regex, text, flags=re.DOTALL))
if not matches:
return text, []
outside = re.sub(block_regex, "", text, flags=re.DOTALL)
inside = " ".join(match.group(0) for match in matches).strip()
if not inside:
return outside, []
try:
return outside, parse_function_call(inside)
except Exception:
return outside, []

34
src/speech_to_speech/LLM/tool_call/function_tool.py Normal file
View File

@ -0,0 +1,34 @@
import textwrap
from openai.types.realtime import RealtimeFunctionTool
from speech_to_speech.LLM.tool_call.signature_from_schema import signature_from_schema
class FunctionTool(RealtimeFunctionTool):
def to_code_prompt(self, include_args_doc: bool = True) -> str:
"""Generate a code-style prompt string for this function tool.
Args:
include_args_doc: If True, include argument descriptions in the docstring.
⚠️ This lets the model see each argument's purpose but significantly increases
token usage (e.g. 906 tokens without vs 3434 with for the default Reachy Mini
tool profile). Enable depending on the model's capabilities and context limit.
"""
signature = signature_from_schema(self.parameters)
tool_doc = self.description or ""
if isinstance(self.parameters, dict) and include_args_doc:
props = self.parameters.get("properties", {})
if props:
arg_lines = []
for arg_name, arg_schema in props.items():
desc = arg_schema.get("description", "") if isinstance(arg_schema, dict) else ""
arg_lines.append(f"{arg_name}: {desc}")
args_doc = f"Args:\n{textwrap.indent(chr(10).join(arg_lines), ' ')}"
tool_doc += f"\n\n{args_doc}"
tool_doc = f'"""{tool_doc}\n"""'
return f"def {self.name}{signature}:\n{textwrap.indent(tool_doc, ' ')}"

108
src/speech_to_speech/LLM/tool_call/signature_from_schema.py Normal file
View File

@ -0,0 +1,108 @@
import inspect
from typing import Any, Literal, Union
JSON_TYPE_TO_PYTHON_TYPE = {
"string": str,
"number": float,
"boolean": bool,
"integer": int,
"object": dict,
"array": list,
"null": type(None),
}
def _dedupe_types(types: list[Any]) -> list[Any]:
seen = []
for t in types:
if t not in seen:
seen.append(t)
return seen
def _annotation_from_spec(spec: dict[str, Any]) -> Any:
if not spec or not isinstance(spec, dict):
return Any
# const → Literal[value]
if "const" in spec:
return Literal[spec["const"]]
# enum → Literal[val1, val2, ...]
if "enum" in spec:
values = spec["enum"]
if not values:
return Any
return Literal[tuple(values)]
# anyOf / oneOf → Union[Type1, Type2, ...]
for key in ("anyOf", "oneOf"):
if key in spec:
variants = [_annotation_from_spec(s) for s in spec[key]]
unique = _dedupe_types(variants)
if len(unique) == 0:
return Any
if len(unique) == 1:
return unique[0]
return Union[tuple(unique)]
# allOf → merge sub-schemas then resolve
if "allOf" in spec:
merged = {}
for sub in spec["allOf"]:
merged.update(sub)
return _annotation_from_spec(merged)
json_type = spec.get("type")
if json_type is None:
return Any
# type as list, e.g. ["string", "null"] → Union[str, None] (Optional)
if isinstance(json_type, list):
types = [JSON_TYPE_TO_PYTHON_TYPE.get(t, Any) for t in json_type]
unique = _dedupe_types(types)
if len(unique) == 0:
return Any
if len(unique) == 1:
return unique[0]
return Union[tuple(unique)]
# array with items → list[ItemType]
if json_type == "array" and "items" in spec:
item_type = _annotation_from_spec(spec["items"])
return list[item_type] # type: ignore[valid-type]
return JSON_TYPE_TO_PYTHON_TYPE.get(json_type, Any)
def signature_from_schema(schema: object | None) -> inspect.Signature:
if not schema or not isinstance(schema, dict):
return inspect.Signature()
props = schema.get("properties", {})
required = set(schema.get("required", []))
params = []
for name, spec in props.items():
annotation = _annotation_from_spec(spec)
has_schema_default = "default" in spec if isinstance(spec, dict) else False
if name in required and not has_schema_default:
default = inspect.Parameter.empty
elif has_schema_default:
default = spec["default"]
else:
default = None
params.append(
inspect.Parameter(
name,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
default=default,
annotation=annotation,
)
)
return inspect.Signature(params)

111
src/speech_to_speech/LLM/tool_call/tool_prompt.py Normal file
View File

@ -0,0 +1,111 @@
"""
Optional system-prompt builder that instructs a local LLM to output tool calls
inside delimited code blocks (e.g. ``<code>func(arg='val')</code>``).
The prompt is rendered from a Jinja2 template and relies on
``FunctionTool.to_code_prompt()`` to expose each tool's Python-style signature.
"""
import re
from jinja2 import Template
from speech_to_speech.LLM.tool_call.function_tool import FunctionTool
# ---------------------------------------------------------------------------
# Default delimiters
# ---------------------------------------------------------------------------
ENTER_CODE = "<code>"
END_CODE = "</code>"
# ---------------------------------------------------------------------------
# Jinja2 template
# ---------------------------------------------------------------------------
# ``enter_code`` / ``end_code`` are the block delimiters the model must wrap
# every tool call in. ``tools`` is a list of FunctionTool instances whose
# ``.to_code_prompt()`` is called inside the template.
# ---------------------------------------------------------------------------
TOOL_PROMPT_TEMPLATE = Template(
"""\
Available tools:
{% for tool in tools %}
{{ tool.to_code_prompt() }}
{% endfor %}
To call a tool, put exactly one named-argument function call inside {{ enter_code }}...{{ end_code }}:
{{ enter_code }}function_name(required_arg='value'){{ end_code }}
Rules:
- You may say one brief natural sentence before the tool call; for slow information tools, briefly say that you will check.
- For expression/background tools, always speak first. For requested expressions, use a short pattern like "Sure, here's my best <emotion>."; otherwise use a fitting empathetic sentence.
- Do not mention tags, functions, or tools. Keep prose outside tags brief, and do not claim tool results before a tool result is available.
- Use named arguments only; quote strings. Omit optional args instead of placeholder values like "random", "none", "", or null.
- Only one tool call may appear in a response.\
""",
keep_trailing_newline=True,
)
# Text-channel variant: same call format and structural rules, without the
# voice-specific "speak first" prose.
TEXT_TOOL_PROMPT_TEMPLATE = Template(
"""\
Available tools:
{% for tool in tools %}
{{ tool.to_code_prompt() }}
{% endfor %}
To call a tool, put exactly one named-argument function call inside {{ enter_code }}...{{ end_code }}:
{{ enter_code }}function_name(required_arg='value'){{ end_code }}
Rules:
- Call a tool directly when it helps fulfill the request; no preamble sentence is required.
- Do not mention tags, functions, or tools in your prose, and do not claim tool results before a tool result is available.
- Use named arguments only; quote strings. Omit optional args instead of placeholder values like "random", "none", "", or null.
- Only one tool call may appear in a response.\
""",
keep_trailing_newline=True,
)
# ---------------------------------------------------------------------------
# Public helpers
# ---------------------------------------------------------------------------
def build_tool_system_prompt(
tools: list[FunctionTool],
enter_code: str = ENTER_CODE,
end_code: str = END_CODE,
*,
text_only: bool = False,
) -> str:
"""Render the tool-calling system-prompt section.
Returns an empty string when *tools* is empty so it can be
unconditionally appended to a base system prompt. When *text_only* is set,
the written-channel variant is used (no voice "speak first" prose).
"""
if not tools:
return ""
template = TEXT_TOOL_PROMPT_TEMPLATE if text_only else TOOL_PROMPT_TEMPLATE
return template.render(
tools=tools,
enter_code=enter_code,
end_code=end_code,
)
def build_block_regex(
enter_code: str = ENTER_CODE,
end_code: str = END_CODE,
) -> str:
"""Build a regex that matches a single code block (non-greedy).
>>> build_block_regex("<code>", "</code>")
'<code>.*?</code>'
"""
return f"{re.escape(enter_code)}.*?{re.escape(end_code)}"

75
src/speech_to_speech/LLM/utils.py Normal file
View File

@ -0,0 +1,75 @@
import base64
import io
import re
from typing import Optional
import requests # type: ignore[import-untyped]
from PIL import Image
SMART_PUNCT_TRANSLATION = str.maketrans(
{
"\u2018": "'",
"\u2019": "'",
"\u201c": '"',
"\u201d": '"',
}
)
SPEECHABLE_PATTERN = re.compile(
r"[^\w\s.,!?;:'\"\-()\/\\@#%&*+=$€£¥₹₽¢\[\]{}<>~`^|…—–\n\r\t]",
flags=re.UNICODE,
)
def remove_unspeechable(text: str) -> str:
"""Keep only speechable characters: letters, digits, punctuation, whitespace.
support unicode characters (english, arabic, chinese, japanese, korean, etc.)
"""
text = text.translate(SMART_PUNCT_TRANSLATION)
return SPEECHABLE_PATTERN.sub("", text)
WHISPER_LANGUAGE_TO_LLM_LANGUAGE = {
"en": "english",
"fr": "french",
"es": "spanish",
"zh": "chinese",
"ja": "japanese",
"ko": "korean",
"hi": "hindi",
"de": "german",
"pt": "portuguese",
"pl": "polish",
"it": "italian",
"nl": "dutch",
}
def resolve_auto_language(language_code: Optional[str]) -> tuple[Optional[str], Optional[str]]:
"""Strip the ``-auto`` suffix and resolve the human-readable language name.
Returns ``(clean_code, language_name)``. ``language_name`` is non-None
when the code (with or without ``-auto``) maps to a known language.
"""
if not language_code:
return language_code, None
if language_code.endswith("-auto"):
language_code = language_code[:-5]
if language_code not in WHISPER_LANGUAGE_TO_LLM_LANGUAGE:
return language_code, None
return language_code, WHISPER_LANGUAGE_TO_LLM_LANGUAGE.get(language_code)
def image_url_to_pil(image_url: str) -> Image.Image:
"""Convert an image URL or base64 data URI to a PIL Image.
Accepts:
- 'data:image/...;base64,<b64>' data URIs
- 'https://...`` or ``http://...' URLs (fetched with a 10s timeout)
"""
if image_url.startswith("data:"):
_, b64_data = image_url.split(",", 1)
return Image.open(io.BytesIO(base64.b64decode(b64_data)))
resp = requests.get(image_url, timeout=10)
resp.raise_for_status()
return Image.open(io.BytesIO(resp.content))

48
src/speech_to_speech/LLM/voice_prompt.py Normal file
View File

@ -0,0 +1,48 @@
"""Voice-channel system prompt: lead + session prompt + tail (strongest constraints last)."""
VOICE_SYSTEM_PROMPT_LEAD = """\
You are in a spoken conversation. The user speaks and hears you.
The session prompt defines persona, facts, goals, and tool descriptions. These channel rules only control spoken output and tool-use behavior.
"""
VOICE_SYSTEM_PROMPT_TAIL = """\
## Voice Rules
- Keep replies brief by default: usually one spoken sentence, two if needed. Go longer only when asked.
- Speak naturally. No markdown, bullets, headings, visual formatting, or action/emote text like *laughs*.
- Treat transcripts as noisy. Correct likely mishearings only if asked or meaning depends on it.
- Speech is the default. Use at most one tool when it helps fulfill the request or clearly fits the moment.
- Before a tool call, use a brief natural utterance unless the user asked for silence or tool-only output. For slow information tools, briefly say that you will check.
- For expression/background tools, speak first. If asked to show an expression, use a short pattern like "Sure, here's my best <emotion>." Otherwise use a fitting empathetic sentence. Never mention tools.
- After completed expression/background/physical-action tools, do not add a second spoken comment unless the result has user-facing information.
- Use motion, dance, emotion, and similar tools sparingly when they add empathy, celebration, playfulness, or a requested physical action.
- If unsure whether a tool is needed, just speak.
"""
# Skeleton for the assembled system message (placeholders filled in build_voice_system_prompt).
_VOICE_SYSTEM_PROMPT_FULL = """\
{lead}
Session Prompt:
{session_prompt}{optional_tools}
{tail}
"""
def build_voice_system_prompt(session_prompt: str, *, tool_section: str = "") -> str:
"""Context → session prompt → optional tool block → strongest voice rules last."""
tools = tool_section.strip()
optional_tools = f"\n\n{tools}" if tools else ""
return _VOICE_SYSTEM_PROMPT_FULL.format(
lead=VOICE_SYSTEM_PROMPT_LEAD.rstrip(),
session_prompt=session_prompt.strip(),
optional_tools=optional_tools,
tail=VOICE_SYSTEM_PROMPT_TAIL.rstrip(),
)
# Full voice instructions without a separate session block (legacy / rare direct use).
VOICE_SYSTEM_PROMPT = "{lead}\n\n{tail}".format(
lead=VOICE_SYSTEM_PROMPT_LEAD.rstrip(),
tail=VOICE_SYSTEM_PROMPT_TAIL.rstrip(),
)

15
src/speech_to_speech/RAG/__init__.py Normal file
View File

@ -0,0 +1,15 @@
from speech_to_speech.RAG.retriever import (
Chunk,
RAGRetriever,
SearchResult,
get_global_rag,
set_global_rag,
)
__all__ = [
"Chunk",
"RAGRetriever",
"SearchResult",
"get_global_rag",
"set_global_rag",
]

1149
src/speech_to_speech/RAG/retriever.py Normal file
View File

File diff suppressed because it is too large Load Diff

319
src/speech_to_speech/RAG/router.py Normal file
View File

@ -0,0 +1,319 @@
"""Router HTTP FastAPI per amministrazione RAG a runtime.
Montato sullo stesso FastAPI che serve il websocket /v1/realtime (vedi
:func:`speech_to_speech.api.openai_realtime.websocket_router.create_app`).
Tutti gli endpoint sono JSON e si appoggiano all'istanza globale
``get_global_rag()``; se RAG non è attivo rispondono 503.
"""
from __future__ import annotations
import logging
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field
from speech_to_speech.RAG.retriever import get_global_rag
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/v1/rag", tags=["RAG"])
# ── Pydantic schemi request / response ────────────────────────────────────────
class AddDocumentRequest(BaseModel):
text: str = Field(..., description="Testo completo documento da aggiungere.")
source: str = Field(
...,
description="Nome univoco sorgente (es. 'crm/cliente_123_nota_20260826').",
min_length=1,
)
metadata: Optional[dict[str, Any]] = Field(
default=None,
description="Metadata arbitrari (non usati dal retrieval, ma salvati).",
)
persist: bool = Field(
default=True,
description="Se True, salva in kb/_dynamic.jsonl per il prossimo avvio.",
)
dynamic: bool = Field(
default=True,
description="Se True, marca come chunk dinamico (rimovibile via API).",
)
class AddChunksRequest(BaseModel):
items: list[dict[str, Any]] = Field(
...,
description="Lista di chunk: ognuno con chiavi {text, source?, chunk_index?, metadata?}.",
min_length=1,
)
persist: bool = True
dynamic: bool = True
class RemoveRequest(BaseModel):
source_prefix: str = Field(
...,
description="Sorgente o prefisso da rimuovere (vedi `exact`).",
min_length=1,
)
exact: bool = Field(
default=False,
description="False = prefisso; True = match esatto su source.",
)
persist: bool = True
class SearchRequest(BaseModel):
query: str = Field(..., description="Testo da cercare.", min_length=1)
top_k: Optional[int] = Field(default=None, ge=1, le=50)
threshold: Optional[float] = Field(default=None, ge=0.0, le=1.0)
class ReloadRequest(BaseModel):
force_rebuild: bool = Field(
default=True,
description="Ricostruzione totale da file md/txt/jsonl in kb/.",
)
class ListSourcesResponse(BaseModel):
count: int
items: list[dict[str, Any]]
class ListChunksRequest(BaseModel):
source_prefix: Optional[str] = Field(default=None, description="Filtro prefisso sorgente.")
source_exact: Optional[str] = Field(default=None, description="Filtro esatto sorgente (vince se entrambi sono settati).")
query: Optional[str] = Field(default=None, description="Se fornito, ordina per rilevanza embedding coseno.")
min_score: Optional[float] = Field(default=None, ge=0.0, le=1.0, description="Solo con `query`: score minimo.")
offset: int = Field(default=0, ge=0)
limit: int = Field(default=100, ge=1, le=1000)
include_text: bool = Field(default=True)
include_embedding: bool = Field(default=False, description="Attenzione: output grosso.")
class UpdateChunkRequest(BaseModel):
"""Identificazione + modifiche. Uno e un solo dei gruppi deve essere valorizzato:
- ``index``
- ``source`` + ``chunk_index`` (opzionale)
"""
index: Optional[int] = Field(default=None, ge=0, description="Posizione nell'array dei chunk (da list_chunks).")
source: Optional[str] = Field(default=None, description="Sorgente. Se unico chunk basta questo.")
chunk_index: Optional[int] = Field(default=None, description="Indice chunk interno alla source.")
new_text: Optional[str] = Field(default=None, description="Se non vuoto/None → re-embedding del testo.")
new_metadata: Optional[dict[str, Any]] = Field(default=None, description="Sostituisce TUTTI i metadata (flag _dynamic viene preservato).")
new_source: Optional[str] = Field(default=None, description="Cambia nome sorgente.")
persist: bool = True
class UpsertDocumentRequest(BaseModel):
"""Update atomico di un intero documento identificato da `source`."""
text: str = Field(..., description="Nuovo testo completo. Stringa vuota = DELETE atomico per questa source.")
source: str = Field(..., min_length=1)
metadata: Optional[dict[str, Any]] = None
persist: bool = True
dynamic: bool = True
# ── Helpers ──────────────────────────────────────────────────────────────────
def _require_rag():
rag = get_global_rag()
if rag is None:
raise HTTPException(
status_code=503,
detail={
"code": "RAG_NOT_ENABLED",
"message": "RAG non attivo: avvia con --rag_enabled o verifica start_pipeline_rag.sh",
},
)
return rag
# ── Endpoints ─────────────────────────────────────────────────────────────────
@router.get("/status")
def rag_status() -> dict[str, Any]:
"""Stato generale dell'istanza RAG (chunk, embeddings, sorgenti, device)."""
return _require_rag().status()
@router.get("/sources")
def rag_list_sources() -> dict[str, Any]:
"""Lista sorgenti uniche nell'indice con il conteggio chunk per ognuna."""
items = _require_rag().list_sources()
return {"count": len(items), "items": items}
@router.get("/chunks")
def rag_list_chunks_get(
source_prefix: Optional[str] = Query(default=None),
source_exact: Optional[str] = Query(default=None),
query: Optional[str] = Query(default=None),
min_score: Optional[float] = Query(default=None, ge=0.0, le=1.0),
offset: int = Query(default=0, ge=0),
limit: int = Query(default=100, ge=1, le=1000),
include_text: bool = Query(default=True),
include_embedding: bool = Query(default=False),
) -> dict[str, Any]:
"""GET friendly (query params): lista chunk con filtri + paginazione."""
rag = _require_rag()
return rag.list_chunks(
source_prefix=source_prefix,
source_exact=source_exact,
query=query,
min_score=min_score,
offset=offset,
limit=limit,
include_text=include_text,
include_embedding=include_embedding,
)
@router.post("/chunks/list")
def rag_list_chunks_post(req: ListChunksRequest) -> dict[str, Any]:
"""Stesso list_chunks ma con body JSON (consigliato per query complesse)."""
rag = _require_rag()
return rag.list_chunks(**req.model_dump())
@router.post("/chunks/update")
def rag_update_chunk(req: UpdateChunkRequest) -> dict[str, Any]:
"""Aggiorna un singolo chunk (testo / metadata / sorgente)."""
if (
req.index is None
and req.source is None
):
raise HTTPException(
status_code=400,
detail={
"code": "IDENTIFIER_REQUIRED",
"message": "Devi specificare almeno `index`, oppure `source` (con chunk_index opzionale).",
},
)
rag = _require_rag()
res = rag.update_chunk(
index=req.index,
source=req.source,
chunk_index=req.chunk_index,
new_text=req.new_text,
new_metadata=req.new_metadata,
new_source=req.new_source,
persist=req.persist,
)
if res.get("updated", 0) == 0:
err = res.get("error", "CHUNK_NOT_FOUND")
msg = res.get("message", "Nessun chunk trovato")
if err == "CHUNK_NOT_FOUND":
status = 404
elif err == "AMBIGUOUS_SOURCE":
status = 409
else:
status = 400
raise HTTPException(status_code=status, detail={"code": err, "message": msg, **{k: v for k, v in res.items() if k not in {"updated", "error", "message"}}})
return res
@router.post("/upsert/document")
def rag_upsert_document(req: UpsertDocumentRequest) -> dict[str, Any]:
"""Upsert atomico di un intero documento (delete source exact + split nuovo testo).
Consigliato per l'update "normale" di un documento (non devi preoccuparti
del chunking). Se mandi ``text: ""`` → DELETE atomico della sorgente.
"""
rag = _require_rag()
return rag.upsert_document(
req.text,
source=req.source,
metadata=req.metadata,
persist=req.persist,
dynamic=req.dynamic,
)
@router.post("/search")
def rag_search(req: SearchRequest) -> dict[str, Any]:
"""Cerca chunk rilevanti — stesso algoritmo usato per l'iniezione nel prompt."""
rag = _require_rag()
results = rag.search(req.query, top_k=req.top_k, threshold=req.threshold)
return {
"query": req.query,
"count": len(results),
"results": [
{
"text": r.chunk.text,
"source": r.chunk.source,
"chunk_index": r.chunk.chunk_index,
"score": r.score,
"metadata": r.chunk.metadata,
}
for r in results
],
}
@router.post("/add/document")
def rag_add_document(req: AddDocumentRequest) -> dict[str, Any]:
"""Splitta un documento in chunk (stesso chunker di build_index) e li aggiunge.
Consigliato quando il tuo payload è un testo libero lungo (es. una nota cliente,
un articolo, una mail).
"""
rag = _require_rag()
return rag.add_document(
req.text,
source=req.source,
metadata=req.metadata,
persist=req.persist,
dynamic=req.dynamic,
)
@router.post("/add/chunks")
def rag_add_chunks(req: AddChunksRequest) -> dict[str, Any]:
"""Aggiungi chunk pre-costruiti senza chunking automatico.
Consigliato quando hai già splittato i dati lato sorgente (es. records DB,
risultati query strutturati).
"""
rag = _require_rag()
return rag.add_chunks(
req.items,
persist=req.persist,
dynamic=req.dynamic,
)
@router.post("/remove")
def rag_remove(req: RemoveRequest) -> dict[str, Any]:
"""Rimuove chunk.
- ``exact=false`` (default): tutti i chunk la cui source INIZIA con ``source_prefix``
- ``exact=true``: solo i chunk la cui source è ESATTAMENTE uguale
"""
rag = _require_rag()
return rag.remove_by_source(
req.source_prefix,
exact=req.exact,
persist=req.persist,
)
@router.post("/reload")
def rag_reload(req: Optional[ReloadRequest] = None) -> dict[str, Any]:
"""Ricostruisce l'indice da zero leggendo i file in kb/.
Equivalente a riavviare con ``--rag_force_rebuild``, ma a caldo.
"""
rag = _require_rag()
fr = req.force_rebuild if req is not None else True
return rag.reload_from_disk(force_rebuild=fr)

163
src/speech_to_speech/STT/README.md Normal file
View File

@ -0,0 +1,163 @@
# STT Summary
This document summarizes the Speech-to-Text (STT) implementations in the `STT/` folder, including language support, language abbreviations, and usage in `s2s_pipeline.py`.
## Available STT Modes (`--stt`)
- `whisper` → `STT/whisper_stt_handler.py`
- `whisper-mlx` → `STT/lightning_whisper_mlx_handler.py`
- `mlx-audio-whisper` → `STT/mlx_audio_whisper_handler.py`
- `faster-whisper` → `STT/faster_whisper_handler.py`
- `parakeet-tdt` → `STT/parakeet_tdt_handler.py`
- `paraformer` → `STT/paraformer_handler.py`
## Language Support by Handler
### 1) Whisper (`--stt whisper`)
- Handler: `WhisperSTTHandler`
- Language input flag: `--language` (from shared Whisper args)
- Supports fixed language (e.g. `en`) or `auto`
- Internal supported language list:
- `en`, `fr`, `es`, `zh`, `ja`, `ko`, `hi`, `de`, `pt`, `pl`, `it`, `nl`
- Behavior:
- Detects language from token output
- If detected language is outside the supported list, it falls back to the previous language
### 2) Lightning Whisper MLX (`--stt whisper-mlx`)
- Handler: `LightningWhisperSTTHandler`
- Uses same shared `--language` argument as Whisper
- Internal supported language list:
- `en`, `fr`, `es`, `zh`, `ja`, `ko`, `hi`, `de`, `pt`, `pl`, `it`, `nl`
- Behavior:
- If `--language auto`, model auto-detects each utterance
- If detected language is unsupported, falls back to last supported language
### 3) MLX Audio Whisper (`--stt mlx-audio-whisper`)
- Handler: `MLXAudioWhisperSTTHandler`
- Model flag: `--mlx_audio_whisper_model_name`
- Language still comes from shared `--language` flag (wired in pipeline)
- Internal supported language list:
- `en`, `fr`, `es`, `zh`, `ja`, `ko`, `hi`, `de`, `pt`, `pl`, `it`, `nl`
- Behavior:
- Uses fixed language unless `--language auto`
- Falls back to last known supported language when needed
### 4) Faster-Whisper (`--stt faster-whisper`)
- Handler: `FasterWhisperSTTHandler`
- Language flag: `--faster_whisper_stt_gen_language`
- Default language: `en`
- Note:
- This handler passes generation kwargs directly to `faster_whisper.WhisperModel.transcribe(...)`
- Effective language coverage depends on selected Faster-Whisper/OpenAI Whisper model
### 5) Parakeet TDT (`--stt parakeet-tdt`)
- Handler: `ParakeetTDTSTTHandler`
- Language flag: `--parakeet_tdt_language` (optional)
- Supports auto language detection when language not specified
- Declared supported language list (25 European languages):
- `en`, `de`, `fr`, `es`, `it`, `pt`, `nl`, `pl`, `ru`, `uk`, `cs`, `sk`, `hu`, `ro`, `bg`, `hr`, `sl`, `sr`, `da`, `no`, `sv`, `fi`, `et`, `lv`, `lt`
- Backend behavior:
- On macOS/MPS: MLX (`mlx-community/parakeet-tdt-0.6b-v3`)
- On CUDA/CPU: nano-parakeet (`nvidia/parakeet-tdt-0.6b-v3`)
### 6) Paraformer (`--stt paraformer`)
- Handler: `ParaformerSTTHandler`
- Model flag: `--paraformer_stt_model_name`
- Default model: `paraformer-zh`
- No dedicated language flag in current args class
- Practical support:
- Depends on selected FunASR model checkpoint
- Default setup is Chinese-oriented (`zh`)
## Language Abbreviations (ISO-style codes seen in STT handlers)
| Code | Language |
|---|---|
| `en` | English |
| `fr` | French |
| `es` | Spanish |
| `zh` | Chinese |
| `ja` | Japanese |
| `ko` | Korean |
| `hi` | Hindi |
| `de` | German |
| `pt` | Portuguese |
| `pl` | Polish |
| `it` | Italian |
| `nl` | Dutch |
| `ru` | Russian |
| `uk` | Ukrainian |
| `cs` | Czech |
| `sk` | Slovak |
| `hu` | Hungarian |
| `ro` | Romanian |
| `bg` | Bulgarian |
| `hr` | Croatian |
| `sl` | Slovenian |
| `sr` | Serbian |
| `da` | Danish |
| `no` | Norwegian |
| `sv` | Swedish |
| `fi` | Finnish |
| `et` | Estonian |
| `lv` | Latvian |
| `lt` | Lithuanian |
| `auto` | Per-utterance automatic language detection |
## Usage Examples
### Whisper (Transformers)
```bash
python s2s_pipeline.py --stt whisper --language en
python s2s_pipeline.py --stt whisper --language auto
```
### Whisper MLX (LightningWhisperMLX)
```bash
python s2s_pipeline.py --stt whisper-mlx --language auto --device mps
```
### MLX Audio Whisper
```bash
python s2s_pipeline.py --stt mlx-audio-whisper \
--mlx_audio_whisper_model_name mlx-community/whisper-large-v3-turbo \
--language auto
```
### Faster-Whisper
```bash
python s2s_pipeline.py --stt faster-whisper \
--faster_whisper_stt_model_name large-v3 \
--faster_whisper_stt_gen_language en
```
### Parakeet TDT
```bash
python s2s_pipeline.py --stt parakeet-tdt --parakeet_tdt_device auto
python s2s_pipeline.py --stt parakeet-tdt --parakeet_tdt_language de
```
With live transcription (MLX or CUDA/nano-parakeet backend):
```bash
python s2s_pipeline.py --stt parakeet-tdt \
--enable_live_transcription \
--live_transcription_update_interval 0.25
```
### Paraformer
```bash
python s2s_pipeline.py --stt paraformer --paraformer_stt_model_name paraformer-zh
```

1
src/speech_to_speech/STT/__init__.py Normal file
View File

@ -0,0 +1 @@

212
src/speech_to_speech/STT/base_stt_handler.py Normal file
View File

@ -0,0 +1,212 @@
from __future__ import annotations
import logging
from collections import Counter, OrderedDict
from time import perf_counter
from typing import Any
from speech_to_speech.baseHandler import BaseHandler
from speech_to_speech.pipeline.handler_types import STTIn, STTOut
from speech_to_speech.pipeline.messages import PartialTranscription, Transcription, VADAudio
from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker
logger = logging.getLogger(__name__)
class BaseSTTHandler(BaseHandler[STTIn, STTOut]):
"""Base STT handler with speculative-turn stale input filtering."""
_MAX_COMPLETED_FINAL_REVISIONS = 2048
speculative_turns: SpeculativeTurnTracker | None = None
final_revision_settle_s: float = 0.0
def should_process_input(self, item: STTIn) -> bool:
mode = getattr(item, "mode", None)
turn_id = getattr(item, "turn_id", None)
turn_revision = getattr(item, "turn_revision", None)
if self._is_completed_final_revision(item):
queued_drops = self._drop_stale_queued_inputs()
self._log_stale_turn_item(item, "input-after-final", queued_drops=queued_drops)
return False
if mode == "progressive" and self._has_queued_final_for_revision(item):
self._log_stale_turn_item(item, "progressive-before-final")
return False
wait_for_stability = mode == "final"
gate_start = perf_counter()
is_latest = self._is_latest_turn_item(
item,
wait_for_pending_reopen=True,
wait_for_stability=wait_for_stability,
)
gate_wait_s = perf_counter() - gate_start
if gate_wait_s >= 0.05:
logger.info(
"%s: STT input gate waited %.3fs for turn=%s rev=%s mode=%s latest=%s age=%.3fs queue=%s",
self.__class__.__name__,
gate_wait_s,
turn_id,
turn_revision,
mode,
is_latest,
self._item_age_s(item),
self._safe_qsize(),
)
if not is_latest:
queued_drops = self._drop_stale_queued_inputs()
self._log_stale_turn_item(item, "input", queued_drops=queued_drops)
return False
return True
def should_emit_output(self, output: STTOut) -> bool:
if isinstance(output, PartialTranscription) and self._is_completed_final_revision(output):
self._log_stale_turn_item(output, "output-after-final")
return False
if not self._is_latest_turn_item(output, wait_for_pending_reopen=True, wait_for_stability=False):
self._log_stale_turn_item(output, "output")
return False
return True
def before_emit_output(self, output: STTOut) -> None:
if isinstance(output, Transcription):
self._mark_completed_final_revision(output)
def _is_latest_turn_item(
self,
item: object,
*,
wait_for_pending_reopen: bool,
wait_for_stability: bool,
) -> bool:
if self.speculative_turns is None:
return True
turn_id = getattr(item, "turn_id", None)
turn_revision = getattr(item, "turn_revision", None)
if turn_id is None or turn_revision is None:
return True
if wait_for_stability:
is_latest = self.speculative_turns.is_latest_after_stability_window(
turn_id,
turn_revision,
self.final_revision_settle_s,
)
elif wait_for_pending_reopen:
is_latest = self.speculative_turns.is_latest_after_pending_reopen(turn_id, turn_revision)
else:
is_latest = self.speculative_turns.is_latest(turn_id, turn_revision)
return is_latest
def _drop_stale_queued_inputs(self) -> int:
if self.speculative_turns is None or not hasattr(self.queue_in, "mutex") or not hasattr(self.queue_in, "queue"):
return 0
dropped = 0
with self.queue_in.mutex:
kept: list[Any] = []
while self.queue_in.queue:
queued_item = self.queue_in.queue.popleft()
if isinstance(queued_item, VADAudio) and (
self._is_completed_final_revision(queued_item)
or (queued_item.mode == "progressive" and self._has_queued_final_for_revision_locked(queued_item))
or not self._is_latest_turn_item(
queued_item,
wait_for_pending_reopen=False,
wait_for_stability=False,
)
):
dropped += 1
else:
kept.append(queued_item)
self.queue_in.queue.extend(kept)
if dropped:
self.queue_in.not_full.notify_all()
return dropped
def _log_stale_turn_item(self, item: object, stage: str, *, queued_drops: int = 0) -> None:
turn_id = getattr(item, "turn_id", None)
turn_revision = getattr(item, "turn_revision", None)
if turn_id is None or turn_revision is None:
return
if not hasattr(self, "_stale_drop_counts"):
self._stale_drop_counts: Counter[tuple[str, str, int]] = Counter()
key = (stage, turn_id, turn_revision)
self._stale_drop_counts[key] += 1
message = "%s: dropping stale STT %s for turn=%s rev=%s age=%.3fs"
args: tuple[object, ...] = (
self.__class__.__name__,
stage,
turn_id,
turn_revision,
self._item_age_s(item),
)
if queued_drops:
message += " (+%d queued)"
args = (*args, queued_drops)
if self._stale_drop_counts[key] == 1:
logger.info(message, *args)
else:
logger.debug(message, *args)
def _item_age_s(self, item: object) -> float:
created_at_s = getattr(item, "created_at_s", None)
if not isinstance(created_at_s, float):
return 0.0
return max(0.0, perf_counter() - created_at_s)
def _safe_qsize(self) -> int | str:
try:
return self.queue_in.qsize()
except NotImplementedError:
return "unknown"
def _has_queued_final_for_revision(self, item: object) -> bool:
if not hasattr(self.queue_in, "mutex") or not hasattr(self.queue_in, "queue"):
return False
with self.queue_in.mutex:
return self._has_queued_final_for_revision_locked(item)
def _has_queued_final_for_revision_locked(self, item: object) -> bool:
key = self._revision_key(item)
if key is None:
return False
return any(
isinstance(queued_item, VADAudio) and queued_item.mode == "final" and self._revision_key(queued_item) == key
for queued_item in self.queue_in.queue
)
def _revision_key(self, item: object) -> tuple[str, int] | None:
turn_id = getattr(item, "turn_id", None)
turn_revision = getattr(item, "turn_revision", None)
if not isinstance(turn_id, str) or not isinstance(turn_revision, int):
return None
return (turn_id, turn_revision)
def _completed_final_revisions(self) -> OrderedDict[tuple[str, int], None]:
if not hasattr(self, "_completed_final_revision_keys"):
self._completed_final_revision_keys: OrderedDict[tuple[str, int], None] = OrderedDict()
return self._completed_final_revision_keys
def _is_completed_final_revision(self, item: object) -> bool:
key = self._revision_key(item)
return key is not None and key in self._completed_final_revisions()
def _mark_completed_final_revision(self, output: Transcription) -> None:
key = self._revision_key(output)
if key is None:
return
completed = self._completed_final_revisions()
completed[key] = None
completed.move_to_end(key)
while len(completed) > self._MAX_COMPLETED_FINAL_REVISIONS:
completed.popitem(last=False)
def on_session_end(self) -> None:
if hasattr(self, "_completed_final_revision_keys"):
self._completed_final_revision_keys.clear()

68
src/speech_to_speech/STT/faster_whisper_handler.py Normal file
View File

@ -0,0 +1,68 @@
from __future__ import annotations
import logging
import os
from typing import Any, Iterator
from faster_whisper import WhisperModel
from rich.console import Console
from speech_to_speech.pipeline.handler_types import STTIn, STTOut
from speech_to_speech.pipeline.messages import Transcription
from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
console = Console()
logger = logging.getLogger(__name__)
class FasterWhisperSTTHandler(BaseSTTHandler):
"""
Handles the Speech To Text generation using a Whisper model.
"""
def setup(
self,
model_name: str = "tiny.en",
device: str = "auto",
compute_type: str = "auto",
gen_kwargs: dict[str, Any] = {},
) -> None:
self.gen_kwargs = self.adapt_gen_kwargs(gen_kwargs)
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
self.model = WhisperModel(model_name, device=device, compute_type=compute_type)
def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
logger.debug("infering faster whisper...")
segments, info = self.model.transcribe(vad_audio.audio, **self.gen_kwargs)
output_text = []
for segment in segments:
logger.debug("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
output_text.append(segment.text)
pred_text = " ".join(output_text).strip()
logger.debug("finished whisper inference")
if pred_text:
console.print(f"[yellow]USER: {pred_text}")
yield Transcription(
text=pred_text,
turn_id=vad_audio.turn_id,
turn_revision=vad_audio.turn_revision,
speech_stopped_at_s=vad_audio.created_at_s,
)
else:
logger.debug("no text detected. skipping...")
def cleanup(self) -> None:
print("Stopping FasterWhisperSTTHandler")
del self.model
def adapt_gen_kwargs(self, gen_kwargs: dict[str, Any]) -> dict[str, Any]:
gen_kwargs["without_timestamps"] = not gen_kwargs.pop("return_timestamps", True)
return gen_kwargs

103
src/speech_to_speech/STT/lightning_whisper_mlx_handler.py Normal file
View File

@ -0,0 +1,103 @@
from __future__ import annotations
import logging
from typing import Any, Iterator, Optional
import numpy as np
import torch
from lightning_whisper_mlx import LightningWhisperMLX
from rich.console import Console
from speech_to_speech.pipeline.handler_types import STTIn, STTOut
from speech_to_speech.pipeline.messages import Transcription
from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
logger = logging.getLogger(__name__)
console = Console()
SUPPORTED_LANGUAGES = [
"en",
"fr",
"es",
"zh",
"ja",
"ko",
"hi",
"de",
"pt",
"pl",
"it",
"nl",
]
class LightningWhisperSTTHandler(BaseSTTHandler):
"""
Handles the Speech To Text generation using a Whisper model.
"""
def setup(
self,
model_name: str = "distil-large-v3",
device: str = "mps",
torch_dtype: str = "float16",
compile_mode: Optional[str] = None,
language: Optional[str] = None,
gen_kwargs: dict[str, Any] = {},
) -> None:
if len(model_name.split("/")) > 1:
model_name = model_name.split("/")[-1]
self.device = device
self.model = LightningWhisperMLX(model=model_name, batch_size=6, quant=None)
self.start_language = language
self.last_language = language
self.warmup()
def warmup(self) -> None:
logger.info(f"Warming up {self.__class__.__name__}")
# 2 warmup steps for no compile or compile mode with CUDA graphs capture
n_steps = 1
dummy_input = np.array([0] * 512)
for _ in range(n_steps):
_ = self.model.transcribe(dummy_input)["text"].strip()
def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
logger.debug("infering whisper...")
audio = vad_audio.audio
if self.start_language != "auto":
transcription_dict = self.model.transcribe(audio, language=self.start_language)
else:
transcription_dict = self.model.transcribe(audio)
language_code = transcription_dict["language"]
if language_code not in SUPPORTED_LANGUAGES:
logger.warning(f"Whisper detected unsupported language: {language_code}")
if self.last_language in SUPPORTED_LANGUAGES: # reprocess with the last language
transcription_dict = self.model.transcribe(audio, language=self.last_language)
else:
transcription_dict = {"text": "", "language": "en"}
else:
self.last_language = language_code
pred_text = transcription_dict["text"].strip()
language_code = transcription_dict["language"]
torch.mps.empty_cache()
logger.debug("finished whisper inference")
console.print(f"[yellow]USER: {pred_text}")
logger.debug(f"Language Code Whisper: {language_code}")
if self.start_language == "auto":
language_code += "-auto"
yield Transcription(
text=pred_text,
language_code=language_code,
turn_id=vad_audio.turn_id,
turn_revision=vad_audio.turn_revision,
speech_stopped_at_s=vad_audio.created_at_s,
)

155
src/speech_to_speech/STT/mlx_audio_whisper_handler.py Normal file
View File

@ -0,0 +1,155 @@
from __future__ import annotations
import logging
from typing import Any, Iterator, Optional
import numpy as np
from rich.console import Console
from speech_to_speech.pipeline.handler_types import STTIn, STTOut
from speech_to_speech.pipeline.messages import Transcription
from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
logger = logging.getLogger(__name__)
console = Console()
SUPPORTED_LANGUAGES = [
"en",
"fr",
"es",
"zh",
"ja",
"ko",
"hi",
"de",
"pt",
"pl",
"it",
"nl",
]
class MLXAudioWhisperSTTHandler(BaseSTTHandler):
"""
Handles the Speech To Text generation using MLX Audio's Whisper implementation.
Optimized for Apple Silicon using the MLX framework.
"""
def setup(
self,
model_name: str = "mlx-community/whisper-large-v3-turbo",
language: Optional[str] = None,
gen_kwargs: dict[str, Any] = {},
) -> None:
from mlx_audio.stt.generate import load_model
from transformers import WhisperProcessor
self.model_name = model_name
self.start_language = language
self.last_language = language
self.gen_kwargs = gen_kwargs
# Load the model directly
logger.info(f"Loading model {model_name}...")
self.model = load_model(model_name)
# Check if processor was loaded, if not, load it manually from original model
if self.model._processor is None:
logger.info("Processor not found in MLX model, loading from original Whisper model...")
# Map MLX model names to their original Whisper counterparts
processor_model_map = {
"mlx-community/whisper-large-v3-turbo": "openai/whisper-large-v3",
"mlx-community/whisper-large-v3": "openai/whisper-large-v3",
"mlx-community/whisper-medium": "openai/whisper-medium",
"mlx-community/whisper-small": "openai/whisper-small",
"mlx-community/whisper-base": "openai/whisper-base",
"mlx-community/whisper-tiny": "openai/whisper-tiny",
}
# Get the appropriate processor model name
processor_model = processor_model_map.get(model_name, "openai/whisper-large-v3")
logger.info(f"Loading processor from {processor_model}...")
try:
self.model._processor = WhisperProcessor.from_pretrained(processor_model)
logger.info("Processor loaded successfully")
except Exception as e:
logger.error(f"Failed to load processor: {e}")
raise
logger.info(f"Model {model_name} loaded successfully")
self.warmup()
def warmup(self) -> None:
logger.info(f"Warming up {self.__class__.__name__}")
# Warmup with a dummy input
dummy_audio = np.zeros(16000, dtype=np.float32)
try:
# Pre-warm the model by running a transcription
_ = self.model.generate(dummy_audio, verbose=False)
logger.info("Model warmed up and ready")
except Exception as e:
logger.warning(f"Warmup failed: {e}")
def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
logger.debug("inferring mlx-audio whisper...")
assert isinstance(vad_audio.audio, np.ndarray), "Audio must be a numpy array"
audio_input = vad_audio.audio.astype(np.float32)
# Prepare generation kwargs - only pass valid parameters
gen_kwargs = {}
# Add language if specified
if self.start_language and self.start_language != "auto":
gen_kwargs["language"] = self.start_language
try:
# Generate transcription directly using model.generate
result = self.model.generate(audio_input, verbose=False, **gen_kwargs)
# Extract text from result
pred_text = result.text.strip() if hasattr(result, "text") else str(result).strip()
# Try to detect language from result if available
if hasattr(result, "language"):
language_code = result.language
elif self.start_language and self.start_language != "auto":
language_code = self.start_language
else:
# Default to last known language or English
language_code = self.last_language if self.last_language else "en"
# Validate language code
if language_code not in SUPPORTED_LANGUAGES:
logger.warning(f"Detected unsupported language: {language_code}")
if self.last_language in SUPPORTED_LANGUAGES:
language_code = self.last_language
else:
language_code = "en"
else:
self.last_language = language_code
except Exception as e:
logger.error(f"MLX Audio Whisper inference failed: {e}")
pred_text = ""
language_code = self.last_language if self.last_language else "en"
logger.debug("finished mlx-audio whisper inference")
console.print(f"[yellow]USER: {pred_text}")
logger.debug(f"Language Code: {language_code}")
if self.start_language == "auto":
language_code += "-auto"
yield Transcription(
text=pred_text,
language_code=language_code,
turn_id=vad_audio.turn_id,
turn_revision=vad_audio.turn_revision,
speech_stopped_at_s=vad_audio.created_at_s,
)

79
src/speech_to_speech/STT/paraformer_handler.py Normal file
View File

@ -0,0 +1,79 @@
from __future__ import annotations
import logging
from typing import Any, Iterator
import numpy as np
import torch
from rich.console import Console
from speech_to_speech.pipeline.handler_types import STTIn, STTOut
from speech_to_speech.pipeline.messages import PartialTranscription, Transcription
from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
logging.basicConfig(
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)
console = Console()
class ParaformerSTTHandler(BaseSTTHandler):
"""
Handles the Speech To Text generation using a Paraformer model.
The default for this model is set to Chinese.
This model was contributed by @wuhongsheng.
"""
def setup(
self,
model_name: str = "paraformer-zh",
device: str = "cuda",
gen_kwargs: dict[str, Any] = {},
) -> None:
print(model_name)
if len(model_name.split("/")) > 1:
model_name = model_name.split("/")[-1]
self.device = device
try:
from funasr import AutoModel
except ModuleNotFoundError as exc:
raise ModuleNotFoundError(
"Paraformer STT requires the optional 'paraformer' extra. "
"Install it with `pip install speech-to-speech[paraformer]`."
) from exc
self.model = AutoModel(model=model_name, device=device)
self.warmup()
def warmup(self) -> None:
logger.info(f"Warming up {self.__class__.__name__}")
# 2 warmup steps for no compile or compile mode with CUDA graphs capture
n_steps = 1
dummy_input = np.array([0] * 512, dtype=np.float32)
for _ in range(n_steps):
_ = self.model.generate(dummy_input)[0]["text"].strip().replace(" ", "")
def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
logger.debug("infering paraformer...")
pred_text = self.model.generate(vad_audio.audio)[0]["text"].strip().replace(" ", "")
torch.mps.empty_cache()
logger.debug("finished paraformer inference")
console.print(f"[yellow]USER: {pred_text}")
if vad_audio.mode == "progressive":
yield PartialTranscription(
text=pred_text,
turn_id=vad_audio.turn_id,
turn_revision=vad_audio.turn_revision,
)
else:
yield Transcription(
text=pred_text,
turn_id=vad_audio.turn_id,
turn_revision=vad_audio.turn_revision,
speech_stopped_at_s=vad_audio.created_at_s,
)

648
src/speech_to_speech/STT/parakeet_tdt_handler.py Normal file
View File

@ -0,0 +1,648 @@
"""
Parakeet TDT Speech-to-Text Handler
Supports NVIDIA Parakeet TDT model for high-quality multilingual ASR.
- On Apple Silicon (MPS): Uses mlx-audio with mlx-community/parakeet-tdt-0.6b-v3
- On CUDA/CPU: Uses nano-parakeet (pure PyTorch) with nvidia/parakeet-tdt-0.6b-v3
Model supports 25 European languages with automatic language detection.
"""
from __future__ import annotations
import logging
from contextlib import contextmanager
from sys import platform
from threading import Lock
from time import perf_counter
from typing import Any, Iterator, Optional
import numpy as np
from rich.console import Console
from rich.text import Text
from speech_to_speech.pipeline.handler_types import STTIn, STTOut
from speech_to_speech.pipeline.messages import PartialTranscription, Transcription
from speech_to_speech.STT.base_stt_handler import BaseSTTHandler
from speech_to_speech.STT.smart_progressive_streaming import PartialTranscription as ProgressiveStreamPartial
from speech_to_speech.utils.mlx_lock import MLXLockContext
try:
from lingua import Language, LanguageDetectorBuilder
LINGUA_AVAILABLE = True
except ImportError:
LINGUA_AVAILABLE = False
logger = logging.getLogger(__name__)
console = Console()
# Parakeet TDT v3 supports 25 European languages
SUPPORTED_LANGUAGES = [
"en",
"de",
"fr",
"es",
"it",
"pt",
"nl",
"pl",
"ru",
"uk",
"cs",
"sk",
"hu",
"ro",
"bg",
"hr",
"sl",
"sr",
"da",
"no",
"sv",
"fi",
"et",
"lv",
"lt",
]
# Lingua uses "nb" (Bokmål) for Norwegian instead of "no"
_LINGUA_CODE_MAP = {"no": "nb"}
if LINGUA_AVAILABLE:
_lingua_iso_to_code = {
lang.iso_code_639_1.name.lower(): lang for lang in Language.all() if lang.iso_code_639_1 is not None
}
_lingua_languages = [
_lingua_iso_to_code[_LINGUA_CODE_MAP.get(code, code)]
for code in SUPPORTED_LANGUAGES
if _LINGUA_CODE_MAP.get(code, code) in _lingua_iso_to_code
]
def _build_lingua_detector():
# Preloading can take multiple seconds on some hardware, including the
# deployed server. Pay that cost at startup instead of on the first user
# request, where it would look like slow STT.
return LanguageDetectorBuilder.from_languages(*_lingua_languages).with_preloaded_language_models().build()
_lingua_detector = _build_lingua_detector()
class ParakeetTDTSTTHandler(BaseSTTHandler):
"""
Handles Speech-to-Text using NVIDIA Parakeet TDT model.
On Apple Silicon (MPS): Uses mlx-audio with the MLX-converted model.
On CUDA/CPU: Uses nano-parakeet (pure PyTorch) for NeMo-free inference.
Parakeet TDT 0.6B v3 is a 600M parameter multilingual ASR model
supporting 25 European languages with automatic language detection.
"""
def setup(
self,
model_name: Optional[str] = None,
device: str = "auto",
compute_type: str = "float16",
language: Optional[str] = None,
gen_kwargs: dict[str, Any] = {},
enable_live_transcription: bool = False,
live_transcription_update_interval: float = 0.5,
) -> None:
"""
Initialize the Parakeet TDT model.
Args:
model_name: Model identifier. Defaults are:
- MPS: "mlx-community/parakeet-tdt-0.6b-v3"
- CUDA/CPU: "nvidia/parakeet-tdt-0.6b-v3"
device: Device to use ("auto", "cuda", "mps", "cpu")
compute_type: Compute precision ("float16", "float32")
language: Target language code (optional, model auto-detects)
gen_kwargs: Additional generation kwargs
"""
self.gen_kwargs = gen_kwargs
self.start_language = language
self.last_language = language if language else "en"
self.enable_live_transcription = enable_live_transcription
self.live_transcription_update_interval = live_transcription_update_interval
self.compute_lock = Lock()
self.sample_rate = 16000
# Determine device
if device == "auto":
if platform == "darwin":
self.device = "mps"
else:
import torch
self.device = "cuda" if torch.cuda.is_available() else "cpu"
else:
self.device = device
# Set default model based on device
if model_name is None:
if self.device == "mps":
model_name = "mlx-community/parakeet-tdt-0.6b-v3"
else:
model_name = "nvidia/parakeet-tdt-0.6b-v3"
self.model_name = model_name
self.compute_type = compute_type
logger.info(f"Loading Parakeet TDT model: {model_name} on {self.device}")
if self.device == "mps":
self._setup_mlx(model_name)
else:
self._setup_nano_parakeet(model_name)
# Setup streaming handler if live transcription is enabled
self.streaming_handler = None
if self.enable_live_transcription:
from speech_to_speech.STT.smart_progressive_streaming import (
SmartProgressiveStreamingHandler,
)
self.streaming_handler = SmartProgressiveStreamingHandler(
self.model,
emission_interval=self.live_transcription_update_interval,
max_window_size=15.0,
sentence_buffer=2.0,
)
self.processing_final = False # Track if we're processing final audio
logger.info(f"Live transcription enabled for Parakeet TDT ({self.backend})")
self._live_transcription_active = False
self._live_turn_key: tuple[str | None, int | None] | None = None
self.warmup()
def _setup_mlx(self, model_name: str) -> None:
"""Setup for Apple Silicon using mlx-audio."""
try:
from mlx_audio.stt.generate import load_model
self.backend = "mlx"
self.model = load_model(model_name)
logger.info("MLX Audio Parakeet model loaded successfully")
except ImportError as e:
raise ImportError(
"mlx-audio is required for Parakeet TDT on Apple Silicon. Install with: pip install mlx-audio"
) from e
def _setup_nano_parakeet(self, model_name: str) -> None:
"""Setup for CUDA/CPU using nano-parakeet."""
try:
import torch
from nano_parakeet import from_pretrained
self.backend = "nano_parakeet"
if self.device == "cuda" and not torch.cuda.is_available():
logger.warning("CUDA requested but not available. Falling back to CPU for nano-parakeet.")
self.device = "cpu"
self.model = from_pretrained(model_name=model_name, device=self.device)
logger.info(f"nano-parakeet model loaded successfully on {self.device}")
except ImportError as e:
raise ImportError(
"nano-parakeet is required for Parakeet TDT on CUDA/CPU. Install with: pip install nano-parakeet"
) from e
def warmup(self) -> None:
"""Warm up the model with a dummy input."""
logger.info(f"Warming up {self.__class__.__name__}")
# Create 1 second of silence at 16kHz
dummy_audio = np.zeros(16000, dtype=np.float32)
try:
if self.backend == "mlx":
import mlx.core as mx
# Convert to mx.array and call decode_chunk directly
audio_mx = mx.array(dummy_audio, dtype=mx.float32)
_ = self.model.decode_chunk(audio_mx, verbose=False)
elif self.backend == "nano_parakeet":
_ = self.model.transcribe(dummy_audio)
else:
_ = self.model.transcribe([dummy_audio], batch_size=1, verbose=False)
logger.info("Model warmed up and ready")
except Exception as e:
logger.warning(f"Warmup failed: {e}")
def process(self, vad_audio: STTIn) -> Iterator[STTOut]:
"""
Process audio and generate transcription.
Yields:
:class:`PartialTranscription` or :class:`Transcription`
"""
process_start_s = perf_counter()
is_progressive = vad_audio.mode == "progressive"
audio_input = vad_audio.audio
# Ensure audio is float32 numpy array
if not isinstance(audio_input, np.ndarray):
audio_input = np.array(audio_input, dtype=np.float32)
else:
audio_input = audio_input.astype(np.float32)
audio_duration_s = len(audio_input) / getattr(self, "sample_rate", 16000)
item_age_s = self._item_age_s(vad_audio)
self._prepare_live_transcription_turn(vad_audio.turn_id, vad_audio.turn_revision)
# Handle progressive updates: yield tagged partial for TranscriptionNotifier
if self.enable_live_transcription and is_progressive:
# Ignore progressive updates if we're already processing final audio
if self.processing_final:
logger.debug("Skipping stale progressive update (final audio already received)")
return
# Try to acquire lock with short timeout - skip if busy
lock_scope_start_s = perf_counter()
with self._compute_lock_context(handler_name="ParakeetSTT-Progressive", timeout=0.01) as acquired:
if acquired:
try:
inference_start_s = perf_counter()
progressive_text = self._show_progressive_transcription(audio_input)
inference_s = perf_counter() - inference_start_s
if inference_s >= 0.25:
logger.info(
"Parakeet progressive STT timing turn=%s rev=%s audio=%.3fs age=%.3fs "
"lock_scope=%.3fs inference=%.3fs chars=%d",
vad_audio.turn_id,
vad_audio.turn_revision,
audio_duration_s,
item_age_s,
perf_counter() - lock_scope_start_s,
inference_s,
len(progressive_text),
)
if progressive_text:
yield PartialTranscription(
text=progressive_text,
turn_id=vad_audio.turn_id,
turn_revision=vad_audio.turn_revision,
)
return
except Exception as e:
logger.debug(f"Progressive transcription failed: {e}")
else:
logger.debug("Skipping progressive update (compute busy)")
return
# Handle final transcription (send to LLM)
logger.info(
"Parakeet final STT start turn=%s rev=%s audio=%.3fs age=%.3fs",
vad_audio.turn_id,
vad_audio.turn_revision,
audio_duration_s,
item_age_s,
)
inference_s = 0.0
lock_scope_s = 0.0
try:
if self.enable_live_transcription:
# Mark that we're processing final audio (ignore stale progressive updates)
self.processing_final = True
# Acquire lock with longer timeout for final transcription
lock_scope_start_s = perf_counter()
with self._compute_lock_context(handler_name="ParakeetSTT-Final", timeout=5.0) as acquired:
lock_scope_s = perf_counter() - lock_scope_start_s
if not acquired:
logger.error("Failed to acquire compute lock for final transcription")
pred_text = ""
language_code = self.last_language
else:
inference_start_s = perf_counter()
if self.backend == "mlx":
pred_text, language_code = self._process_mlx_final(audio_input)
else:
pred_text, language_code = self._process_nano_parakeet(audio_input)
inference_s = perf_counter() - inference_start_s
lock_scope_s = perf_counter() - lock_scope_start_s
# Validate and update language
if language_code and language_code in SUPPORTED_LANGUAGES:
self.last_language = language_code
else:
language_code = self.last_language
except Exception as e:
logger.error(f"Parakeet TDT inference failed: {e}")
pred_text = ""
language_code = self.last_language
total_s = perf_counter() - process_start_s
logger.info(
"Parakeet final STT done turn=%s rev=%s total=%.3fs lock_scope=%.3fs inference=%.3fs chars=%d",
vad_audio.turn_id,
vad_audio.turn_revision,
total_s,
lock_scope_s,
inference_s,
len(pred_text),
)
logger.debug("Finished Parakeet TDT inference")
self._clear_live_transcription_line()
if pred_text.strip():
console.print(f"[yellow]USER: {pred_text.strip()}")
if language_code:
console.print(f"[dim]Language: {language_code}[/dim]")
# Reset per-utterance live transcription state only after final STT
# completes. The streaming handler carries fixed sentence timing within
# an utterance, and stale timing must not leak into the next turn.
if self.enable_live_transcription:
self.processing_final = False
self._reset_live_transcription_state(clear_turn=True)
yield Transcription(
text=pred_text,
language_code=language_code,
turn_id=vad_audio.turn_id,
turn_revision=vad_audio.turn_revision,
speech_stopped_at_s=vad_audio.created_at_s,
)
@property
def timing_log_level(self) -> int:
return logging.INFO
def should_log_timing(self, output: STTOut) -> bool:
return isinstance(output, Transcription) and self.last_time > self.min_time_to_debug
def _detect_language_from_text(self, text: str) -> Optional[str]:
"""
Detect language from transcribed text using lingua-py.
Args:
text: Transcribed text string
Returns:
Detected language code or None if detection fails
"""
if not LINGUA_AVAILABLE:
logger.warning("lingua-py not available, cannot detect language from text")
return None
# Skip very short utterances where language ID is still too noisy.
if not text or len(text.strip()) < 20:
return None
detected = _lingua_detector.detect_language_of(text)
if detected is None:
return None
code = detected.iso_code_639_1.name.lower()
# Map back lingua-specific codes to our supported codes
return {v: k for k, v in _LINGUA_CODE_MAP.items()}.get(code, code)
@contextmanager
def _compute_lock_context(self, handler_name: str, timeout: float) -> Iterator[bool]:
if self.backend == "mlx":
with MLXLockContext(handler_name=handler_name, timeout=timeout) as acquired:
yield acquired
return
lock_start_s = perf_counter()
acquired = self.compute_lock.acquire(timeout=timeout)
wait_s = perf_counter() - lock_start_s
hold_start_s: float | None = None
if acquired:
if wait_s >= 0.25:
logger.info("%s: compute lock acquired after %.2fs", handler_name, wait_s)
else:
logger.debug("%s: compute lock acquired after %.3fs", handler_name, wait_s)
hold_start_s = perf_counter()
else:
logger.warning("%s: Failed to acquire compute lock after %.3fs (timeout=%s)", handler_name, wait_s, timeout)
try:
yield acquired
finally:
if acquired:
assert hold_start_s is not None
self.compute_lock.release()
hold_s = perf_counter() - hold_start_s
if hold_s >= 0.25:
logger.info("%s: compute lock released after holding %.2fs", handler_name, hold_s)
else:
logger.debug("%s: compute lock released after holding %.3fs", handler_name, hold_s)
def _show_progressive_transcription(self, audio_input: np.ndarray) -> str:
"""Run progressive transcription, print to console, and return the text."""
result = self.streaming_handler.transcribe_incremental(audio_input)
rich_text = Text()
if result.fixed_text:
rich_text.append("Live: ", style="dim")
rich_text.append(result.fixed_text, style="yellow")
if result.active_text:
rich_text.append(" ", style="dim")
if result.active_text:
if not result.fixed_text:
rich_text.append("Live: ", style="dim")
rich_text.append(result.active_text, style="cyan dim")
progressive_text = self._build_progressive_text(result)
if progressive_text:
self._print_live_transcription(rich_text, progressive_text)
return progressive_text
def _print_live_transcription(self, rich_text: Text, progressive_text: str) -> None:
is_terminal = bool(getattr(console, "is_terminal", False))
if is_terminal:
self._write_live_control("\r\x1b[2K")
if rich_text:
console.print(self._truncate_live_transcription(rich_text), end="")
else:
fallback = Text("Live: ", style="dim")
fallback.append(progressive_text, style="cyan dim")
console.print(self._truncate_live_transcription(fallback), end="")
self._write_live_control("\r")
self._live_transcription_active = True
return
if rich_text:
console.print(rich_text)
else:
console.print(f"[dim]Live: [/dim]{progressive_text}")
def _clear_live_transcription_line(self) -> None:
if not getattr(self, "_live_transcription_active", False):
return
self._write_live_control("\r\x1b[2K")
self._live_transcription_active = False
def _write_live_control(self, sequence: str) -> None:
file = getattr(console, "file", None)
if file is None:
return
file.write(sequence)
file.flush()
def _truncate_live_transcription(self, text: Text) -> Text:
text = text.copy()
width = getattr(console, "width", 80)
try:
max_width = max(1, int(width) - 1)
except (TypeError, ValueError):
max_width = 79
text.truncate(max_width, overflow="ellipsis")
return text
def _prepare_live_transcription_turn(self, turn_id: str | None, turn_revision: int | None) -> None:
if not self.enable_live_transcription:
return
turn_key = (turn_id, turn_revision)
if getattr(self, "_live_turn_key", None) == turn_key:
return
self._reset_live_transcription_state(clear_turn=False)
self._live_turn_key = turn_key
def _reset_live_transcription_state(self, clear_turn: bool) -> None:
self._clear_live_transcription_line()
streaming_handler = getattr(self, "streaming_handler", None)
if streaming_handler is not None:
streaming_handler.reset()
if clear_turn:
self._live_turn_key = None
def _build_progressive_text(self, result: ProgressiveStreamPartial) -> str:
parts = []
if result.fixed_text:
parts.append(result.fixed_text.strip())
if result.active_text:
parts.append(result.active_text.strip())
return " ".join(part for part in parts if part).strip()
def _process_mlx_final(self, audio_input: np.ndarray) -> tuple[str, str]:
"""Process final audio using MLX backend with streaming handler."""
# If we have fixed sentences from progressive updates, only transcribe the new part
if (
self.streaming_handler is not None
and hasattr(self.streaming_handler, "fixed_sentences")
and self.streaming_handler.fixed_sentences
):
self._clear_live_transcription_line()
# Get fixed text from previous progressive updates
fixed_text = " ".join(self.streaming_handler.fixed_sentences).strip()
# Calculate where fixed part ends in audio
fixed_end_time = self.streaming_handler.fixed_end_time
sample_rate = 16000
fixed_end_sample = int(fixed_end_time * sample_rate)
if fixed_end_sample > len(audio_input):
logger.warning(
"Ignoring stale progressive fixed text: fixed_end_sample=%d exceeds final audio samples=%d",
fixed_end_sample,
len(audio_input),
)
pred_text, language_code = self._process_mlx(audio_input)
return pred_text, language_code
# Only transcribe the part after fixed sentences
if fixed_end_sample < len(audio_input):
remaining_audio = audio_input[fixed_end_sample:]
import mlx.core as mx
audio_mx = mx.array(remaining_audio, dtype=mx.float32)
result = self.model.decode_chunk(audio_mx, verbose=False)
if hasattr(result, "text"):
new_text = result.text.strip()
else:
new_text = str(result).strip()
# Combine fixed + new
pred_text = f"{fixed_text} {new_text}".strip() if new_text else fixed_text
else:
# All audio already transcribed in progressive updates
pred_text = fixed_text
# Reset streaming handler for next utterance
self.streaming_handler.reset()
else:
# No progressive updates, transcribe everything
pred_text, language_code = self._process_mlx(audio_input)
return pred_text, language_code
# Determine language
if self.start_language and self.start_language != "auto":
language_code = self.start_language
else:
detected_lang = self._detect_language_from_text(pred_text)
if detected_lang:
language_code = detected_lang
else:
language_code = self.last_language
return pred_text, language_code
def _process_mlx(self, audio_input: np.ndarray) -> tuple[str, str]:
"""Process audio using MLX backend."""
import mlx.core as mx
# Convert numpy array to mx.array
audio_mx = mx.array(audio_input, dtype=mx.float32)
# Call decode_chunk directly with the audio array
result = self.model.decode_chunk(audio_mx, verbose=False)
# Extract text from result
if hasattr(result, "text"):
pred_text = result.text.strip()
else:
pred_text = str(result).strip()
# Determine language:
# 1. Use fixed language if specified by user
# 2. Try to detect from transcribed text using langdetect
# 3. Fall back to last known language
if self.start_language and self.start_language != "auto":
language_code = self.start_language
else:
# Detect language from transcribed text
detected_lang = self._detect_language_from_text(pred_text)
if detected_lang:
language_code = detected_lang
else:
language_code = self.last_language
return pred_text, language_code
def _process_nano_parakeet(self, audio_input: np.ndarray) -> tuple[str, str]:
"""Process audio using nano-parakeet backend."""
pred_text = self.model.transcribe(audio_input).strip()
if self.start_language and self.start_language != "auto":
language_code = self.start_language
else:
detected_lang = self._detect_language_from_text(pred_text)
if detected_lang:
language_code = detected_lang
else:
language_code = self.last_language
return pred_text, language_code
def cleanup(self) -> None:
"""Clean up model resources."""
logger.info(f"Cleaning up {self.__class__.__name__}")
if hasattr(self, "model"):
del self.model
def on_session_end(self) -> None:
super().on_session_end()
self.last_language = self.start_language if self.start_language else "en"
if self.enable_live_transcription:
self.processing_final = False
self._reset_live_transcription_state(clear_turn=True)
logger.debug("Parakeet TDT session state reset")

Some files were not shown because too many files have changed in this diff Show More