commit b5f82fb48c942da55c29ce20076c2e5bc9933b5d Author: valenti Date: Wed Aug 26 11:30:14 2026 +0000 first git diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..791678f Binary files /dev/null and b/.DS_Store differ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..af66ddd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +tmp +cache +.venv +.git +.mypy_cache +.pytest_cache +.ruff_cache +__pycache__ +**/__pycache__ +*.py[cod] +*.egg-info +dist +build +Dockerfile +docker-compose.yml +.dockerignore +.gitignore diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..15f7bdd --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + groups: + actions: + patterns: ["*"] diff --git a/.github/scripts/star_history.py b/.github/scripts/star_history.py new file mode 100644 index 0000000..9f5f004 --- /dev/null +++ b/.github/scripts/star_history.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Fetch stargazer history for a repo and render it as a static SVG. + +Runs in CI with the automatic GITHUB_TOKEN. No third-party dependencies. + +Usage: + GITHUB_TOKEN=... python star_history.py owner/repo output.svg +""" + +import json +import math +import os +import sys +import urllib.request +from datetime import datetime, timezone + +API = "https://api.github.com/graphql" +PER_PAGE = 100 +MAX_PAGES = 400 # preserve the REST implementation's 40k-star cap +MAX_POINTS = 120 # downsample the curve to at most this many points + + +def gh_post(query: str, variables: dict, token: str) -> dict: + payload = json.dumps({"query": query, "variables": variables}).encode() + req = urllib.request.Request(API, data=payload) + req.add_header("Accept", "application/vnd.github+json") + req.add_header("Content-Type", "application/json") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=30) as resp: + result = json.loads(resp.read()) + if result.get("errors"): + raise RuntimeError(json.dumps(result["errors"])) + return result + + +def fetch_star_dates(repo: str, token: str) -> list[datetime]: + owner, name = repo.split("/", 1) + query = """ + query($owner: String!, $name: String!, $cursor: String, $perPage: Int!) { + repository(owner: $owner, name: $name) { + stargazers(first: $perPage, after: $cursor) { + edges { starredAt } + pageInfo { hasNextPage endCursor } + } + } + } + """ + dates = [] + page = 1 + cursor = None + while True: + result = gh_post( + query, + { + "owner": owner, + "name": name, + "cursor": cursor, + "perPage": PER_PAGE, + }, + token, + ) + stargazers = result["data"]["repository"]["stargazers"] + for edge in stargazers["edges"]: + ts = edge.get("starredAt") + if ts: + dates.append( + datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace( + tzinfo=timezone.utc + ) + ) + page_info = stargazers["pageInfo"] + if not page_info["hasNextPage"]: + break + page += 1 + if page > MAX_PAGES: + break + cursor = page_info["endCursor"] + dates.sort() + return dates + + +def downsample(dates: list[datetime]) -> list[tuple[datetime, int]]: + n = len(dates) + points = [(dates[0], 1)] + if n > 1: + step = max(1, n // MAX_POINTS) + for i in range(step, n, step): + points.append((dates[i], i + 1)) + points.append((dates[-1], n)) + now = datetime.now(timezone.utc) + points.append((now, n)) + return points + + +def nice_ceil(v: float) -> int: + if v <= 0: + return 1 + mag = 10 ** int(math.floor(math.log10(v))) + for mult in (1, 1.2, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10): + if v <= mag * mult: + return int(mag * mult) + return int(mag * 10) + + +def render_svg(points: list[tuple[datetime, int]], repo: str) -> str: + width, height = 800, 420 + ml, mr, mt, mb = 70, 30, 50, 60 + pw, ph = width - ml - mr, height - mt - mb + + t0 = points[0][0].timestamp() + t1 = points[-1][0].timestamp() + tspan = max(t1 - t0, 1) + ymax = nice_ceil(points[-1][1] * 1.08) + + def x(ts: float) -> float: + return ml + (ts - t0) / tspan * pw + + def y(v: float) -> float: + return mt + ph - v / ymax * ph + + line = " ".join( + f"{'M' if i == 0 else 'L'}{x(d.timestamp()):.1f},{y(c):.1f}" + for i, (d, c) in enumerate(points) + ) + area = ( + line + + f" L{x(points[-1][0].timestamp()):.1f},{y(0):.1f}" + + f" L{x(points[0][0].timestamp()):.1f},{y(0):.1f} Z" + ) + + grid, ylabels = [], [] + for i in range(6): + v = ymax * i / 5 + yy = y(v) + grid.append( + f'' + ) + label = f"{v / 1000:.1f}k".replace(".0k", "k") if v >= 1000 else f"{int(v)}" + ylabels.append( + f'{label}' + ) + + xlabels = [] + for i in range(6): + ts = t0 + tspan * i / 5 + d = datetime.fromtimestamp(ts, tz=timezone.utc) + xlabels.append( + f'{d.strftime("%b %Y")}' + ) + + total = points[-1][1] + return f""" + + {repo} star history + {total:,} stars + {"".join(grid)} + {"".join(ylabels)} + {"".join(xlabels)} + + + + +""" + + +def main() -> int: + if len(sys.argv) != 3: + print(__doc__) + return 2 + repo, out = sys.argv[1], sys.argv[2] + token = os.environ.get("GITHUB_TOKEN", "") + dates = fetch_star_dates(repo, token) + if not dates: + print("no stargazers found", file=sys.stderr) + return 1 + os.makedirs(os.path.dirname(out) or ".", exist_ok=True) + svg = render_svg(downsample(dates), repo) + with open(out, "w") as f: + f.write(svg) + print(f"wrote {out} ({len(dates):,} stars)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5ae14e7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,98 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: astral-sh/setup-uv@v8.2.0 + with: + version: "latest" + - run: uv sync --group dev + - name: Ruff check + run: uv run ruff check src/ tests/ + - name: Ruff format + run: uv run ruff format --check src/ tests/ + + mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: astral-sh/setup-uv@v8.2.0 + with: + version: "latest" + - run: uv sync --group dev + - name: Mypy + run: uv run mypy src/ + + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: astral-sh/setup-uv@v8.2.0 + with: + version: "latest" + - run: uv sync --group dev + - name: NLTK data (punkt_tab for sent_tokenize) + run: uv run python -c "import nltk; nltk.download('punkt_tab')" + - name: Pytest + run: uv run pytest tests/ -x -q + + package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: astral-sh/setup-uv@v8.2.0 + with: + version: "latest" + - name: Build distributions + run: uv build + - name: Check package metadata + run: uvx twine check --strict dist/* + - uses: actions/upload-artifact@v7 + with: + name: dist + path: dist/* + + install-smoke: + name: Install smoke (${{ matrix.label }}) + needs: package + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - label: linux + os: ubuntu-latest + - label: macos-arm64 + os: macos-14 + steps: + - uses: actions/checkout@v7.0.0 + - uses: actions/setup-python@v6.3.0 + with: + python-version: "3.11" + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist + - name: Install wheel with pip + shell: bash + run: | + python -m venv .venv-smoke + source .venv-smoke/bin/activate + python -m pip install --upgrade pip + python -m pip install dist/speech_to_speech-*.whl + python -m pip check + - name: Run installed CLI smoke test + shell: bash + run: | + source .venv-smoke/bin/activate + unset OPENAI_API_KEY + # Validate the installed entry point without loading models or calling OpenAI. + python tests/install_smoke.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..e72931b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,48 @@ +name: Publish + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + version: "latest" + enable-cache: false + - name: Build distributions + run: uv build + - name: Check package metadata + run: uvx twine check --strict dist/* + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dist + path: dist/* + + publish: + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/speech-to-speech/ + permissions: + contents: read + id-token: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist + path: dist + - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 diff --git a/.github/workflows/star-history.yml b/.github/workflows/star-history.yml new file mode 100644 index 0000000..fb6386e --- /dev/null +++ b/.github/workflows/star-history.yml @@ -0,0 +1,38 @@ +name: Update star history chart + +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + update-chart: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Generate chart + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 .github/scripts/star_history.py "${{ github.repository }}" assets/star-history.svg + + - name: Commit to update branch if changed + env: + BASE_BRANCH: ${{ github.event.repository.default_branch }} + UPDATE_BRANCH: star-history-update-${{ github.run_id }}-${{ github.run_attempt }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add assets/star-history.svg + if ! git diff --cached --quiet; then + git commit -m "chore: update star history chart [skip ci]" + git push origin "HEAD:refs/heads/${UPDATE_BRANCH}" + pr_url="https://github.com/${{ github.repository }}/compare/${BASE_BRANCH}...${UPDATE_BRANCH}?expand=1" + echo "Open a PR: ${pr_url}" + echo "Open a PR: [${UPDATE_BRANCH}](${pr_url})" >> "$GITHUB_STEP_SUMMARY" + else + echo "No changes" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..55db326 --- /dev/null +++ b/.gitignore @@ -0,0 +1,231 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ +# Temporary file for partial code execution +tempCodeRunnerFile.py + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml + +.mypy_cache/ +.pytype/ +.pyre/ +.dmypy.json +dmypy.json +.pytest_cache/ +.hypothesis/ +.coverage* +.coverage +cover/ +.tox/ +uv.lock \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c1e4353 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# Repository Instructions + +- Never include `codex` in branch names or pull request titles. +- Keep release pull requests focused on version metadata and release documentation. +- Do not commit local build artifacts such as `dist/`, `build/`, or generated wheel/sdist files. + +## Publishing to PyPI + +PyPI publishing is handled by GitHub Actions in `.github/workflows/publish.yml`. The workflow runs on pushed tags that match `v*`, builds the package with `uv build`, checks the artifacts with `twine check --strict`, and publishes through the configured `pypi` environment. + +To prepare a release: + +1. Confirm the intended version is not already published on PyPI. +2. Bump `version` in `pyproject.toml`. +3. Bump `__version__` in `src/speech_to_speech/__init__.py`. +4. Open and merge a pull request with only the release preparation changes. + +To publish after the release PR is merged: + +1. Update `main` locally: `git checkout main && git pull origin main`. +2. Create an annotated tag for the version: `git tag -a vX.Y.Z -m "Release vX.Y.Z"`. +3. Push the tag: `git push origin vX.Y.Z`. +4. Watch the `Publish` GitHub Actions workflow complete successfully. +5. Verify the new version appears at `https://pypi.org/project/speech-to-speech/`. + +Only upload manually if the GitHub Actions workflow is unavailable and the maintainers have explicitly chosen that fallback. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..99d58f7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV PATH="/usr/src/app/.venv/bin:${PATH}" + +WORKDIR /usr/src/app + +# Install packages +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + libportaudio2 \ + libsndfile1 \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +RUN python3 -m pip install --no-cache-dir --break-system-packages uv + +COPY pyproject.toml README.md LICENSE MANIFEST.in ./ +RUN uv sync --python /usr/bin/python3 --no-install-project --no-dev + +COPY . . +RUN uv sync --python /usr/bin/python3 --no-dev +RUN python -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')" diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 new file mode 100644 index 0000000..948574b --- /dev/null +++ b/Dockerfile.arm64 @@ -0,0 +1,28 @@ +ARG BASE_PLATFORM=linux/arm64 +FROM --platform=$BASE_PLATFORM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV PATH="/usr/src/app/.venv/bin:${PATH}" + +WORKDIR /usr/src/app + +# Install packages +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + libportaudio2 \ + libsndfile1 \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +RUN python3 -m pip install --no-cache-dir --break-system-packages uv + +COPY pyproject.toml README.md LICENSE MANIFEST.in ./ +RUN uv sync --python /usr/bin/python3 --no-install-project --no-dev + +COPY . . +RUN uv sync --python /usr/bin/python3 --no-dev +RUN python -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..48999a3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2024] [The HuggingFace Inc. team] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..054ec2d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +prune tests +global-exclude __pycache__ +global-exclude *.py[cod] +global-exclude .DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..a717760 --- /dev/null +++ b/README.md @@ -0,0 +1,606 @@ +
+
 
+ + +# Speech To Speech: Build voice agents with open-source models + +[![PyPI](https://img.shields.io/pypi/v/speech-to-speech)](https://pypi.org/project/speech-to-speech/) +[![Python](https://img.shields.io/pypi/pyversions/speech-to-speech)](https://pypi.org/project/speech-to-speech/) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue)](./LICENSE) + +
+ +A low-latency, fully modular voice-agent pipeline: **VAD -> STT -> LLM -> TTS**, exposed through an **OpenAI Realtime-compatible WebSocket API**. Every component is swappable. The LLM slot speaks OpenAI-compatible protocols, so you can point it at a hosted provider, at [HF Inference Providers](https://huggingface.co/inference-providers), or at a vLLM or llama.cpp server on your own hardware for a fully local, fully open stack. + +This pipeline runs in production as the conversation backend for thousands of [Reachy Mini](https://huggingface.co/blog/reachy-mini) robots. + +

+ + + + Switching an OpenAI Realtime client endpoint from hosted OpenAI to a self-hosted speech-to-speech server + +

+ +## Quickstart + +```bash +pip install speech-to-speech +export OPENAI_API_KEY=... +speech-to-speech +``` + +This starts an OpenAI Realtime-compatible server at `ws://localhost:8765/v1/realtime` using Parakeet TDT for local STT, an OpenAI-compatible LLM, and Qwen3-TTS for local speech output. + +From a source checkout, talk to it from a second terminal: + +```bash +python scripts/listen_and_play_realtime.py --host 127.0.0.1 --port 8765 +``` + +Prefer to keep the LLM on your own machine? Serve Gemma 4 with llama.cpp: + +```bash +llama-server -hf ggml-org/gemma-4-E4B-it-GGUF -np 2 -c 65536 -fa on --swa-full +``` + +Then point the OpenAI-compatible LLM backend at it: + +```bash +speech-to-speech \ + --model_name "ggml-org/gemma-4-E4B-it-GGUF" \ + --responses_api_base_url "http://127.0.0.1:8080/v1" \ + --responses_api_api_key "" +``` + +Any OpenAI Realtime-compatible client can connect. See [Realtime API](#realtime-api) for the protocol and [LLM backends](#llm-backends) for provider and local-server options. + +## Index + +* [How it works](#how-it-works) +* [Installation](#installation) +* [Supported components](#supported-components) +* [Run modes](#run-modes) +* [Realtime API](#realtime-api) +* [LLM backends](#llm-backends) +* [Multi-language support](#multi-language-support) +* [Pocket TTS](#pocket-tts) +* [CLI reference](#cli-reference) +* [Contributing](#contributing) +* [Star history](#star-history) +* [Citations](#citations) + +## How it works + +The pipeline is a cascade of four components, each running in its own thread and connected by queues: + +1. **Voice Activity Detection (VAD)**: [Silero VAD v5](https://github.com/snakers4/silero-vad) detects speech boundaries and turn-taking. +2. **Speech to Text (STT)**: transcribes the user's turn, with optional live partial transcripts. +3. **Language Model (LLM)**: generates the response, streaming text and tool calls. +4. **Text to Speech (TTS)**: synthesizes audio and streams it back to the client. + +Every stage has multiple interchangeable backends, selected via CLI flags. The code is designed for easy modification, with a focus on models available through Transformers and the Hugging Face Hub. + +## Installation + +Requires Python 3.10+. + +```bash +pip install speech-to-speech +``` + +The default install covers the standard realtime path: + +- Parakeet TDT for STT +- OpenAI-compatible API for the language model +- Qwen3-TTS for speech output, using the GGML backend by default on non-macOS platforms and `mlx-audio` on Apple Silicon +- local audio and realtime server modes + +macOS and non-macOS dependencies are resolved automatically via platform markers in `pyproject.toml`. + +### CUDA Note for Qwen3-TTS + +On Linux, the Qwen3-TTS GGML backend comes from `faster-qwen3-tts[ggml]`. Its default `qwentts-cpp-python` wheel on PyPI targets CUDA 12.8. If your machine does not have the CUDA 12 runtime that wheel expects, install the matching wheel from the Hugging Face wheelhouse before installing `speech-to-speech`: + +```bash +# CUDA 13.x +pip install "qwentts-cpp-python==0.3.1+cu130" \ + -f https://huggingface.co/datasets/andito/qwentts-cpp-python-wheels/tree/main/whl/cu130 + +# CUDA 12.4 +pip install "qwentts-cpp-python==0.3.1+cu124" \ + -f https://huggingface.co/datasets/andito/qwentts-cpp-python-wheels/tree/main/whl/cu124 + +# CPU-only fallback +pip install "qwentts-cpp-python==0.3.1+cpu" \ + -f https://huggingface.co/datasets/andito/qwentts-cpp-python-wheels/tree/main/whl/cpu + +pip install speech-to-speech +``` + +To use the previous CUDA-graphs implementation instead of GGML, pass `--qwen3_tts_backend torch`. + +### Optional Backends + +Extra backends are installed with pip extras: + +```bash +pip install "speech-to-speech[kokoro]" # Kokoro-82M TTS on non-macOS +pip install "speech-to-speech[pocket]" # Pocket TTS +pip install "speech-to-speech[chattts]" # ChatTTS +pip install "speech-to-speech[facebook-mms]" # MMS TTS +pip install "speech-to-speech[faster-whisper]" # Faster Whisper STT +pip install "speech-to-speech[whisper-mlx]" # Lightning Whisper MLX STT on macOS +pip install "speech-to-speech[paraformer]" # Paraformer STT through FunASR +pip install "speech-to-speech[mlx-lm]" # mlx-vlm support for vision models on macOS +``` + +Deprecated implementations, including MeloTTS, live in [`archive/`](./archive) and are no longer wired into the CLI. + +**Note on DeepFilterNet:** DeepFilterNet, used for optional audio enhancement in VAD, requires `numpy<2` and conflicts with Pocket TTS, which requires `numpy>=2`. Install it manually only in environments where you are not using Pocket TTS. + +### From Source + +```bash +git clone https://github.com/huggingface/speech-to-speech.git +cd speech-to-speech +uv sync +``` + +This installs the package in editable mode and makes the `speech-to-speech` CLI available. + +## Supported Components + +| Component | Backend | Platforms | Install | +|---|---|---|---| +| VAD | [Silero VAD v5](https://github.com/snakers4/silero-vad) | all | built-in | +| STT | [Parakeet TDT](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) (default) | CUDA / CPU through nano-parakeet, Apple Silicon through MLX | built-in | +| STT | [Whisper](https://huggingface.co/docs/transformers/en/model_doc/whisper) through Transformers | CUDA / CPU | built-in | +| STT | [Faster Whisper](https://github.com/SYSTRAN/faster-whisper) | CUDA / CPU | `faster-whisper` | +| STT | [Lightning Whisper MLX](https://github.com/mustafaaljadery/lightning-whisper-mlx) | Apple Silicon | `whisper-mlx` | +| STT | [MLX Audio Whisper](https://github.com/huggingface/mlx-audio) | Apple Silicon | built-in on macOS | +| STT | [Paraformer](https://github.com/modelscope/FunASR) | CUDA / CPU | `paraformer` | +| LLM | OpenAI-compatible API (`responses-api`, `chat-completions`) | hosted providers or self-hosted servers | built-in | +| LLM | [Transformers](https://huggingface.co/models?pipeline_tag=text-generation&sort=trending) | CUDA / CPU | built-in | +| LLM | [mlx-lm](https://github.com/ml-explore/mlx-lm) | Apple Silicon | built-in on macOS | +| TTS | [Qwen3-TTS](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) (default) | GGML / CUDA on Linux, mlx-audio on macOS | built-in | +| TTS | [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M) | CUDA / CPU, Apple Silicon | `kokoro` on non-macOS; built-in on macOS | +| TTS | [Pocket TTS](https://github.com/kyutai-labs/pocket-tts) | CPU / CUDA | `pocket` | +| TTS | [ChatTTS](https://github.com/2noise/ChatTTS) | CUDA / CPU | `chattts` | +| TTS | [MMS TTS](https://huggingface.co/docs/transformers/model_doc/mms) | CUDA / CPU | `facebook-mms` | + +Select implementations with `--stt`, `--llm_backend`, and `--tts`. Run `speech-to-speech -h` for exact values and backend-specific flags. + +## Run Modes + +| Mode | Transport | Use it when | +|---|---|---| +| `realtime` (default) | WebSocket, OpenAI Realtime protocol at `/v1/realtime` | You are building an app or device against a standard voice API. | +| `local` | Your machine's microphone and speakers | You want to talk to the pipeline directly, no client needed. | +| `websocket` | Raw PCM over WebSocket | You want a minimal custom client without the Realtime protocol. | +| `socket` | Raw PCM over TCP | Models run on a remote server, with a simple microphone/playback client. | + +### Realtime Server + +```bash +export OPENAI_API_KEY=... +speech-to-speech +``` + +This is equivalent to: + +```bash +speech-to-speech \ + --thresh 0.6 \ + --stt parakeet-tdt \ + --llm_backend responses-api \ + --tts qwen3 \ + --qwen3_tts_model_name Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ + --qwen3_tts_speaker Aiden \ + --qwen3_tts_language auto \ + --qwen3_tts_backend ggml \ + --qwen3_tts_non_streaming_mode True \ + --qwen3_tts_mlx_quantization 6bit \ + --model_name gpt-5.4-mini \ + --chat_size 30 \ + --responses_api_stream \ + --enable_live_transcription \ + --mode realtime +``` + +The default model is `gpt-5.4-mini` through the OpenAI Responses API. Override it with `--model_name`, and set `--responses_api_base_url` for another OpenAI-compatible provider or server. + +### Local Mac + +```bash +speech-to-speech --local_mac_optimal_settings +``` + +Optionally with a specific LLM: + +```bash +speech-to-speech \ + --local_mac_optimal_settings \ + --model_name mlx-community/Qwen3-4B-Instruct-2507-bf16 +``` + +This setting: + +- Adds `--device mps` to use MPS for all models. +- Sets Parakeet TDT for STT. +- Sets MLX LM as the LLM backend. +- Sets Qwen3-TTS for TTS, using `mlx-audio` with the `6bit` MLX variant by default. +- Sets `--mode local`. + +`--tts pocket` and `--tts kokoro` are also valid on macOS. + +To compare the MLX quantization variants locally: + +```bash +python scripts/benchmark_tts.py \ + --handlers qwen3 \ + --iterations 3 \ + --qwen3_mlx_quantizations bf16 4bit 6bit 8bit +``` + +### WebSocket + +1. Run the pipeline in WebSocket mode: + + ```bash + speech-to-speech --mode websocket --ws_host 0.0.0.0 --ws_port 8765 + ``` + +2. Connect from your client at `ws://:8765`. Send raw audio bytes as 16 kHz, int16, mono PCM and receive generated audio bytes back. + +### TCP Socket + +TCP socket mode is intentionally minimal. It streams raw PCM audio, but does not provide the full Realtime API feature set, including interruption handling, live transcript events, or tool-call events. + +1. Run the pipeline on the server: + + ```bash + speech-to-speech --mode socket --recv_host 0.0.0.0 --send_host 0.0.0.0 + ``` + +2. Run the client locally to handle microphone input and playback: + + ```bash + python scripts/listen_and_play.py --host + ``` + +### Docker + +Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html), then: + +```bash +docker compose up +``` + +The compose file starts a llama.cpp server with Gemma 4, starts the TCP socket server, and exposes ports `8080`, `12345`, and `12346`. + +## Realtime API + +Realtime mode streams audio over a WebSocket using the OpenAI Realtime protocol, with live transcription and low-latency turn-taking. The server exposes `/v1/realtime`, and any OpenAI Realtime-compatible client can connect: + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:8765/v1", + websocket_base_url="ws://localhost:8765/v1", + api_key="not-needed", +) + +with client.realtime.connect(model="local") as conn: + conn.send( + { + "type": "session.update", + "session": { + "type": "realtime", + "instructions": "You are a helpful assistant.", + "audio": { + "input": { + "turn_detection": { + "type": "server_vad", + "interrupt_response": True, + } + } + }, + }, + } + ) + + for event in conn: + print(event.type) +``` + +The server implements the core Realtime event set: `input_audio_buffer.append`, `session.update`, `conversation.item.create`, `response.create`, and `response.cancel` inbound; speech start/stop, streaming transcription, audio deltas, tool calls, and `response.done` outbound. The full event reference, architecture, and design details live in the [Realtime Engine README](./src/speech_to_speech/api/openai_realtime/README.md). + +## LLM Backends + +The LLM is the most compute-intensive and highest-latency component in the pipeline. A single forward pass through a large model can dominate end-to-end response time, so choosing the right backend for your hardware and latency budget matters. The pipeline supports: + +- **Local inference**: `transformers` on CUDA / CPU and `mlx-lm` on Apple Silicon. +- **Self-hosted servers**: `responses-api` and `chat-completions` can point at a local [vLLM](https://github.com/vllm-project/vllm) or [llama.cpp](https://github.com/ggerganov/llama.cpp) server. +- **Provider APIs**: the same backends work with OpenAI, [HF Inference Providers](https://huggingface.co/inference-providers), [OpenRouter](https://openrouter.ai), and other OpenAI-compatible providers. + +Two API backends are available, sharing the same `--responses_api_*` connection flags: + +- `--llm_backend responses-api` (default) targets `/v1/responses`. +- `--llm_backend chat-completions` targets `/v1/chat/completions`. + +The examples below pair Parakeet TDT for local STT and Qwen3-TTS for local TTS with different LLM backends. + +### Responses API Backend + +Works with any provider or server that implements the OpenAI Responses API. Point `--responses_api_base_url` at the endpoint and set `--model_name` accordingly: + +| Provider / server | `--responses_api_base_url` | `--responses_api_api_key` | +|---|---|---| +| OpenAI | omit, uses OpenAI default | `$OPENAI_API_KEY` | +| HF Inference Providers | `https://router.huggingface.co/v1` | `$HF_TOKEN` | +| OpenRouter | `https://openrouter.ai/api/v1` | `$OPENROUTER_API_KEY` | +| vLLM | `http://localhost:8000/v1` | omit or any string | +| llama.cpp | `http://127.0.0.1:8080/v1` | empty string | + +```bash +# OpenAI +speech-to-speech \ + --mode local \ + --stt parakeet-tdt \ + --llm_backend responses-api \ + --tts qwen3 \ + --qwen3_tts_mlx_quantization 6bit \ + --model_name "gpt-4o-mini" \ + --responses_api_api_key "$OPENAI_API_KEY" \ + --responses_api_stream \ + --enable_live_transcription +``` + +```bash +# HF Inference Providers: Qwen3.5-9B via Together +speech-to-speech \ + --mode local \ + --stt parakeet-tdt \ + --llm_backend responses-api \ + --tts qwen3 \ + --qwen3_tts_mlx_quantization 6bit \ + --model_name "Qwen/Qwen3.5-9B:together" \ + --responses_api_base_url "https://router.huggingface.co/v1" \ + --responses_api_api_key "$HF_TOKEN" \ + --responses_api_stream \ + --enable_live_transcription +``` + +```bash +# HF Inference Providers: GPT-oss-20B via Groq +speech-to-speech \ + --stt parakeet-tdt \ + --llm_backend responses-api \ + --tts qwen3 \ + --qwen3_tts_mlx_quantization 6bit \ + --model_name "openai/gpt-oss-20b:groq" \ + --responses_api_base_url "https://router.huggingface.co/v1" \ + --responses_api_api_key "$HF_TOKEN" \ + --responses_api_stream \ + --enable_live_transcription +``` + +### Chat Completions Backend + +Identical configuration to `responses-api`, reusing the same `--responses_api_*` connection flags, but talks to `/v1/chat/completions` instead of `/v1/responses`. Prefer it when: + +- the provider ignores `chat_template_kwargs.enable_thinking` on the Responses path and needs a `reasoning_effort` knob to suppress reasoning, or +- the server's Responses streaming tool-call path is unreliable, while its Chat Completions tool-call streaming is solid. This is useful for some vLLM builds; see [#312](https://github.com/huggingface/speech-to-speech/issues/312). + +Add `--responses_api_reasoning_effort none` to disable reasoning on providers where the chat-template flag has no effect: + +```bash +# vLLM serving a Qwen model with tool calling +speech-to-speech \ + --mode realtime \ + --stt parakeet-tdt \ + --llm_backend chat-completions \ + --tts qwen3 \ + --model_name "Qwen/Qwen3-4B-Instruct-2507" \ + --responses_api_base_url "http://localhost:8000/v1" \ + --responses_api_stream +``` + +```bash +# Gemma 4 31B via the HF router on Cerebras, with reasoning disabled for low voice latency +speech-to-speech \ + --mode realtime \ + --stt parakeet-tdt \ + --llm_backend chat-completions \ + --tts qwen3 \ + --model_name "google/gemma-4-31B-it:cerebras" \ + --responses_api_base_url "https://router.huggingface.co/v1" \ + --responses_api_api_key "$HF_TOKEN" \ + --responses_api_reasoning_effort none \ + --responses_api_stream +``` + +### Fully Local + +Run the LLM in a separate llama.cpp process for the lowest-friction fully local setup, as shown in the [Reachy Mini local conversation guide](https://huggingface.co/blog/local-reachy-mini-conversation): + +```bash +# Terminal 1: llama.cpp serving Gemma 4 +llama-server -hf ggml-org/gemma-4-E4B-it-GGUF -np 2 -c 65536 -fa on --swa-full +``` + +```bash +# Terminal 2: speech-to-speech using that local LLM server +speech-to-speech \ + --mode realtime \ + --stt parakeet-tdt \ + --llm_backend responses-api \ + --tts qwen3 \ + --model_name "ggml-org/gemma-4-E4B-it-GGUF" \ + --responses_api_base_url "http://127.0.0.1:8080/v1" \ + --responses_api_api_key "" \ + --responses_api_stream \ + --enable_live_transcription +``` + +You can use `--mode local` instead of `--mode realtime` when you want to talk through the machine running the server directly. In-process local backends are still available with `--llm_backend mlx-lm` on Apple Silicon or `--llm_backend transformers` on CUDA / CPU. + +## Multi-Language Support + +Language coverage depends on the STT and TTS backends you pick, not on the pipeline itself: + +| Component | Backend | Languages | +|---|---|---| +| STT | Parakeet TDT (default) | 25 European languages | +| STT | Whisper / Whisper MLX / Faster Whisper | Broad multilingual coverage, depending on the selected Whisper checkpoint | +| STT | Paraformer | Depends on the selected FunASR checkpoint; the default is Chinese-oriented | +| TTS | Qwen3-TTS (default) | Multilingual, with `--qwen3_tts_language auto` by default | +| TTS | Kokoro | Multiple language/voice mappings, depending on backend availability | +| TTS | ChatTTS | English and Chinese | +| TTS | MMS TTS | Broad multilingual coverage through MMS checkpoints | + +Make sure the STT, LLM, and TTS you pair all cover your target language(s). Two usage patterns: + +- **Single language**: set `--language` to the target language code. The default is `en`. +- **Language switching**: set `--language auto`. The STT detects the language of each spoken prompt and forwards it to the LLM. Optionally add `--enable_lang_prompt` to append a "Please reply to my message in ..." instruction. It defaults to `False`; large LLMs usually infer the language from context, but the explicit instruction can help smaller models. + +Automatic language detection: + +```bash +speech-to-speech \ + --stt parakeet-tdt \ + --language auto \ + --llm_backend mlx-lm \ + --model_name "mlx-community/Qwen3-4B-Instruct-2507-bf16" +``` + +A single non-English language, Chinese in this example: + +```bash +speech-to-speech \ + --stt whisper-mlx \ + --stt_model_name large-v3 \ + --language zh \ + --llm_backend mlx-lm \ + --model_name mlx-community/Qwen3-4B-Instruct-2507-bf16 +``` + +Both commands also work on top of `--local_mac_optimal_settings`; explicit `--stt` flags override the defaults it sets. + +## Pocket TTS + +Pocket TTS from Kyutai Labs provides streaming TTS with voice cloning: + +```bash +speech-to-speech \ + --tts pocket \ + --pocket_tts_voice jean \ + --pocket_tts_device cpu +``` + +Available voice presets: `alba`, `marius`, `javert`, `jean`, `fantine`, `cosette`, `eponine`, `azelma`. Custom voice files and Hugging Face paths also work. + +## CLI Reference + +References for all CLI arguments live in the [arguments classes](./src/speech_to_speech/arguments_classes) and in `speech-to-speech -h`. + +### Module-Level Parameters + +See [ModuleArguments](./src/speech_to_speech/arguments_classes/module_arguments.py). It allows setting: + +- a common `--device`, if every part should run on the same device +- `--mode`: `realtime` (default), `local`, `socket`, or `websocket` +- STT implementation (`--stt`) +- LLM backend (`--llm_backend`: `transformers`, `mlx-lm`, `responses-api`, or `chat-completions`) +- TTS implementation (`--tts`) +- logging level +- realtime pipeline pool size (`--num_pipelines`) + +### VAD Parameters + +See [VADHandlerArguments](./src/speech_to_speech/arguments_classes/vad_arguments.py). Notable options: + +- `--thresh`: threshold value to trigger voice activity detection. +- `--min_speech_ms`: minimum duration of detected voice activity to be considered speech. +- `--min_speech_continuation_ms`: sustain-bar hysteresis threshold for speech that continues a reopenable soft-ended, uncommitted turn within the reopen window. The default and recommended pairing is `--min_speech_ms 384 --min_speech_continuation_ms 192`. +- `--min_silence_ms`: minimum length of silence intervals for segmenting speech. Default is 64 ms. +- `--short_segment_merge_ms`: optional merge window for stitching adjacent VAD segments that are each shorter than `--min_speech_ms`. +- `--unanswered_reopen_ms`: sanity cap on how long a soft-ended speculative turn that has not yet received any assistant output stays reopenable. + +### STT, LLM, and TTS Parameters + +`model_name`, `torch_dtype`, and `device` are exposed for each STT, LLM, and TTS implementation. STT and TTS parameters use the handler prefix, for example `--stt_model_name` or `--qwen3_tts_device`. LLM model selection and chat settings are shared across backends via unprefixed flags, for example `--model_name` and `--chat_size`; backend-specific flags use the `responses_api_` prefix for the `responses-api` and `chat-completions` backends and the `llm_` prefix for local backends. + +For example: + +```bash +# Local transformers/mlx-lm backend +--model_name google/gemma-2b-it + +# OpenAI-compatible backend +--llm_backend responses-api --model_name deepseek-chat --responses_api_base_url https://api.deepseek.com +``` + +### Generation Parameters + +Other generation parameters can be set using the handler prefix plus `_gen_`, for example `--stt_gen_max_new_tokens 128` or `--llm_gen_temperature 0.7`. Parameters not yet exposed can be added to the relevant arguments class. + +## Contributing + +Issues and PRs are welcome. Good starting points are the [open issues](https://github.com/huggingface/speech-to-speech/issues). For larger changes, open an issue first to discuss the approach. + +For local development: + +```bash +uv sync +pytest +ruff check +``` + +## Star History + +[![Star History Chart](assets/star-history.svg)](https://github.com/huggingface/speech-to-speech/stargazers) + +## Citations + +If you use this pipeline, please also cite the component models you run. The defaults are: + +### Silero VAD + +```bibtex +@misc{SileroVAD, + author = {Silero Team}, + title = {Silero VAD: pre-trained enterprise-grade Voice Activity Detector (VAD), Number Detector and Language Classifier}, + year = {2021}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/snakers4/silero-vad}}, + email = {hello@silero.ai} +} +``` + +### Parakeet TDT + +```bibtex +@misc{parakeet-tdt, + author = {NVIDIA}, + title = {Parakeet TDT 0.6B v3}, + publisher = {Hugging Face}, + howpublished = {\url{https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3}} +} +``` + +### Qwen3-TTS + +```bibtex +@misc{qwen3-tts, + author = {Qwen Team}, + title = {Qwen3-TTS}, + publisher = {Hugging Face}, + howpublished = {\url{https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}} +} +``` + +Citations for optional backends such as Kokoro, Pocket TTS, ChatTTS, Whisper variants, Paraformer, and MMS live in the respective [component READMEs](./src/speech_to_speech). diff --git a/archive/README.md b/archive/README.md new file mode 100644 index 0000000..86db6f6 --- /dev/null +++ b/archive/README.md @@ -0,0 +1,11 @@ +# Archived Models + +This directory stores sunset model implementations that are kept in-repo but are no longer wired into `s2s_pipeline.py`. + +- STT: `moonshine` -> `archive/STT/moonshine_handler.py` +- TTS: `parler` -> `archive/TTS/parler_handler.py` +- TTS: `melo` -> `archive/TTS/melo_handler.py` +- Legacy args: `archive/arguments_classes/parler_tts_arguments.py` +- Legacy args: `archive/arguments_classes/melo_tts_arguments.py` + +These models are also removed from default requirements. If you want to run them manually, install their dependencies separately. diff --git a/archive/STT/__init__.py b/archive/STT/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/archive/STT/moonshine_handler.py b/archive/STT/moonshine_handler.py new file mode 100644 index 0000000..ce6becd --- /dev/null +++ b/archive/STT/moonshine_handler.py @@ -0,0 +1,72 @@ +import os + +os.environ['KERAS_BACKEND'] = 'torch' + +import logging + +import moonshine +import torch +from rich.console import Console + +from speech_to_speech.baseHandler import BaseHandler +from speech_to_speech.pipeline.messages import VADAudio + +logger = logging.getLogger(__name__) +console = Console() + + +class MoonshineSTTHandler(BaseHandler[VADAudio]): + """ + Handles the Speech To Text generation using a Moonshine model. + """ + + def setup( + self, + model_name="moonshine/base", + torch_dtype="float16", + gen_kwargs={}, + ): + self.torch_dtype = getattr(torch, torch_dtype) + self.gen_kwargs = gen_kwargs + + self.tokenizer = moonshine.load_tokenizer() + self.model = moonshine.load_model(model_name) + + self.warmup() + + def warmup(self): + logger.info(f"Warming up {self.__class__.__name__}") + + n_steps = 2 + dummy_input = torch.randn( + (1, 16000), + dtype=self.torch_dtype, + ) + + if torch.cuda.is_available(): + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + start_event.record() + + for _ in range(n_steps): + _ = self.model.generate(dummy_input) + + if torch.cuda.is_available(): + end_event.record() + torch.cuda.synchronize() + + logger.info( + f"{self.__class__.__name__}: warmed up! time: {start_event.elapsed_time(end_event) * 1e-3:.3f} s" + ) + + def process(self, vad_audio: VADAudio): + logger.debug("infering moonshine...") + + pred_ids = self.model.generate(vad_audio.audio[None, :]) + pred_text = self.tokenizer.decode_batch(pred_ids)[0] + + logger.debug("finished whisper inference") + console.print(f"[yellow]USER: {pred_text}") + + yield (pred_text, "en") diff --git a/archive/TTS/__init__.py b/archive/TTS/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/archive/TTS/melo_handler.py b/archive/TTS/melo_handler.py new file mode 100644 index 0000000..6ac5d10 --- /dev/null +++ b/archive/TTS/melo_handler.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import logging +from threading import Event +from typing import Any, Iterator + +import librosa +import numpy as np +import torch +from melo.api import TTS +from rich.console import Console + +from speech_to_speech.baseHandler import BaseHandler +from speech_to_speech.pipeline.cancel_scope import CancelScope +from speech_to_speech.pipeline.handler_types import TTSIn, TTSOut +from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse +from speech_to_speech.pipeline.speculative_turns import SpeculativeTurnTracker + +logger = logging.getLogger(__name__) + +console = Console() + +WHISPER_LANGUAGE_TO_MELO_LANGUAGE = { + "en": "EN", + "fr": "FR", + "es": "ES", + "zh": "ZH", + "ja": "JP", + "ko": "KR", +} + +WHISPER_LANGUAGE_TO_MELO_SPEAKER = { + "en": "EN-BR", + "fr": "FR", + "es": "ES", + "zh": "ZH", + "ja": "JP", + "ko": "KR", +} + + +class MeloTTSHandler(BaseHandler[TTSIn, TTSOut]): + def setup( + self, + should_listen: Event, + device: str = "mps", + language: str = "en", + speaker_to_id: str = "en", + gen_kwargs: dict[str, Any] = {}, # Unused + blocksize: int = 512, + cancel_scope: CancelScope | None = None, + speculative_turns: SpeculativeTurnTracker | None = None, + ) -> None: + self.should_listen = should_listen + self.cancel_scope = cancel_scope + self.speculative_turns = speculative_turns + self.device = device + self.language = language + self.model = TTS(language=WHISPER_LANGUAGE_TO_MELO_LANGUAGE[self.language], device=device) + self.speaker_id = self.model.hps.data.spk2id[WHISPER_LANGUAGE_TO_MELO_SPEAKER[speaker_to_id]] + self.blocksize = blocksize + self._initial_language = self.language + self.warmup() + + def warmup(self) -> None: + logger.info(f"Warming up {self.__class__.__name__}") + _ = self.model.tts_to_file("text", self.speaker_id, quiet=True) + + def process(self, tts_input: TTSIn) -> Iterator[TTSOut]: + if isinstance(tts_input, EndOfResponse): + yield AUDIO_RESPONSE_DONE + return + + speculative_turns = getattr(self, "speculative_turns", None) + if speculative_turns and not speculative_turns.is_latest( + tts_input.turn_id, + tts_input.turn_revision, + ): + logger.debug("Dropping stale TTS input for turn=%s rev=%s", tts_input.turn_id, tts_input.turn_revision) + return + + gen = self.cancel_scope.generation if self.cancel_scope else None + language_code = tts_input.language_code + text = tts_input.text + + console.print(f"[green]ASSISTANT: {text}") + + if language_code is not None and self.language != language_code: + try: + self.model = TTS( + language=WHISPER_LANGUAGE_TO_MELO_LANGUAGE[language_code], + device=self.device, + ) + self.speaker_id = self.model.hps.data.spk2id[WHISPER_LANGUAGE_TO_MELO_SPEAKER[language_code]] + self.language = language_code + except KeyError: + console.print(f"[red]Language {language_code} not supported by Melo. Using {self.language} instead.") + + if self.device == "mps": + import time + + start = time.time() + torch.mps.synchronize() # Waits for all kernels in all streams on the MPS device to complete. + torch.mps.empty_cache() # Frees all memory allocated by the MPS device. + _ = time.time() - start # Removing this line makes it fail more often. I'm looking into it. + + try: + audio_chunk = self.model.tts_to_file(text, self.speaker_id, quiet=True) + except (AssertionError, RuntimeError) as e: + logger.error(f"Error in MeloTTSHandler: {e}") + audio_chunk = np.array([]) + if len(audio_chunk) == 0: + return + audio_chunk = librosa.resample(audio_chunk, orig_sr=44100, target_sr=16000) + audio_chunk = (audio_chunk * 32768).astype(np.int16) + for i in range(0, len(audio_chunk), self.blocksize): + if gen is not None and self.cancel_scope is not None and self.cancel_scope.is_stale(gen): + logger.info("TTS generation cancelled (interruption)") + return + yield np.pad( + audio_chunk[i : i + self.blocksize], + (0, self.blocksize - len(audio_chunk[i : i + self.blocksize])), + ) + + def on_session_end(self) -> None: + if self.language != self._initial_language: + self.language = self._initial_language + self.model = TTS(language=WHISPER_LANGUAGE_TO_MELO_LANGUAGE[self.language], device=self.device) + self.speaker_id = self.model.hps.data.spk2id[WHISPER_LANGUAGE_TO_MELO_SPEAKER[self.language]] + logger.debug("Melo TTS session state reset") diff --git a/archive/TTS/parler_handler.py b/archive/TTS/parler_handler.py new file mode 100644 index 0000000..1580a76 --- /dev/null +++ b/archive/TTS/parler_handler.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import logging +from threading import Event, Thread +from time import perf_counter +from typing import Any, Optional + +import librosa +import numpy as np +import torch +from parler_tts import ParlerTTSForConditionalGeneration, ParlerTTSStreamer +from rich.console import Console +from transformers import ( + AutoTokenizer, +) +from transformers.utils.import_utils import ( + is_flash_attn_2_available, +) + +from speech_to_speech.baseHandler import BaseHandler +from speech_to_speech.pipeline.messages import AUDIO_RESPONSE_DONE, EndOfResponse, TTSInput +from speech_to_speech.utils.utils import next_power_of_2 + +torch._inductor.config.fx_graph_cache = True +# mind about this parameter ! should be >= 2 * number of padded prompt sizes for TTS +torch._dynamo.config.cache_size_limit = 15 + +logger = logging.getLogger(__name__) + +console = Console() + + +if not is_flash_attn_2_available() and torch.cuda.is_available(): + logger.warn( + """Parler TTS works best with flash attention 2, but is not installed + Given that CUDA is available in this system, you can install flash attention 2 with `uv pip install flash-attn --no-build-isolation`""" + ) + + +WHISPER_LANGUAGE_TO_PARLER_SPEAKER = { + "en": "Jason", + "fr": "Christine", + "es": "Steven", + "de": "Nicole", + "pt": "Sophia", + "pl": "Alex", + "it": "Richard", + "nl": "Mark", +} + + +class ParlerTTSHandler(BaseHandler[TTSInput | EndOfResponse]): + def setup( + self, + should_listen, + model_name="parler-tts/parler-mini-v1-jenny", + device="cuda", + torch_dtype="float16", + compile_mode=None, + gen_kwargs={}, + max_prompt_pad_length=8, + description=( + "Jenny speaks at a slightly slow pace with an animated delivery with clear audio quality." + ), + play_steps_s=1, + blocksize=512, + use_default_speakers_list=True, + cancel_response: Event | None = None, + ): + self.should_listen = should_listen + self.cancel_response = cancel_response + self.device = device + self.torch_dtype = getattr(torch, torch_dtype) + self.gen_kwargs = gen_kwargs + self.compile_mode = compile_mode + self.max_prompt_pad_length = max_prompt_pad_length + self.use_default_speakers_list = use_default_speakers_list + if self.use_default_speakers_list: + description = description.replace("Jenny", "") + + self.speaker = "Jason" + self.description = description + + self.model = ParlerTTSForConditionalGeneration.from_pretrained( + model_name, torch_dtype=self.torch_dtype + ).to(device) + + self.description_tokenizer = AutoTokenizer.from_pretrained(self.model.config.text_encoder._name_or_path) + self.prompt_tokenizer = AutoTokenizer.from_pretrained(model_name) + + + framerate = self.model.audio_encoder.config.frame_rate + self.play_steps = int(framerate * play_steps_s) + self.blocksize = blocksize + + if self.compile_mode not in (None, "default"): + logger.warning( + "Torch compilation modes that captures CUDA graphs are not yet compatible with the TTS part. Reverting to 'default'" + ) + self.compile_mode = "default" + + if self.compile_mode: + self.model.generation_config.cache_implementation = "static" + self.model.forward = torch.compile( + self.model.forward, mode=self.compile_mode, fullgraph=True + ) + + self.warmup() + + def prepare_model_inputs( + self, + prompt, + max_length_prompt=50, + pad=False, + ): + pad_args_prompt = ( + {"padding": "max_length", "max_length": max_length_prompt} if pad else {} + ) + + description = self.description + if self.use_default_speakers_list: + description = self.speaker + " " + self.description + + tokenized_description = self.description_tokenizer( + description, return_tensors="pt" + ).to(self.device) + input_ids = tokenized_description.input_ids + attention_mask = tokenized_description.attention_mask + + tokenized_prompt = self.prompt_tokenizer( + prompt, return_tensors="pt", **pad_args_prompt + ).to(self.device) + prompt_input_ids = tokenized_prompt.input_ids + prompt_attention_mask = tokenized_prompt.attention_mask + + gen_kwargs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "prompt_input_ids": prompt_input_ids, + "prompt_attention_mask": prompt_attention_mask, + **self.gen_kwargs, + } + + return gen_kwargs + + def warmup(self): + logger.info(f"Warming up {self.__class__.__name__}") + + if self.device == "cuda": + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + # 2 warmup steps for no compile or compile mode with CUDA graphs capture + n_steps = 1 if self.compile_mode == "default" else 2 + + if self.device == "cuda": + torch.cuda.synchronize() + start_event.record() + if self.compile_mode: + pad_lengths = [2**i for i in range(2, self.max_prompt_pad_length)] + for pad_length in pad_lengths[::-1]: + model_kwargs = self.prepare_model_inputs( + "dummy prompt", max_length_prompt=pad_length, pad=True + ) + for _ in range(n_steps): + _ = self.model.generate(**model_kwargs) + logger.info(f"Warmed up length {pad_length} tokens!") + else: + model_kwargs = self.prepare_model_inputs("dummy prompt") + for _ in range(n_steps): + _ = self.model.generate(**model_kwargs) + + if self.device == "cuda": + end_event.record() + torch.cuda.synchronize() + logger.info( + f"{self.__class__.__name__}: warmed up! time: {start_event.elapsed_time(end_event) * 1e-3:.3f} s" + ) + + def process(self, tts_input: TTSInput | EndOfResponse): + if isinstance(tts_input, EndOfResponse): + yield AUDIO_RESPONSE_DONE + return + + runtime_config = tts_input.runtime_config + response = tts_input.response + language_code = tts_input.language_code + text = tts_input.text + + voice: Optional[str] = None + if response and response.audio and response.audio.output: + voice = str(response.audio.output.voice) if response.audio.output.voice is not None else None + if not voice and runtime_config: + audio_cfg = runtime_config.session.audio + audio_output = audio_cfg.output if audio_cfg is not None else None + voice = str(audio_output.voice) if audio_output is not None and audio_output.voice else None + if voice: + self.speaker = voice + elif language_code: + self.speaker = WHISPER_LANGUAGE_TO_PARLER_SPEAKER.get(language_code, "Jason") + + console.print(f"[green]ASSISTANT: {text}") + nb_tokens = len(self.prompt_tokenizer(text).input_ids) + + pad_args: dict[str, Any] = {} + if self.compile_mode: + # pad to closest upper power of two + pad_length = next_power_of_2(nb_tokens) + logger.debug(f"padding to {pad_length}") + pad_args["pad"] = True + pad_args["max_length_prompt"] = pad_length + + tts_gen_kwargs = self.prepare_model_inputs( + text, + **pad_args, + ) + + streamer = ParlerTTSStreamer( + self.model, device=self.device, play_steps=self.play_steps + ) + tts_gen_kwargs = {"streamer": streamer, **tts_gen_kwargs} + torch.manual_seed(0) + thread = Thread(target=self.model.generate, kwargs=tts_gen_kwargs) + thread.start() + + pipeline_start = perf_counter() + for i, audio_chunk in enumerate(streamer): + if self.cancel_response and self.cancel_response.is_set(): + logger.info("TTS generation cancelled (interruption)") + return + if i == 0: + logger.info( + f"Time to first audio: {perf_counter() - pipeline_start:.3f}s" + ) + audio_chunk = librosa.resample(audio_chunk, orig_sr=44100, target_sr=16000) + audio_chunk = (audio_chunk * 32768).astype(np.int16) + for i in range(0, len(audio_chunk), self.blocksize): + yield np.pad( + audio_chunk[i : i + self.blocksize], + (0, self.blocksize - len(audio_chunk[i : i + self.blocksize])), + ) + + if not runtime_config: + self.should_listen.set() diff --git a/archive/__init__.py b/archive/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/archive/arguments_classes/__init__.py b/archive/arguments_classes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/archive/arguments_classes/melo_tts_arguments.py b/archive/arguments_classes/melo_tts_arguments.py new file mode 100644 index 0000000..283045b --- /dev/null +++ b/archive/arguments_classes/melo_tts_arguments.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass, field + + +@dataclass +class MeloTTSHandlerArguments: + melo_language: str = field( + default="en", + metadata={"help": "The language of the text to be synthesized. Default is 'EN_NEWEST'."}, + ) + melo_device: str = field( + default="auto", + metadata={"help": "The device to be used for speech synthesis. Default is 'auto'."}, + ) + melo_speaker_to_id: str = field( + default="en", + metadata={"help": "Mapping of speaker names to speaker IDs. Default is ['EN-Newest']."}, + ) diff --git a/archive/arguments_classes/parler_tts_arguments.py b/archive/arguments_classes/parler_tts_arguments.py new file mode 100644 index 0000000..eea37cb --- /dev/null +++ b/archive/arguments_classes/parler_tts_arguments.py @@ -0,0 +1,68 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class ParlerTTSHandlerArguments: + tts_model_name: str = field( + default="parler-tts/parler-mini-v1-jenny", + metadata={ + "help": "The pretrained TTS model to use. Default is 'parler-tts/parler-mini-v1-jenny'." + }, + ) + tts_device: str = field( + default="cuda", + metadata={ + "help": "The device type on which the model will run. Default is 'cuda' for GPU acceleration." + }, + ) + tts_torch_dtype: str = field( + default="float16", + metadata={ + "help": "The PyTorch data type for the model and input tensors. One of `float32` (full-precision), `float16` or `bfloat16` (both half-precision)." + }, + ) + tts_compile_mode: Optional[str] = field( + default=None, + metadata={ + "help": "Compile mode for torch compile. Either 'default', 'reduce-overhead' and 'max-autotune'. Default is None (no compilation)" + }, + ) + tts_gen_min_new_tokens: int = field( + default=64, + metadata={ + "help": "Maximum number of new tokens to generate in a single completion. Default is 64, which corresponds to ~0.74 secs" + }, + ) + tts_gen_max_new_tokens: int = field( + default=1024, + metadata={ + "help": "Maximum number of new tokens to generate in a single completion. Default is 1024, which corresponds to ~12 secs" + }, + ) + description: str = field( + default=( + "Jenny speaks at a slightly slow pace with an animated delivery with clear audio quality." + ), + metadata={ + "help": "Description of the speaker's voice and speaking style to guide the TTS model." + }, + ) + play_steps_s: float = field( + default=1.0, + metadata={ + "help": "The time interval in seconds for playing back the generated speech in steps. Default is 1.0 seconds." + }, + ) + max_prompt_pad_length: int = field( + default=8, + metadata={ + "help": "When using compilation, the prompt as to be padded to closest power of 2. This parameters sets the maximun power of 2 possible." + }, + ) + use_default_speakers_list: bool = field( + default=False, + metadata={ + "help": "Whether to use the default list of speakers or not." + }, + ) diff --git a/assets/star-history.svg b/assets/star-history.svg new file mode 100644 index 0000000..06ce698 --- /dev/null +++ b/assets/star-history.svg @@ -0,0 +1,15 @@ + + + huggingface/speech-to-speech star history + 6,165 stars + + 01.6k3.2k4.8k6.4k8k + Aug 2024Jan 2025May 2025Oct 2025Feb 2026Jul 2026 + + + + diff --git a/chile_female.mp3 b/chile_female.mp3 new file mode 100644 index 0000000..1f14a74 Binary files /dev/null and b/chile_female.mp3 differ diff --git a/chile_female.wav b/chile_female.wav new file mode 100644 index 0000000..90bfa5f Binary files /dev/null and b/chile_female.wav differ diff --git a/demo.zip b/demo.zip new file mode 100644 index 0000000..59b4609 Binary files /dev/null and b/demo.zip differ diff --git a/demo/.dockerignore b/demo/.dockerignore new file mode 100644 index 0000000..88b4b68 --- /dev/null +++ b/demo/.dockerignore @@ -0,0 +1,12 @@ +.git +.gitignore +__pycache__/ +*.pyc +.venv/ +venv/ +.vscode/ +.idea/ +.DS_Store +*.log +docs/ +.claude/ diff --git a/demo/.gitignore b/demo/.gitignore new file mode 100644 index 0000000..f43f76b --- /dev/null +++ b/demo/.gitignore @@ -0,0 +1,21 @@ +# macOS metadata +.DS_Store + +# Editor scratch +.vscode/ +.idea/ +*.swp + +# Static HTTP server logs (we don't ship them) +*.log + +# Python +__pycache__/ +*.pyc +.venv/ +venv/ + +# Local-only replica deploy script (never tracked) +deploy_replica.py + +.env \ No newline at end of file diff --git a/demo/CONTEXT.md b/demo/CONTEXT.md new file mode 100644 index 0000000..b4d980a --- /dev/null +++ b/demo/CONTEXT.md @@ -0,0 +1,108 @@ +# Context glossary + +Canonical terms for this space. A glossary, not a spec — it defines what words mean, +not how anything is built. Keep design/implementation detail in `DESIGN.md` and the +code. + +## Speech-to-speech demo +The product: a voice conversation you have with a model by tapping the orb and +talking. "The demo" and "the space" refer to this same thing. It runs on Hugging +Face's open `speech-to-speech` backend. + +## The pipeline +The ordered path a turn travels, from your voice to the orb's reply. Order is +meaningful — each stage consumes the previous one's output: + +`you speak → VAD → STT → VLM → TTS → orb replies` + +- **VAD** — voice activity detection. Decides *when* you are speaking, so the system + knows a turn has started and ended. Model: silero-vad. +- **STT** — speech to text. Transcribes your speech into words. Model: + nvidia/parakeet-tdt-1.1b. +- **VLM** — the vision-language model that composes the reply. Served via Cerebras. + Model: google/gemma-4-31B-it. +- **TTS** — text to speech. Speaks the reply back in the chosen voice. Model: + Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice. + +## Builder +A Hugging Face user credited with making the space, shown by HF username. Current +builders: tfrere, A-Mahla and andito. Distinct from the *models'* authors (nvidia, google, +Qwen, snakers4), who are credited per pipeline stage. + +## Powered by +The infrastructure running the pipeline, named in the about panel: Hugging Face +Inference Endpoints (hosting) and Cerebras (LLM inference). Distinct from "built by" +(the people) and from the model authors (who trained each model). + +## Tool +A function the model can call mid-conversation to do something the pipeline +can't do on its own (look something up, look through the camera). Tools are +declared to the backend in the session config; the model decides when to call +one. Distinct from the *pipeline stages* (VAD/STT/VLM/TTS), which always run. + +## Tool executor +The client-side component that runs a tool when the model calls it and returns +the result to the backend, so the model can speak the answer. It is the missing +half of the round-trip: the backend already emits the call, the executor runs it +and replies. Distinct from the *tool* itself (the thing being run). + +## Web search tool +A tool that looks something up on the web for the model. The model calls it with +a query; the tool executor forwards the query to the search proxy and returns the +results as the tool result. Activates only when a search key is available. + +## Camera snapshot tool +A tool that lets the model look through the user's webcam. While enabled, a live +self-view is shown in the page (bottom-left); when the model calls the tool, the +executor captures a frame and sends it to the model as an image so the VLM can +see it. Distinct from the *preview* (what the user sees) and the *snapshot* (the +single frame sent to the model). + +## Search proxy +The same-origin server route (`/search`) that holds the search key and calls the +external search provider on the client's behalf, so the key never reaches the +browser. Lives in the same container as the page. Distinct from the *s2s backend* +(the separate load-balanced speech-to-speech service). + +## Tools panel +The dialog opened from the "Tools" button in the top-right, holding one switch per +tool (and the web-search key status). Turning a switch on/off declares or removes +that tool on the live session. Distinct from *Settings* (connection, voice, +instructions) and the *About panel* (project info). + +## Identity block +The top-left corner of the topbar (replacing the old wordmark): the demo name, a +one-line blurb, and the "powered by" / "built by" credits, shown directly rather +than hidden behind a click. + +## About panel +The popup opened from the (i) icon to the right of the identity block. Holds the +general introduction to the speech-to-speech project (with a repo link) and the +pipeline. Identity itself now lives in the corner, not here. + +## Queue +The line of users waiting for a free conversation slot when every compute is +busy. You join the queue instead of being turned away; you leave it by reaching +the front or by giving up. Distinct from a *session* (an actual live +conversation) — being in the queue is not yet talking, and time spent waiting +never counts against your usage limit. + +## Ticket +Your held place in the queue. Created when you join, it is what the demo checks +to tell you your position and to notice if you have left. A ticket is not a +session: it only promises a spot in line, not a compute. + +## Position +How many people are ahead of you in the queue, shown while you wait ("You're #3 +in line"). It only ever counts down. Distinct from an *estimated wait* — the +demo shows position, never a time, because wait time is unpredictable. + +## Claim +The moment you reach the front and a free slot becomes yours — the queue hands +off to a real session and the conversation begins. This is also the point where +your usage limit first starts to matter (never while waiting). + +## At capacity +The state where the queue itself is full, so new users can't even join the line +and are asked to try again shortly. Distinct from simply *busy* (all computes +taken but the queue still has room to wait in). diff --git a/demo/DESIGN.md b/demo/DESIGN.md new file mode 100644 index 0000000..d0ca548 --- /dev/null +++ b/demo/DESIGN.md @@ -0,0 +1,205 @@ +# Design language + +The reference for keeping this app visually coherent as it grows. Read it before +touching `style.css`, `index.html`, or any DOM-building code in `main.js`. Every +rule here is already live in the codebase — this file explains the *why* so changes +extend the system instead of drifting from it. + +--- + +## The thesis: color belongs to the voice + +This is a voice app. The one thing in the room that should have color is the thing +that is talking. So: + +- **The orb** carries saturated color. It glows, and the glow's hue changes with + conversational state. +- **Everything else is monochrome** — a precise cool-grey dark canvas. Surfaces, + borders, buttons, panels, the transcript: all greyscale. +- The **only** exceptions are tiny *role echoes* (a one-word mono label, a small + icon) that borrow the orb's state hue so the transcript reads in the same color + language the orb speaks. They are accents the size of a word, never fills. +- **Brand logos keep their own color.** The Hugging Face mark (`#ffd21e`) and the + Cerebras mark (`#f15a29`) render in their brand colors in the identity credits and + the about panel — a deliberate, owner-approved exception. It applies to those two + logos only; do not generalize it to other chrome. + +If you find yourself adding a tinted background, a colored border, or a bright +button anywhere outside the orb, stop — that color almost certainly belongs to the +orb instead, or shouldn't exist. + +--- + +## Color tokens + +All defined in `:root` in `style.css`. Use the variables, never raw hex in rules. + +### Canvas (the monochrome world) +| Token | Value | Use | +|---|---|---| +| `--bg` | `#0a0b10` | Page background (a cool near-black) | +| `--bg-elev` | `#13151c` | Raised surfaces: bubbles, panels, icon buttons | +| `--bg-elev-2` | `#1b1e29` | Surfaces on surfaces: history bodies, inputs | +| `--border` | `rgba(255,255,255,.08)` | Default hairline | +| `--border-strong` | `rgba(255,255,255,.16)` | Emphasised hairline | +| `--text` | `#f5f6fa` | Primary text; also the *primary button* fill | +| `--text-dim` | `rgba(245,246,250,.65)` | Secondary text | +| `--text-faint` | `rgba(245,246,250,.42)` | Captions, labels, footer | + +### Voice (the only saturated hues) +These are the orb's state colors. They appear on the orb, and as small role echoes +in the transcript — nowhere else. + +| Token | Value | Meaning | +|---|---|---| +| `--accent` / `--speaking` | `#8b7dff` violet | Assistant speaking | +| `--accent-2` / `--listening` | `#22d3ee` cyan | You / listening | +| `--processing` | `#f59e0b` amber | Thinking / tool call | +| `--error` | `#ff6a75` | Error | +| `--success` | `#34d399` | Ready / connected | + +### Role echoes (semantic aliases — use these in chat code) +| Token | Maps to | Where it shows | +|---|---|---| +| `--voice-user` | cyan | `YOU` label + user bubble accents | +| `--voice-assistant` | violet | `ASSISTANT` label + assistant accents | +| `--voice-tool` | amber | `TOOL CALL` label, wrench icon | + +**Why this mapping:** it mirrors the orb exactly — when *you* speak the orb is cyan +(`state-listening`), when the *assistant* speaks it's violet (`state-ai-speaking`), +when it's working it's amber (`state-processing`). The transcript is a quiet replay +of the orb's color story. + +### Orb state → glow (`.circle.state-*` → `--glow`) +| State | Glow | +|---|---| +| `signed-out` | violet `#8b7dff` | +| `authenticated` / `ready` | green `#34d399` | +| `connecting` / `connected` / `starting` | yellow `#facc15` | +| `listening` / `user-speaking` | cyan (`--listening`) | +| `processing` | amber (`--processing`) | +| `ai-speaking` | violet (`--speaking`) | +| `error` | red (`--error`) | + +Adding a new state? Give it a `--glow`, and if it surfaces in the transcript, add a +matching `--voice-*` alias rather than a one-off color. + +--- + +## Typography + +Two faces, two jobs. Never reach for a third. + +- **Inter** — body and UI. Wordmark, buttons, inputs, panel titles, prose, history + message bodies. The workhorse; it should feel neutral and get out of the way. +- **Geist Mono** (`--font-mono`) — the **machine voice**. Reserved for text the + *system* emits or identifiers it reports, never for human prose. + +### When mono is correct +Mono signals "this is the machine talking or naming itself." Use it for: +- the orb's status caption (`.circle-caption`) +- role eyebrows (`YOU` / `ASSISTANT` / `TOOL CALL`) +- tool-call names and argument JSON +- the `·WebSocket` transport tag, bitrate readouts, connection identifiers +- the empty-state label + +Mono text is set uppercase with `letter-spacing: ~0.1–0.14em` and weight `500`, so it +reads as a typed status line, not a headline. Body copy, button labels, and +explanatory `small` text stay **Inter** — putting prose in mono breaks the metaphor. + +The font is loaded in `index.html`; the stack falls back to system mono gracefully if +the CDN is blocked. + +--- + +## Layout + +- **One continuous canvas.** No dividers under the topbar or above the footer, no + panel chrome competing with content. The topbar and footer float over the stage. + Keep it that way — a new section earns a hairline (`--border`) only if it genuinely + needs separating. +- **The orb is the hero and the center of gravity.** It sits dead-center on the + stage. Controls flank it (mic / stop), captions sit beneath. Don't crowd it. +- **Hairlines, not boxes.** Separation comes from `1px` borders at 8–16% white and + from spacing, not from heavy fills or shadows. Shadows are soft and low + (`0 4px 18px rgba(0,0,0,.32)`), used only to lift floating elements (bubbles, + panels, modal). +- **Radii:** `--radius-sm: 8px` (buttons, inputs, chips), `--radius-md: 14px` + (bubbles, message bodies, modal), `--radius-lg: 22px` (reserved). Pick by element + size; don't invent new values. +- **Two reading surfaces for the transcript:** ephemeral bubbles top-right (desktop + only) that log and fade, and a slide-in history panel for review. On phones the + bubble stream is dropped and the panel goes full-screen — the panel is the single + source of truth there. + +--- + +## Components + +- **Buttons.** Default (`.btn`) is a neutral elevated surface. The *primary* button + is **near-white on dark** (`--text` fill, `--bg` text) — the highest-contrast thing + on the page that *isn't* the orb. There is no colored button; emphasis comes from + contrast, not hue. +- **Icon buttons** (`.icon-btn`) are `36px`, elevated surface, dim icon that brightens + on hover. Side controls (`.side-btn`) are circular, collapse to zero size until the + session is live (by width on desktop, by height in the mobile column). +- **Chat bubbles & history messages** share one neutral surface. They are + distinguished by **side** (you = left, assistant = right) plus the **mono role + label** in the role-echo hue — not by tinted fills. Tool entries use the wrench icon + + mono + amber, on the same neutral surface. +- **Badge** (new-message dot) is monochrome white — a signal, not a color accent. +- **Focus** is visible and neutral: inputs focus to `--text-dim`; the orb uses a + `--glow`-colored outline (it's the orb, so color is allowed). + +--- + +## Motion + +- **The orb is audio-reactive, not timer-driven.** Mic RMS (`--audio-level`) and the + assistant output level (`--ai-audio-level`) drive scale/opacity at display rate, so + every syllable moves it. This is the signature animation — keep new motion + subordinate to it. +- **Quiet by default.** Breathing/glow throbs are slow (1.4–2.4s) and low-contrast. + Resist adding scattered micro-animations; an orchestrated moment beats many small + ones, and excess motion reads as AI-generated. +- **First paint is frozen.** `body.booting` disables all transitions until the first + frame commits (stripped after one rAF in `main.js`). Anything new that would + otherwise animate-in on load must respect this. +- Honor `prefers-reduced-motion` for any motion you add. + +--- + +## Writing / copy voice + +- Sentence case, plain verbs, no filler. Tuned and quiet — match the minimal canvas. +- Name things by what the user controls, not by the system's internals. A button says + exactly what it does, and keeps the same word through the flow. +- **Empty states invite action** ("Tap the orb and start talking"), they don't just + set a mood. +- **Errors state what happened and how to recover**, in the interface's voice — they + don't apologize and are never vague. +- Mono labels are terse identifiers (`YOU`, `TOOL CALL`); prose stays in Inter. + +--- + +## Responsive floor (non-negotiable) + +Every change ships meeting these: +- Works down to a `360px`-wide phone. The `@media (max-width: 600px)` block already + handles the phone layout — extend it, don't fight it. +- Visible keyboard focus on every interactive element. +- `prefers-reduced-motion` respected. +- Tap targets ≥ `44px`; `touch-action: manipulation` on anything tappable. + +--- + +## Before you ship — the mirror check + +1. Is every saturated color either on the orb or a word-sized role echo? If a fill or + border is colored, remove that accessory. +2. Is mono used only for machine/system text, and Inter for everything human? +3. Does any new state have both a `--glow` and (if it appears in chat) a `--voice-*` + alias? +4. Are separations hairlines + spacing, not boxes and heavy shadows? +5. Did you add motion? Is it quieter than the orb and reduced-motion-safe? +6. Remove one accessory. The minimal look survives on precision, not addition. diff --git a/demo/Dockerfile b/demo/Dockerfile new file mode 100644 index 0000000..5d3e074 --- /dev/null +++ b/demo/Dockerfile @@ -0,0 +1,14 @@ +# Single container: serves the static front-end AND the /api/search proxy. +# HF Spaces (sdk: docker) routes traffic to $PORT, default 7860. +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 7860 + +CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "7860"] diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 0000000..779be5b --- /dev/null +++ b/demo/README.md @@ -0,0 +1,190 @@ +--- +title: HF Realtime Voice +emoji: 🎙️ +colorFrom: indigo +colorTo: purple +sdk: docker +app_port: 7860 +pinned: false +short_description: Voice chat over WebSocket against a HF speech-to-speech +hf_oauth: true +--- + +# Realtime Voice Demo (WebSocket transport) + +Browser voice-chat UI for the +[huggingface/speech-to-speech](https://github.com/huggingface/speech-to-speech) +backend. The browser streams mic audio over a WebSocket using the OpenAI +Realtime **GA** protocol and plays back the assistant's audio as it arrives. + +## Quick start (local) + +1. **Start the speech-to-speech backend** in realtime mode (from the repo root; + see the [backend README](https://github.com/huggingface/speech-to-speech/blob/main/src/speech_to_speech/api/openai_realtime/README.md) + for more model combinations): + + ```bash + uv run speech-to-speech \ + --mode realtime \ + --stt parakeet-tdt \ + --llm_backend transformers \ + --tts kokoro \ + --model_name "Qwen/Qwen3-4B-Instruct-2507" \ + --llm_device mps \ + --llm_torch_dtype float16 \ + --enable_live_transcription + ``` + + The realtime server listens on `ws://localhost:8765/v1/realtime` by default + (`--ws_host` / `--ws_port` to change). + +2. **Start this app**, pointing it at the backend with `SPEECH_TO_SPEECH_URL`: + + ```bash + uv pip install -r demo/requirements.txt + export SPEECH_TO_SPEECH_URL=ws://localhost:8765/v1/realtime + export SERPER_API_KEY=... # optional; web search is disabled without it + uv run uvicorn --app-dir demo server:app --reload --port 7860 + ``` + + Or with Docker: + + ```bash + docker build -t s2s-demo demo/ + docker run -p 7860:7860 -e SPEECH_TO_SPEECH_URL=ws://host.docker.internal:8765/v1/realtime s2s-demo + ``` + +3. Open , click the orb, allow the mic, talk. + +> Browsers require **HTTPS or `localhost`** for `getUserMedia()` (mic + camera). +> `127.0.0.1` and `localhost` both work; plain `http://192.168.x.y` does NOT. + +Smoke-test the backend from the shell: + +```bash +websocat ws://localhost:8765/v1/realtime +# -> you should get a session.created event back immediately +``` + +## How it works + +1. The browser opens a WebSocket on the configured `/v1/realtime` URL. +2. Server pushes `session.created` on connect. Client replies with + `session.update` (OpenAI Realtime **GA** schema: `session.audio.input`, + `session.audio.output`, `session.output_modalities`). +3. Client streams mic audio as PCM16 16 kHz mono base64 chunks + (`input_audio_buffer.append`, one frame every ~40 ms). +4. Server pushes `response.output_audio.delta` (PCM16 24 kHz mono base64) + and transcript deltas. + +The backend exposes one concurrent session per pipeline unit +(`--num_pipelines` to serve more). + +## Connecting to a backend + +Three modes, picked by env (`/api/config` tells the client which one is active): + +- **`SPEECH_TO_SPEECH_URL` env** — the mode you want for local use, and the + highest priority. The browser connects **directly** to this realtime + WebSocket URL; it's shown read-only in Settings. Setting it disables the + load-balancer logic entirely (no `/api/session` proxy, no queue, no + metering, no sign-in). Unlike the LB address it is not a secret. Accepts a + full `ws(s)://host/v1/realtime` URL or a bare host like `localhost:8765` + (the app adds `/v1/realtime`). +- **Neither env set** — **Settings → Speech-to-speech server URL**: paste a + full connect URL or a bare host, and the browser connects to it directly. +- **`LOAD_BALANCER_URL` env** — multi-compute deployments only: the browser + POSTs the same-origin `/api/session` proxy, the server forwards to the LB, + and the browser dials the per-session compute URL the LB hands back. The LB + address never reaches the browser; the Settings URL field is hidden. + +| `SPEECH_TO_SPEECH_URL` | `LOAD_BALANCER_URL` | `SPACE_ID` | Connection | URL field | Metering | +|:---:|:---:|:---:|---|---|---| +| ✅ | any | any | direct → pinned URL | visible, locked | off | +| – | – | any | direct → user URL | editable | off | +| – | ✅ | ✅ | LB proxy | hidden | **on** | +| – | ✅ | – | LB proxy | hidden | off | + +**Settings → Restart** reconnects with the current voice, instructions and URL. + +## Tools + +The assistant can call two tools mid-conversation (toggle them from the **Tools** +button, top-right): + +- **Web search** — Google results via Serper.dev, proxied server-side so the key + never reaches the browser. Set `SERPER_API_KEY` as an env var / Space secret. + Without it, the tool is disabled unless the user pastes their own key in the + Tools panel. +- **Camera** — while enabled, a live self-view shows bottom-left; when the model + calls the tool, the current frame is sent to the vision-language model so it can + see what you're showing it. + +## Usage limits (deployed Space only) + +Conversation time is metered per UTC day by sign-in tier (see `limiter.py` / +`auth.py`), but **only on the deployed Space** — metering turns on only when BOTH +`LOAD_BALANCER_URL` and `SPACE_ID` (injected automatically by the HF Space +runtime) are present. Running locally — even with `LOAD_BALANCER_URL` exported — +leaves the app unmetered. Tunable via env: + +| Env | Default | What | +|-----|---------|------| +| `LIMIT_ANON_SEC` | `300` | Daily seconds for anonymous visitors (5 min) | +| `LIMIT_FREE_SEC` | `600` | Daily seconds for signed-in non-PRO users (10 min) | +| `UNLIMITED_ORGS` | _(adds to defaults)_ | Extra HF org names whose members get **unlimited** usage, like PRO | +| `USAGE_HASH_SECRET` | _(random)_ | HMAC secret for hashing identity keys + signing the anon cookie | + +PRO members are always unlimited. Members of `cerebras`, `HuggingFaceM4`, +`smolagents`, and `pollen-robotics` are unlimited out of the box (shown as +"Team", not "PRO"); set `UNLIMITED_ORGS=my-team` to add more. Matched +case-insensitively against the user's organisations from HF OAuth. + +## Settings (stored in `localStorage`) + +| Key | What | +|-----|------| +| Speech-to-speech server URL | Direct realtime WebSocket URL (hidden/locked when pinned by env) | +| Voice | Qwen3-TTS speaker name (Aiden, Ryan, Dylan, Eric, Ono_Anna, Serena, Sohee, Uncle_Fu, Vivian) | +| Instructions | System prompt sent in `session.update` once the WS opens | + +LocalStorage keys are namespaced `s2s.ws.*` so this app's settings do +NOT collide with the WebRTC variant. + +## Files + +| File | Role | +|------|------| +| `index.html` | Single page, orb + settings modal (identical UI to the WebRTC app) | +| `main.js` | State machine, settings, tools, camera, noise-gate UI wiring | +| `ui/chat.js` | `ChatView`: history panel, ephemeral bubbles, transcript/tool streaming | +| `ui/account.js` | `Account`: HF login chip + popover, daily-limit modal | +| `ui/dom.js` | Shared helpers: `$`, `escHtml`, `truncateError`, `DEBUG` | +| `auth.py` | HF OAuth + per-request identity (tier, hashed keys) | +| `limiter.py` | SQLite per-day talk-time budget (chunked server-clock reservation) | +| `ws/s2s-ws-client.js` | WebSocket handshake + OpenAI Realtime GA protocol | +| `ws/codec.js` | base64 <-> PCM helpers + transcript extraction (pure) | +| `ws/orb-visualizer.js` | `OrbVisualiser`: FFT bands -> orb CSS custom properties | +| `worklets/mic-capture.js` | AudioWorklet: 48 kHz Float32 -> 16 kHz Int16 PCM, posts ~40 ms chunks | +| `worklets/audio-playback.js` | AudioWorklet: 24 kHz Float32 ring buffer -> 48 kHz, linear interp, fade in/out | +| `style.css` | Orb animations, layout, dark theme (verbatim from the WebRTC app) | + +## Audio pipeline notes + +- **Input**: `getUserMedia({ echoCancellation, noiseSuppression, autoGainControl })` + feeds the `mic-capture` worklet at the `AudioContext` rate. The worklet + resamples to 16 kHz (boxcar lowpass + decimation on the 48 -> 16 fast + path, linear interpolation fallback for odd rates) and packs Int16 LE. +- **Output**: `response.output_audio.delta` decodes to Int16 -> Float32 + and is posted to the `audio-playback` worklet. The worklet maintains a + per-context ring buffer, linearly interpolates 24 -> 48, and applies + short 32-frame fades on entry/exit to suppress clicks. +- **Barge-in**: when the server VAD detects user speech mid-response + (`input_audio_buffer.speech_started` while `ai-speaking`), the client + posts `{ kind: "clear" }` to the playback worklet to wipe the queue + immediately. The server itself cancels the in-flight response. + +## Credits + +- Backend: [huggingface/speech-to-speech](https://github.com/huggingface/speech-to-speech) +- UI verbatim from `amir-tfrere/minimal-conversation-app-s2s-backend` (Pollen Robotics × Hugging Face) diff --git a/demo/auth.py b/demo/auth.py new file mode 100644 index 0000000..71700fe --- /dev/null +++ b/demo/auth.py @@ -0,0 +1,229 @@ +""" +HF OAuth + per-request identity for the duration limiter. + +Login uses Hugging Face's native Spaces OAuth via `huggingface_hub` +(`attach_huggingface_oauth` / `parse_huggingface_oauth`). The OAuth env +(`OAUTH_CLIENT_ID`, ...) is injected by the platform when the Space README sets +`hf_oauth: true`, so this only activates on a deployed Space — locally and in +direct mode there's no OAuth and the limiter treats everyone as anonymous. + +Identity: + - signed in -> tier 'pro' | 'free', keyed by hashed HF `sub` + - anonymous -> tier 'anon', keyed by BOTH hashed client IP and a hashed + signed-cookie id (OR-matched in the limiter) +""" + +import logging +import os +import secrets + +import limiter + +logger = logging.getLogger("s2s.auth") + +# huggingface_hub adds these routes when OAuth is attached. Centralised so a +# version change is a one-line fix; the paths are handed to the client via +# /api/me rather than hardcoded there. +OAUTH_LOGIN_PATH = "/oauth/huggingface/login" +OAUTH_LOGOUT_PATH = "/oauth/huggingface/logout" + +ANON_COOKIE = "s2s_anon" +_COOKIE_MAX_AGE = 60 * 60 * 24 * 30 # 30 days + + +# Members of these orgs get unlimited usage (like PRO) out of the box. The +# UNLIMITED_ORGS env adds to this set; it doesn't replace it. +_DEFAULT_UNLIMITED_ORGS = {"cerebras", "huggingfacem4", "smolagents", "pollen-robotics"} + + +def _unlimited_orgs() -> "set[str]": + """Org usernames whose members get unlimited usage (like PRO). + + Defaults to {cerebras, HuggingFaceM4, smolagents}; the UNLIMITED_ORGS env + (comma/space-separated, e.g. `UNLIMITED_ORGS=my-team`) adds more. Matched + case-insensitively against the signed-in user's organisations.""" + raw = os.environ.get("UNLIMITED_ORGS", "") + extra = {o.strip().lower() for o in raw.replace(",", " ").split() if o.strip()} + return _DEFAULT_UNLIMITED_ORGS | extra + +try: + from huggingface_hub import attach_huggingface_oauth, parse_huggingface_oauth + _OAUTH_IMPORTABLE = True +except Exception as exc: # pragma: no cover - import guard + logger.info("huggingface_hub OAuth unavailable (%s); sign-in disabled.", exc) + _OAUTH_IMPORTABLE = False + +# Set by attach(): True once OAuth is actually wired (importable + env present). +oauth_enabled = False + + +def attach(app) -> bool: + """Wire HF OAuth onto the app if it's importable and configured. Returns + whether sign-in is available.""" + global oauth_enabled + if not _OAUTH_IMPORTABLE or not os.environ.get("OAUTH_CLIENT_ID"): + return False + try: + attach_huggingface_oauth(app) + oauth_enabled = True + logger.info("HF OAuth attached (sign-in enabled).") + except Exception as exc: # pragma: no cover - defensive + logger.warning("Failed to attach HF OAuth: %r", exc) + oauth_enabled = False + return oauth_enabled + + +def _field(obj, name, default=None): + """Read a field whether the user-info is an object or a dict.""" + if obj is None: + return default + if isinstance(obj, dict): + return obj.get(name, default) + return getattr(obj, name, default) + + +# Surface what we detect (orgs, tier) in logs and on /api/me when set. Handy for +# verifying org gating on the live Space without guessing. +AUTH_DEBUG = bool(os.environ.get("AUTH_DEBUG")) + +# whoami-v2 org lookups are cached for the process lifetime, keyed by token, so +# /api/me + /api/session don't each hit the Hub. +_orgs_cache: "dict[str, set[str]]" = {} + + +def current_oauth(request): + """The parsed HF OAuth info (user_info + access_token), or None.""" + if not oauth_enabled: + return None + try: + return parse_huggingface_oauth(request) + except Exception: + return None + + +def current_user(request): + """The signed-in HF user-info, or None.""" + return _field(current_oauth(request), "user_info") + + +def _user_org_names(user) -> "set[str]": + """The user's organisations from the OAuth userinfo, by username/name/id.""" + names = set() + for org in _field(user, "orgs", []) or []: + for key in ("preferred_username", "name", "sub"): + val = _field(org, key) + if val: + names.add(str(val).lower()) + return names + + +def _orgs_via_token(token: str) -> "set[str]": + """Fallback org lookup via the Hub `whoami-v2` API, using the user's OAuth + access token. Covers the case where the userinfo claim omits `orgs`.""" + if not token: + return set() + if token in _orgs_cache: + return _orgs_cache[token] + names: "set[str]" = set() + try: + import httpx + + resp = httpx.get( + "https://huggingface.co/api/whoami-v2", + headers={"Authorization": f"Bearer {token}"}, + timeout=5.0, + ) + resp.raise_for_status() + for org in resp.json().get("orgs", []) or []: + for key in ("name", "fullname"): + val = org.get(key) + if val: + names.add(str(val).lower()) + except Exception as exc: # pragma: no cover - network/permission dependent + logger.info("whoami-v2 org lookup failed: %r", exc) + _orgs_cache[token] = names + return names + + +def _org_names(user, token=None, allow=None) -> "set[str]": + """The user's org usernames from the OAuth userinfo claim. If that doesn't + already satisfy `allow`, fall back to the Hub `whoami-v2` API (the claim is + often empty or partial), so membership is resolved either way.""" + names = _user_org_names(user) + if token and (allow is None or not (allow & names)): + names = names | _orgs_via_token(token) + return names + + +def resolve_tier(user, token=None) -> str: + """Tier for a signed-in user: 'pro' (paying), 'org' (allow-listed org + member, unlimited), or 'free'. PRO wins over org if both apply.""" + if bool(_field(user, "is_pro", False)): + return "pro" + allow = _unlimited_orgs() + names = _org_names(user, token, allow) + tier = "org" if (allow & names) else "free" + if AUTH_DEBUG: + logger.info("tier=%s orgs=%s allow=%s", tier, sorted(names), sorted(allow)) + return tier + + +def user_view(request) -> dict: + """Public profile for /api/me.""" + info = current_oauth(request) + user = _field(info, "user_info") + if not user: + return {"loggedIn": False, "tier": "anon"} + token = _field(info, "access_token") + out = { + "loggedIn": True, + "username": _field(user, "preferred_username") or _field(user, "name") or "you", + "avatar": _field(user, "picture"), + "tier": resolve_tier(user, token), + } + if AUTH_DEBUG: + out["orgs"] = sorted(_org_names(user, token)) + return out + + +def _client_ip(request) -> str: + """Real client IP. On HF the app sits behind a proxy, so the user's address + is the first hop in X-Forwarded-For, not request.client.host.""" + xff = request.headers.get("x-forwarded-for", "") + if xff: + return xff.split(",")[0].strip() + return request.client.host if request.client else "unknown" + + +def resolve_identity(request): + """Resolve (tier, keys, set_cookie) for this request. + + `keys` are the limiter usage_daily keys to debit (one for signed-in, two for + anonymous). `set_cookie` is a signed value to Set-Cookie when we minted a new + anonymous id, else None. + """ + info = current_oauth(request) + user = _field(info, "user_info") + if user: + sub = _field(user, "sub") or _field(user, "preferred_username") + token = _field(info, "access_token") + return resolve_tier(user, token), [limiter.hash_key(f"sub:{sub}")], None + + # Anonymous: key by IP and a signed cookie id, minting the cookie if absent. + ip = _client_ip(request) + cookie_id = limiter.verify_cookie(request.cookies.get(ANON_COOKIE, "")) + set_cookie = None + if not cookie_id: + cookie_id = secrets.token_urlsafe(18) + set_cookie = limiter.sign_cookie(cookie_id) + keys = [limiter.hash_key(f"ip:{ip}"), limiter.hash_key(f"cookie:{cookie_id}")] + return "anon", keys, set_cookie + + +def set_anon_cookie(response, signed: str) -> None: + # The Space runs inside an iframe on huggingface.co, so the cookie lives in a + # cross-site context — it must be SameSite=None; Secure or the browser drops it. + response.set_cookie( + ANON_COOKIE, signed, + max_age=_COOKIE_MAX_AGE, httponly=True, samesite="none", secure=True, + ) diff --git a/demo/docs/adr/0001-docker-space-with-search-proxy.md b/demo/docs/adr/0001-docker-space-with-search-proxy.md new file mode 100644 index 0000000..740a506 --- /dev/null +++ b/demo/docs/adr/0001-docker-space-with-search-proxy.md @@ -0,0 +1,29 @@ +# Convert the static Space into a Docker app with a search proxy + +To give the model a web search tool, the executor (which runs in the browser of a +public Space) needs a search key. A `sdk: static` Space serves files as-is with no +runtime process, so it cannot hold a secret the browser uses without exposing it. +We convert the Space from `sdk: static` to `sdk: docker`: a single container runs a +small server (FastAPI + uvicorn) that both serves the existing front-end *unchanged* +and exposes a same-origin `/search` proxy holding `SERPER_API_KEY` server-side. The +whole app lives in that one container; the s2s speech-to-speech backend stays the +separate load-balanced service it already is. + +## Considered options + +- **Stay static, user-supplied key only** — no owner key; search works only if each + user pastes their own. Rejected as the default because it leaves the deployed demo + with no working search. +- **Separate proxy service** — same secrecy, but splits the app across two deploys. + Rejected: the app must live in one container. +- **Key baked into client JS at build time** — would be readable in the served + bundle. Rejected: defeats the point. + +## Consequences + +- Deployment is no longer static: there is a Dockerfile and a server process; the + README front-matter changes from `sdk: static` to `sdk: docker`. +- The client calls `/search` same-origin; the server reads the key from env. A user + may still supply their own key as a fallback, sent per-request to the proxy. +- The front-end, audio pipeline, and s2s handshake are untouched — only the hosting + shape and the new route are added. diff --git a/demo/index.html b/demo/index.html new file mode 100644 index 0000000..dd52bea --- /dev/null +++ b/demo/index.html @@ -0,0 +1,373 @@ + + + + + + Minimal Conversation · S2S backend (WebSocket) + + + + + + + +
+
+
+
+
+ Hugging Face Realtime + +
+

An open, real-time voice chat built on Hugging Face's speech-to-speech stack.

+
+ + Powered by + Inference Endpoints + + Cerebras + + + Built by + Hugging Face + + tfrere + + A-Mahla + + andito + +
+
+
+
+ + + + + + +
+
+ +
+
+
+ + +
+ + + + +
+ +

Tap to start

+ + + +
+ + +
+ + +
+ + + + + +
+
+
+
+

Conversation

+ +
+
+
+
+ + +
+
+

About

+ +
+ + +
+

Speech-to-speech is Hugging Face's open framework for real-time voice agents. Rather than one end-to-end model, it chains four open models from the Hub (speech detection, transcription, a vision-language model, and synthesis), so any stage can be swapped or run locally. This demo wires that pipeline to hosted inference.

+ View the project on GitHub +
+ + +
+

The pipeline

+
    +
  1. You speak
    2. +
  2. + VAD + detects speech + silero-vad +
    3. +
  3. + STT + transcribes it + nvidia/parakeet-tdt-1.1b +
    4. +
  4. + VLM + analyses and composes the reply · via Cerebras + google/gemma-4-31B-it +
    5. +
  5. + TTS + speaks it back + Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice +
    6. +
  6. The orb replies
    7. +
+
+
+ + + +
+
+

Settings

+ +
+ +
+
+ + +
+ +
+ +
+ + Noise gate + Off + +
+ +
+ Off + −3 dB +
+
+ + Mutes the mic below the handle so room noise isn't sent. Slide fully + left to turn it off; the bar shows your live input while in a call. + +
+ + + +
+ + Connect first, then come back here to apply live changes. +
+
+
+ +
+ +
+
+ + + +
+
+

Tools

+ +
+ +

Let the assistant act during the conversation. Changes apply live.

+ +
+
+
+ Web search + Look things up on Google, via Serper. +
+ +
+ + +
+
+ Camera + Let the assistant see through your webcam. +
+ +
+ A live preview shows bottom-left while the camera is on. +
+
+ + + + +
+ + +

That's a wrap for now

+

+ +

+
+ + + + + diff --git a/demo/limiter.py b/demo/limiter.py new file mode 100644 index 0000000..77a7fb2 --- /dev/null +++ b/demo/limiter.py @@ -0,0 +1,276 @@ +""" +Per-day talk-time budget for the speech-to-speech demo. + +Our server isn't in the audio path (the browser dials the compute WebSocket +directly), so it can't cut a live stream. What it *can* do is meter time with a +server-clock, chunked reservation: + + - At grant we reserve the first chunk (CHUNK_SEC) and debit it from the day's + budget. A parallel grant therefore sees the budget already spent. + - The client heartbeats; each heartbeat extends the reservation one chunk at a + time until the daily budget runs out, then we report `expired` so the client + tears down. + - On a clean end (sendBeacon) we reconcile to the real elapsed time and refund + the unused chunk. A crash (no end, no heartbeats) is reaped by a sweep and + forfeits at most one chunk. + +All time is the server's clock. Budgets are per UTC day; a new day is simply a +new row (no explicit reset). Logged-in users are keyed by a hashed HF `sub`; +anonymous users by BOTH a hashed IP and a hashed signed-cookie id, OR-matched +(spent = max of the two) so clearing one identifier doesn't reset the budget. + +Storage is SQLite at $USAGE_DB_PATH, else /data (persistent Spaces storage), +else a /tmp fallback. On /tmp the budget is only per-uptime — flagged in logs. +""" + +import hashlib +import hmac +import logging +import math +import os +import sqlite3 +import tempfile +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + +logger = logging.getLogger("s2s.limiter") + +# ── Tunables (env-overridable) ─────────────────────────────────────────────── +ANON_SEC = int(os.environ.get("LIMIT_ANON_SEC", "300")) # 5 min/day, not signed in +FREE_SEC = int(os.environ.get("LIMIT_FREE_SEC", "600")) # 10 min/day, signed in, no PRO +CHUNK_SEC = int(os.environ.get("RESERVE_CHUNK_SEC", "10")) # reservation granularity +HEARTBEAT_SEC = int(os.environ.get("HEARTBEAT_SEC", "5")) # advertised client cadence +REAP_AFTER_SEC = int(os.environ.get("SESSION_REAP_SEC", "15")) # silence before sweep + +# Stable across restarts or the hashed keys (and signed cookies) rotate and the +# budget effectively resets. Set it as a Space secret. Falls back to a per-boot +# random value (keys then only hold within one uptime). +_HASH_SECRET = (os.environ.get("USAGE_HASH_SECRET", "").strip() or os.urandom(32).hex()).encode() + +_lock = threading.Lock() +_db_path: "Path | None" = None + + +def budget_for(tier: str) -> "int | None": + """Daily second-budget for a tier, or None for unlimited. + + Unlimited tiers: 'pro' (paying PRO members) and 'org' (members of an + allow-listed organisation, see UNLIMITED_ORGS in auth.py).""" + if tier in ("pro", "org"): + return None + if tier == "free": + return FREE_SEC + return ANON_SEC + + +def hash_key(raw: str) -> str: + """HMAC a raw identifier (sub / ip / cookie id) into an opaque storage key.""" + digest = hmac.new(_HASH_SECRET, raw.encode("utf-8"), hashlib.sha256).hexdigest() + return f"k_{digest}" + + +def sign_cookie(value: str) -> str: + """`.` so a forged anon-cookie id is rejected on read.""" + sig = hmac.new(_HASH_SECRET, value.encode("utf-8"), hashlib.sha256).hexdigest()[:32] + return f"{value}.{sig}" + + +def verify_cookie(signed: str) -> "str | None": + """Return the id if the signature checks out, else None.""" + if not signed or "." not in signed: + return None + value, _, sig = signed.rpartition(".") + want = hmac.new(_HASH_SECRET, value.encode("utf-8"), hashlib.sha256).hexdigest()[:32] + return value if hmac.compare_digest(sig, want) else None + + +def _today() -> str: + return datetime.now(timezone.utc).date().isoformat() + + +def _resolve_db_path() -> Path: + explicit = os.environ.get("USAGE_DB_PATH", "").strip() + if explicit: + return Path(explicit) + data = Path("/data") + if data.is_dir() and os.access(data, os.W_OK): + return data / "s2s-usage.sqlite3" + logger.warning("No persistent /data — usage budget falls back to /tmp (per-uptime only).") + return Path(tempfile.gettempdir()) / "s2s-usage.sqlite3" + + +def _connect() -> sqlite3.Connection: + con = sqlite3.connect(_db_path, timeout=5.0) + con.execute("PRAGMA journal_mode=WAL") + con.execute("PRAGMA busy_timeout=5000") + return con + + +def init() -> None: + """Create the schema. Call once at startup.""" + global _db_path + _db_path = _resolve_db_path() + with _lock, _connect() as con: + con.execute( + """CREATE TABLE IF NOT EXISTS usage_daily ( + user_key TEXT NOT NULL, + day TEXT NOT NULL, + spent_sec INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_key, day) + )""" + ) + con.execute( + """CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT PRIMARY KEY, + keys TEXT NOT NULL, -- comma-joined usage_daily keys to debit + day TEXT NOT NULL, + tier TEXT NOT NULL, + grant_ts REAL NOT NULL, + last_seen_ts REAL NOT NULL, + reserved_sec INTEGER NOT NULL, + ended INTEGER NOT NULL DEFAULT 0 + )""" + ) + logger.info("Usage limiter ready at %s (anon=%ss free=%ss chunk=%ss)", _db_path, ANON_SEC, FREE_SEC, CHUNK_SEC) + + +# ── Internal helpers (call under _lock) ─────────────────────────────────────── + +def _spent(con, key: str, day: str) -> int: + row = con.execute( + "SELECT spent_sec FROM usage_daily WHERE user_key=? AND day=?", (key, day) + ).fetchone() + return int(row[0]) if row else 0 + + +def _spent_max(con, keys, day: str) -> int: + """OR-match: the most-spent identifier governs.""" + return max((_spent(con, k, day) for k in keys), default=0) + + +def _add(con, keys, day: str, delta: int) -> None: + now = int(time.time()) + for k in keys: + cur = _spent(con, k, day) + nxt = max(0, cur + delta) + con.execute( + """INSERT INTO usage_daily (user_key, day, spent_sec, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(user_key, day) DO UPDATE SET + spent_sec = excluded.spent_sec, updated_at = excluded.updated_at""", + (k, day, nxt, now), + ) + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def remaining(keys, tier: str) -> "int | None": + """Seconds left today for these keys (None = unlimited). No mutation.""" + budget = budget_for(tier) + if budget is None: + return None + with _lock, _connect() as con: + return max(0, budget - _spent_max(con, keys, _today())) + + +def begin(session_id: str, keys, tier: str) -> int: + """Reserve the first chunk for a new session and record it. Returns the + chunk reserved (0 if the budget is already exhausted — the first heartbeat + will then expire it). PRO (unlimited) is never tracked; don't call it here.""" + day = _today() + budget = budget_for(tier) + now = time.time() + with _lock, _connect() as con: + avail = budget - _spent_max(con, keys, day) if budget is not None else CHUNK_SEC + chunk = max(0, min(CHUNK_SEC, avail)) + if chunk: + _add(con, keys, day, chunk) + con.execute( + """INSERT OR REPLACE INTO sessions + (session_id, keys, day, tier, grant_ts, last_seen_ts, reserved_sec, ended) + VALUES (?, ?, ?, ?, ?, ?, ?, 0)""", + (session_id, ",".join(keys), day, tier, now, now, chunk), + ) + return chunk + + +def heartbeat(session_id: str) -> bool: + """Keep a session alive: extend the reservation toward `elapsed + 1 chunk`, + debiting the budget chunk by chunk. Returns True while alive, False once the + budget is spent (caller should tear down) or the session is unknown/ended.""" + now = time.time() + with _lock, _connect() as con: + row = con.execute( + "SELECT keys, day, tier, grant_ts, reserved_sec, ended FROM sessions WHERE session_id=?", + (session_id,), + ).fetchone() + if not row or row[5]: + return False + keys = row[0].split(",") + day, tier, grant_ts, reserved = row[1], row[2], row[3], int(row[4]) + budget = budget_for(tier) + elapsed = now - grant_ts + + # Grow the reservation one chunk at a time until it covers elapsed + a + # one-chunk lookahead, or the budget runs dry. + while reserved < elapsed + CHUNK_SEC: + if budget is not None and _spent_max(con, keys, day) >= budget: + break + _add(con, keys, day, CHUNK_SEC) + reserved += CHUNK_SEC + + alive = reserved > elapsed # could we cover the time already elapsed? + con.execute( + "UPDATE sessions SET last_seen_ts=?, reserved_sec=?, ended=? WHERE session_id=?", + (now, reserved, 0 if alive else 1, session_id), + ) + if not alive: + _reconcile(con, keys, day, grant_ts, reserved, end_ts=now) + return alive + + +def end(session_id: str) -> None: + """Clean teardown: reconcile to actual elapsed time and refund the unused + reservation. Idempotent.""" + now = time.time() + with _lock, _connect() as con: + row = con.execute( + "SELECT keys, day, grant_ts, reserved_sec, ended FROM sessions WHERE session_id=?", + (session_id,), + ).fetchone() + if not row or row[4]: + return + keys, day, grant_ts, reserved = row[0].split(","), row[1], row[2], int(row[3]) + _reconcile(con, keys, day, grant_ts, reserved, end_ts=now) + con.execute("UPDATE sessions SET ended=1, last_seen_ts=? WHERE session_id=?", (now, session_id)) + + +def sweep() -> None: + """Reap sessions that went silent (crash / closed without a beacon): bill + their elapsed time, refund the rest, mark ended. Forfeits ≤ one chunk.""" + now = time.time() + cutoff = now - REAP_AFTER_SEC + with _lock, _connect() as con: + stale = con.execute( + "SELECT session_id, keys, day, grant_ts, last_seen_ts, reserved_sec FROM sessions " + "WHERE ended=0 AND last_seen_ts < ?", + (cutoff,), + ).fetchall() + for session_id, keys_s, day, grant_ts, last_seen, reserved in stale: + _reconcile(con, keys_s.split(","), day, grant_ts, int(reserved), end_ts=last_seen) + con.execute("UPDATE sessions SET ended=1 WHERE session_id=?", (session_id,)) + if stale: + logger.debug("swept %d stale session(s)", len(stale)) + + +def _reconcile(con, keys, day: str, grant_ts: float, reserved: int, end_ts: float) -> None: + """Refund reserved-but-unused time. Bill elapsed rounded up to a chunk, + capped at what was reserved.""" + elapsed = max(0.0, end_ts - grant_ts) + billed = min(reserved, int(math.ceil(elapsed / CHUNK_SEC) * CHUNK_SEC)) + refund = reserved - billed + if refund > 0: + _add(con, keys, day, -refund) diff --git a/demo/main.js b/demo/main.js new file mode 100644 index 0000000..da07219 --- /dev/null +++ b/demo/main.js @@ -0,0 +1,1440 @@ +// @ts-check +/** + * Minimal voice conversation app, talking to a Hugging Face speech-to-speech + * backend over **WebSocket** (drop-in alternative to the WebRTC variant). + * + * Click the orb -> we ask for the mic, POST a session on the LB, open a + * WebSocket on the routed compute endpoint, push session.update + mic + * audio, play back the TTS audio. The orb visually reflects the live + * state (idle, connecting, listening, user-speaking, processing, + * ai-speaking). + * + * The only meaningful difference vs. the WebRTC main.js is that the + * client owns its own AudioContext (no `attachOutputTrack`), so we hand + * it the MediaStream directly. + * + * @typedef {"idle" | "connecting" | "queued" | "your-turn" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState + */ + +import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js"; +import { $, truncateError, DEBUG } from "./ui/dom.js"; +import { ChatView } from "./ui/chat.js"; +import { Account } from "./ui/account.js"; + +const DEFAULT_VOICE = "Aiden"; +const DEFAULT_INSTRUCTIONS = + "You are a friendly voice assistant. " + + "Keep replies short, warm, and spoken. Avoid long monologues."; + +// Appended to the user's instructions whenever at least one tool is enabled. +// Stops the model from announcing capabilities ("Yes, I can search") and then +// idling for the next turn — it should act immediately in the same response. +const TOOL_USE_HINT = + " When the user's request calls for one of your tools, do not describe your " + + "capabilities or say you can do it and wait for another turn. Instead, say " + + 'a brief acknowledgement like "Let me search for that..." and call the tool ' + + "right away in the same response."; + +const STORAGE_KEYS = { + // Direct s2s server URL, used only when the deploy has no LOAD_BALANCER_URL + // (in LB mode the browser never learns the LB address — it POSTs /api/session). + directUrl: "s2s.ws.directUrl", + voice: "s2s.ws.voice", + instructions: "s2s.ws.instructions", + tools: "s2s.ws.tools", + searchKey: "s2s.ws.searchKey", + noiseGate: "s2s.ws.noiseGate", +}; + +// ── Noise gate ────────────────────────────────────────────────────────────── +// The Settings cursor sets the gate's open threshold in dBFS. Its leftmost +// position is an OFF detent (gate disabled, pure passthrough); the rest of the +// travel is the active threshold. The cursor shares the meter's dB axis, so the +// handle sits on the level bar — raise it until room noise stops lighting it up. +// The slider range IS the shared axis: the live meter fill and the threshold +// thumb both map across [GATE_OFF_DB, GATE_MAX_DB], so the thumb sits exactly +// where the gate cuts on the same scale as the level bar. +const GATE_OFF_DB = -66; // slider minimum = off / bottom of the meter axis +const GATE_MAX_DB = -3; // slider maximum = most aggressive / top of the meter axis +const GATE_DEFAULT_DB = -50; // first-run default: a gentle gate, enabled + +/** @param {number} thresholdDb @returns {import("./ws/s2s-ws-client.js").NoiseGate} */ +function gateParams(thresholdDb) { + return { enabled: thresholdDb > GATE_OFF_DB, thresholdDb }; +} + +// ── Tools ───────────────────────────────────────────────────────────────── +// Function tools we declare to the backend. The model decides when to call +// one; the executor below runs it and returns the result (see runTool). +/** @type {Record} */ +const TOOL_DEFS = { + web_search: { + type: "function", + name: "web_search", + description: + "Search the web for current or factual information you don't already know " + + "(news, prices, facts, documentation). Returns the top results with titles, " + + "snippets and URLs.", + parameters: { + type: "object", + properties: { query: { type: "string", description: "The search query." } }, + required: ["query"], + }, + }, + camera_snapshot: { + type: "function", + name: "camera_snapshot", + description: + "Capture the current frame from the user's webcam so you can see what they " + + "are showing you. Use it whenever the user refers to something visual or " + + "asks you to look.", + parameters: { type: "object", properties: {}, required: [] }, + }, +}; + +/** Longest edge of the snapshot sent to the VLM, in px (keeps payload sane). */ +const SNAPSHOT_MAX_EDGE = 768; +const SNAPSHOT_QUALITY = 0.7; + +function loadSettings() { + return { + directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "", + voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE, + instructions: localStorage.getItem(STORAGE_KEYS.instructions) || DEFAULT_INSTRUCTIONS, + noiseGate: loadGateThreshold(), + }; +} + +/** Stored gate threshold (dBFS), clamped to the slider range. Defaults to a + * gentle enabled gate (GATE_DEFAULT_DB) when the user hasn't set one yet. */ +function loadGateThreshold() { + const stored = localStorage.getItem(STORAGE_KEYS.noiseGate); + // getItem returns null when unset, and Number(null) === 0 (finite!), so guard + // the missing/empty case explicitly before coercing — otherwise the default + // never fires and 0 clamps to the slider max. + if (stored === null || stored === "") return GATE_DEFAULT_DB; + const raw = Number(stored); + if (!Number.isFinite(raw)) return GATE_DEFAULT_DB; + return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(raw))); +} + +/** @param {ReturnType} s */ +function saveSettings(s) { + localStorage.setItem(STORAGE_KEYS.directUrl, s.directUrl); + localStorage.setItem(STORAGE_KEYS.voice, s.voice); + localStorage.setItem(STORAGE_KEYS.instructions, s.instructions); + localStorage.setItem(STORAGE_KEYS.noiseGate, String(s.noiseGate)); +} + +/** @returns {{ web_search: boolean, camera_snapshot: boolean }} */ +function loadTools() { + try { + const raw = JSON.parse(localStorage.getItem(STORAGE_KEYS.tools) || "{}"); + // Both tools default ON (web search still only activates when a key exists). + // We never call getUserMedia on page load — the camera only actually starts + // on a user gesture (conversation start), so a default-on flag doesn't + // silently resume the webcam; an explicit saved `false` is respected. + return { + web_search: raw.web_search ?? true, + camera_snapshot: raw.camera_snapshot ?? true, + }; + } catch { + return { web_search: true, camera_snapshot: true }; + } +} + +function saveTools() { + localStorage.setItem(STORAGE_KEYS.tools, JSON.stringify(toolsEnabled)); +} + +/** @type {Record} */ +const STATE_VIEWS = { + idle: { caption: "Tap to start", disabled: false }, + connecting: { caption: "Connecting", disabled: true }, + queued: { caption: "Finding you a spot…", disabled: true }, + "your-turn": { caption: "You're up! 🎉", disabled: true }, + listening: { caption: "", disabled: false }, + "user-speaking": { caption: "", disabled: false }, + processing: { caption: "", disabled: false }, + "ai-speaking": { caption: "", disabled: false }, + error: { caption: "Tap to retry", disabled: false }, +}; + +/** @type {Record} */ +const STATE_CLASS = { + idle: "state-idle", + connecting: "state-connecting", + queued: "state-queued", + "your-turn": "state-your-turn", + listening: "state-listening", + "user-speaking": "state-user-speaking", + processing: "state-processing", + "ai-speaking": "state-ai-speaking", + error: "state-error", +}; + +/** @type {ReadonlySet} */ +const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]); + +/** @type {HTMLButtonElement} */ +const circleBtn = $("#main-circle"); +/** @type {HTMLParagraphElement} */ +const circleCaption = $("#circle-caption"); +/** @type {HTMLParagraphElement} */ +const circleSubcaption = $("#circle-subcaption"); +/** @type {HTMLElement} */ +const orbWrap = $(".orb-wrap"); +/** @type {HTMLButtonElement} */ +const micBtn = $("#mic-btn"); +/** @type {HTMLButtonElement} */ +const stopBtn = $("#stop-btn"); +/** @type {HTMLElement} */ +const queueActions = $("#queue-actions"); +/** @type {HTMLButtonElement} */ +const joinQueueBtn = $("#join-queue-btn"); +/** @type {HTMLButtonElement} */ +const leaveQueueBtn = $("#leave-queue-btn"); + +/** @type {HTMLButtonElement} */ +const settingsBtn = $("#settings-btn"); +/** @type {HTMLDialogElement} */ +const settingsModal = $("#settings-modal"); + +/** @type {HTMLButtonElement} */ +const aboutBtn = $("#about-btn"); +/** @type {HTMLDialogElement} */ +const aboutModal = $("#about-modal"); +/** @type {HTMLButtonElement} */ +const aboutClose = $("#about-close"); + +/** @type {HTMLButtonElement} */ +const toolsBtn = $("#tools-btn"); +/** @type {HTMLDialogElement} */ +const toolsModal = $("#tools-modal"); +/** @type {HTMLButtonElement} */ +const toolsClose = $("#tools-close"); +/** @type {HTMLInputElement} */ +const toolWebSwitch = $("#tool-web"); +/** @type {HTMLInputElement} */ +const toolCamSwitch = $("#tool-cam"); +/** @type {HTMLElement} */ +const toolWebRow = $("#tool-web-row"); +/** @type {HTMLElement} */ +const toolWebHint = $("#tool-web-hint"); +/** @type {HTMLElement} */ +const toolCamHint = $("#tool-cam-hint"); +/** @type {HTMLInputElement} */ +const searchKeyInput = $("#search-key"); +/** @type {HTMLElement} */ +const camPip = $("#cam-pip"); +/** @type {HTMLVideoElement} */ +const camVideo = $("#cam-video"); + +/** @type {HTMLInputElement} */ +const inputLbUrl = $("#lb-url"); +/** @type {HTMLElement} */ +const connField = $("#conn-field"); +/** @type {HTMLElement} */ +const connHint = $("#conn-hint"); +/** @type {HTMLSelectElement} */ +const inputVoice = $("#voice"); +/** @type {HTMLTextAreaElement} */ +const inputInstructions = $("#instructions"); +/** @type {HTMLInputElement} */ +const inputNoiseGate = $("#noise-gate"); +/** @type {HTMLElement} */ +const gateValue = $("#gate-value"); +/** @type {HTMLElement} */ +const gateMeterFill = $("#gate-meter-fill"); +/** @type {HTMLElement} */ +const micGate = $("#mic-gate"); +const mgaArc = /** @type {SVGSVGElement} */ (document.querySelector("#mic-gate-arc")); +const mgaTrack = /** @type {SVGPathElement} */ (document.querySelector("#mga-track")); +const mgaFill = /** @type {SVGPathElement} */ (document.querySelector("#mga-fill")); +const mgaHit = /** @type {SVGPathElement} */ (document.querySelector("#mga-hit")); +const mgaHandle = /** @type {SVGCircleElement} */ (document.querySelector("#mga-handle")); +/** @type {HTMLButtonElement} */ +const restartBtn = $("#restart-conversation"); +/** @type {HTMLElement} */ +const restartHint = $("#restart-hint"); +const settingsForm = /** @type {HTMLFormElement} */ (settingsModal.querySelector("form")); + +/** @type {AppState} */ +let currentState = "idle"; +let settings = loadSettings(); + +// ── Connection target ──────────────────────────────────────────────────────── +// Three modes, decided by the deploy via /api/config: +// • SPEECH_TO_SPEECH_URL set -> direct mode pinned by the deploy: the browser +// connects straight to that URL, shown read-only in Settings. Overrides the +// load balancer entirely. +// • LOAD_BALANCER_URL set -> original flow: POST the same-origin /api/session +// proxy (the server forwards to the LB; the LB address is never sent here). +// • neither (allowDirect) -> the user sets a speech-to-speech server URL and +// the browser connects to it directly (no load balancer, no /session). +let lbMode = false; +// Fail open: direct entry is allowed unless /api/config reports an LB URL. This +// way a missing/unreachable config (e.g. static hosting) leaves the field +// usable rather than locked. +let allowDirect = true; +// Deploy-pinned s2s URL (SPEECH_TO_SPEECH_URL). Non-empty -> locked direct +// mode: the field displays it read-only and the saved user URL is untouched. +let pinnedUrl = ""; + +// ── Tool state ────────────────────────────────────────────────────────────── +let toolsEnabled = loadTools(); +// Whether the server holds a Serper key (learned from /api/config on load). +let serverSearchKey = false; +// A user-supplied key (fallback when the deploy has none). localStorage only. +let userSearchKey = localStorage.getItem(STORAGE_KEYS.searchKey) || ""; +/** @type {MediaStream | null} */ +let cameraStream = null; + +/** Search is usable if the server has a key or the user supplied one. */ +function searchAvailable() { + return serverSearchKey || !!userSearchKey; +} + +/** Tool definitions for the currently-enabled (and usable) tools. */ +function activeToolDefs() { + const defs = []; + if (toolsEnabled.web_search && searchAvailable()) defs.push(TOOL_DEFS.web_search); + if (toolsEnabled.camera_snapshot) defs.push(TOOL_DEFS.camera_snapshot); + return defs; +} + +/** Instructions plus the hidden tool-use hint when any tool is active. */ +function effectiveInstructions() { + const base = settings.instructions; + return activeToolDefs().length ? base + TOOL_USE_HINT : base; +} + +/** Push the active tool set to a live session so toggles apply mid-call. */ +function pushToolsToSession() { + if (!client || !LIVE_STATES.has(currentState)) return; + client.setTools(activeToolDefs()); + // The hidden tool-use hint depends on whether any tool is active, so refresh + // instructions alongside the tool set. + client.updateSession({ instructions: effectiveInstructions() }); +} + +// ── Chat view ─────────────────────────────────────────────────────────────── +// Owns the history panel, the ephemeral bubbles, and all transcript/tool +// streaming state. The client's events are forwarded to its on* methods. +const chat = new ChatView(); + +// ── Account / limiter ───────────────────────────────────────────────────── +// Login chip + daily-limit modal (inert unless the deploy is in LB mode). The +// server meters conversation time; the client just heartbeats a live session +// and tears down when the server reports the budget is spent. +const account = new Account(); +let limiterOn = false; +let heartbeatTimer = 0; +let trackedSessionId = ""; +let trackedTier = ""; +// The waiting-queue ticket id while we're in line (else ""). Used to leave the +// queue on teardown / tab-close so we don't hold a phantom place. +let queuedTicketId = ""; + +/** @type {S2sWsRealtimeClient | null} */ +let client = null; +/** @type {MediaStream | null} */ +let micStream = null; +let micMuted = false; + +/** @param {AppState} next */ +function setState(next) { + currentState = next; + const view = STATE_VIEWS[next]; + circleBtn.disabled = view.disabled; + circleBtn.className = `circle ${STATE_CLASS[next]}`; + if (next !== "error") setCaption(view.caption); + + const live = LIVE_STATES.has(next); + orbWrap.classList.toggle("live", live); + micBtn.setAttribute("aria-hidden", live ? "false" : "true"); + stopBtn.setAttribute("aria-hidden", live ? "false" : "true"); + micBtn.tabIndex = live ? 0 : -1; + stopBtn.tabIndex = live ? 0 : -1; + + // Queue affordances: "Leave queue" whenever we're in line; "Join now" only once + // it's our turn (a slot is held for us). Both live under #queue-actions. + const yourTurn = next === "your-turn"; + const inLine = next === "queued" || yourTurn; + queueActions.hidden = !inLine; + joinQueueBtn.hidden = !yourTurn; + joinQueueBtn.tabIndex = yourTurn ? 0 : -1; + leaveQueueBtn.hidden = !inLine; + leaveQueueBtn.tabIndex = inLine ? 0 : -1; + if (!yourTurn) stopJoinCountdown(); + + // Warm reassurance under the terse position, only while waiting in line. + if (next === "queued") { + circleSubcaption.textContent = + "Sorry, we overhugged! 🤗 Every slot is busy, so we saved you a spot. Hang tight, you're moving up."; + circleSubcaption.hidden = false; + } else { + circleSubcaption.hidden = true; + } + + updateRestartAvailability(); +} + +function updateRestartAvailability() { + // Restart works from any settled state — it tears down a live call (if any) + // and reconnects with the current settings. Only block while mid-connect or + // while waiting in the queue (restarting from there would just re-queue). + restartBtn.disabled = + currentState === "connecting" || currentState === "queued" || currentState === "your-turn"; + restartHint.hidden = false; + restartHint.textContent = LIVE_STATES.has(currentState) + ? "Reconnects now with the settings above." + : "Starts a conversation with the settings above."; +} + +/** + * @param {string} text + * @param {"" | "error" | "muted"} [kind] + */ +function setCaption(text, kind = "") { + const trimmed = text.trim(); + circleCaption.textContent = trimmed; + circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`; +} + +function openSettings() { + syncConnectionUi(); + inputVoice.value = settings.voice; + inputInstructions.value = settings.instructions; + syncGateUi(); + updateRestartAvailability(); + settingsModal.showModal(); +} + +/** dB position (clamped to the slider axis) as a 0..1 fraction of the track. + * @param {number} db */ +function dbToFraction(db) { + const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db)); + return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB); +} + +/** @param {number} f @returns {number} dB at a 0..1 position on the gate axis. */ +function fractionToDb(f) { + const clamped = Math.min(1, Math.max(0, f)); + return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB)); +} + +// ── Radial gate arc (around the mic button, live during a call) ───────────── +// A 270° arc with the gap facing the orb (right). Fraction 0 (=Off) sits at the +// bottom-ish start; 1 (=max) at the top-ish end. The level fill and the +// threshold handle ride this same axis, mirroring the Settings widget. +const ARC_R = 40; +// A ~200° arc centred on the left (180°) so the wide gap faces the orb (right). +const ARC_SPAN_DEG = 200; +const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // lower-left start; Off end + +/** Point at fraction f (0..1) and radius r, in the 0..100 viewBox. + * @param {number} f @param {number} [r] */ +function arcPoint(f, r = ARC_R) { + const deg = ARC_START_DEG + f * ARC_SPAN_DEG; + const rad = (deg * Math.PI) / 180; + return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) }; +} + +/** SVG path `d` for the full 0..1 arc (clockwise). */ +function fullArcD() { + const a = arcPoint(0); + const b = arcPoint(1); + const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0; + return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`; +} + +/** One-time geometry: track, fill (dash-revealed) and the transparent hit band. */ +function initGateArc() { + const d = fullArcD(); + mgaTrack.setAttribute("d", d); + mgaFill.setAttribute("d", d); + mgaHit.setAttribute("d", d); + // pathLength 100 lets us reveal the fill by fraction via dashoffset. + mgaFill.setAttribute("pathLength", "100"); + mgaFill.style.strokeDasharray = "100 100"; + mgaFill.style.strokeDashoffset = "100"; // empty until levels arrive + renderGateHandle(); +} + +/** Place the threshold bead on the arc at the stored threshold; flag off state. */ +function renderGateHandle() { + const off = settings.noiseGate <= GATE_OFF_DB; + const p = arcPoint(dbToFraction(settings.noiseGate)); + mgaHandle.setAttribute("cx", String(p.x)); + mgaHandle.setAttribute("cy", String(p.y)); + micGate.classList.toggle("gate-off", off); +} + +/** Paint a 0..1 live level onto the arc fill (and the Settings meter if open). + * Brightens the tick when the level crosses the threshold — i.e. the gate is + * actually open — but only when gating is enabled. + * @param {number} rms */ +function paintInputLevel(rms) { + const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB; + const f = dbToFraction(db); + mgaFill.style.strokeDashoffset = String(100 * (1 - f)); + if (settingsModal.open) gateMeterFill.style.width = `${f * 100}%`; + const enabled = settings.noiseGate > GATE_OFF_DB; + micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(settings.noiseGate)); +} + +/** The single place that commits a new gate threshold: updates both controls, + * persists, and applies live to the running session. + * @param {number} db */ +function setGateThreshold(db) { + settings.noiseGate = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db))); + const off = settings.noiseGate <= GATE_OFF_DB; + inputNoiseGate.value = String(settings.noiseGate); + gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; + renderGateHandle(); + localStorage.setItem(STORAGE_KEYS.noiseGate, String(settings.noiseGate)); + if (client && LIVE_STATES.has(currentState)) { + client.setNoiseGate(gateParams(settings.noiseGate)); + } +} + +/** Reflect the stored gate threshold into the slider, label and arc handle. */ +function syncGateUi() { + inputNoiseGate.value = String(settings.noiseGate); + const off = settings.noiseGate <= GATE_OFF_DB; + gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; + renderGateHandle(); +} + +// Drag along the arc band to set the threshold (a tap on the glyph still mutes). +let gateDragging = false; +/** @param {PointerEvent} e */ +function gatePointerToDb(e) { + const rect = mgaArc.getBoundingClientRect(); + const cx = rect.left + rect.width / 2; + const cy = rect.top + rect.height / 2; + let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI; + if (deg < 0) deg += 360; + // Map the on-arc angle to a fraction; angles in the right-side gap fall + // outside [0,1] and fractionToDb clamps them to the nearest end (just-below + // start -> Off, just-past end -> max). + const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG; + return fractionToDb(f); +} +mgaHit.addEventListener("pointerdown", (e) => { + gateDragging = true; + mgaHit.setPointerCapture(e.pointerId); + setGateThreshold(gatePointerToDb(e)); +}); +mgaHit.addEventListener("pointermove", (e) => { + if (gateDragging) setGateThreshold(gatePointerToDb(e)); +}); +const endGateDrag = (/** @type {PointerEvent} */ e) => { + if (!gateDragging) return; + gateDragging = false; + try { mgaHit.releasePointerCapture(e.pointerId); } catch {} +}; +mgaHit.addEventListener("pointerup", endGateDrag); +mgaHit.addEventListener("pointercancel", endGateDrag); + +settingsBtn.addEventListener("click", openSettings); + +// About panel: native , Esc closes for free; also close on the X and +// on a click in the backdrop (a click whose target is the dialog itself). +aboutBtn.addEventListener("click", () => aboutModal.showModal()); +// Mobile twin of the (i), living in the right-hand control cluster. +$("#about-btn-m").addEventListener("click", () => aboutModal.showModal()); +aboutClose.addEventListener("click", () => aboutModal.close()); +aboutModal.addEventListener("click", (e) => { + if (e.target === aboutModal) aboutModal.close(); +}); + +// ── Tools panel ─────────────────────────────────────────────────────────── + +/** Reflect the current tool state into the panel controls. */ +function syncToolsUi() { + const avail = searchAvailable(); + toolWebSwitch.checked = toolsEnabled.web_search && avail; + toolWebSwitch.disabled = !avail; + toolWebRow.classList.toggle("disabled", !avail); + toolCamSwitch.checked = toolsEnabled.camera_snapshot; + + if (serverSearchKey) { + // Key lives server-side: show it as configured, never expose it. + searchKeyInput.value = ""; + searchKeyInput.placeholder = "•••••••• · provided by the server"; + searchKeyInput.disabled = true; + toolWebHint.textContent = "Ready. The search key is held server-side and never sent to your browser."; + } else { + searchKeyInput.disabled = false; + searchKeyInput.value = userSearchKey; + searchKeyInput.placeholder = "Paste a Serper key to enable web search"; + toolWebHint.textContent = userSearchKey + ? "Using your key — stored in this browser only." + : "No server key configured. Add your own Serper key to enable web search."; + } +} + +toolsBtn.addEventListener("click", () => { syncToolsUi(); toolsModal.showModal(); }); +toolsClose.addEventListener("click", () => toolsModal.close()); +toolsModal.addEventListener("click", (e) => { + if (e.target === toolsModal) toolsModal.close(); +}); + +toolWebSwitch.addEventListener("change", () => { + if (toolWebSwitch.checked && !searchAvailable()) { + toolWebSwitch.checked = false; // guard: can't enable without a key + return; + } + toolsEnabled.web_search = toolWebSwitch.checked; + saveTools(); + pushToolsToSession(); +}); + +toolCamSwitch.addEventListener("change", async () => { + if (toolCamSwitch.checked) { + try { + // Flipping the switch always re-requests the camera, so a permission that + // was only dismissed earlier is asked again here. + await enableCamera(); + } catch (err) { + toolCamSwitch.checked = false; + const denied = err instanceof Error && (err.name === "NotAllowedError" || err.name === "SecurityError"); + toolCamHint.textContent = denied + ? "Camera blocked. Allow it from the camera icon in your browser's address bar — it switches on automatically." + : `Camera unavailable${err instanceof Error ? `: ${err.message}` : ""}`; + return; + } + toolsEnabled.camera_snapshot = true; + toolCamHint.textContent = "Camera on. The assistant can take a snapshot when it needs to see."; + } else { + disableCamera(); + toolsEnabled.camera_snapshot = false; + toolCamHint.textContent = "Let the assistant see through your webcam."; + } + saveTools(); + pushToolsToSession(); +}); + +searchKeyInput.addEventListener("input", () => { + if (serverSearchKey) return; + userSearchKey = searchKeyInput.value.trim(); + if (userSearchKey) localStorage.setItem(STORAGE_KEYS.searchKey, userSearchKey); + else localStorage.removeItem(STORAGE_KEYS.searchKey); + + const avail = searchAvailable(); + toolWebSwitch.disabled = !avail; + toolWebRow.classList.toggle("disabled", !avail); + // Losing the key disables a previously-enabled tool. + if (!avail && toolsEnabled.web_search) { + toolsEnabled.web_search = false; + toolWebSwitch.checked = false; + saveTools(); + pushToolsToSession(); + } + toolWebHint.textContent = userSearchKey + ? "Using your key — stored in this browser only." + : "No server key configured. Add your own Serper key to enable web search."; +}); + +// ── Camera ────────────────────────────────────────────────────────────────── + +async function enableCamera() { + if (cameraStream) return; + cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { facingMode: "user" }, + audio: false, + }); + camVideo.srcObject = cameraStream; + try { await camVideo.play(); } catch { /* autoplay quirks; muted video is fine */ } + camPip.classList.add("visible"); + camPip.setAttribute("aria-hidden", "false"); + // Lets the footer reflow to the bottom-right (and hide on mobile) while the + // webcam preview occupies the bottom of the stage. + document.body.classList.add("cam-on"); +} + +function disableCamera() { + if (cameraStream) { + for (const t of cameraStream.getTracks()) t.stop(); + cameraStream = null; + } + camVideo.srcObject = null; + camPip.classList.remove("visible"); + camPip.setAttribute("aria-hidden", "true"); + document.body.classList.remove("cam-on"); +} + +/** Auto-start the webcam on arrival (the camera tool is on by default). If the + * user declines the permission, switch the tool off and reflect it in the UI + * rather than nagging. */ +async function autoStartCamera() { + if (!toolsEnabled.camera_snapshot || cameraStream) return; + try { + await enableCamera(); + } catch (err) { + console.warn("[main] camera auto-start declined/failed:", err); + toolsEnabled.camera_snapshot = false; + saveTools(); + syncToolsUi(); + } +} + +/** Track the browser's camera permission so a later re-grant (e.g. the user + * unblocks it from the address bar after a denial) turns the camera back on + * without another toggle, and a revoke turns it off. Best-effort: the + * Permissions API doesn't support "camera" everywhere (e.g. Safari). */ +async function watchCameraPermission() { + try { + const status = await navigator.permissions?.query?.({ name: /** @type {any} */ ("camera") }); + if (!status) return; + status.addEventListener("change", () => { + if (status.state === "granted") { + if (!toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = true; saveTools(); } + void autoStartCamera(); + syncToolsUi(); + } else if (status.state === "denied") { + disableCamera(); + if (toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = false; saveTools(); } + syncToolsUi(); + } + }); + } catch { + // Permissions API unavailable for "camera" — the toggle still re-asks. + } +} + +/** + * Grab the current webcam frame as a downscaled JPEG data URL. The preview is + * mirrored in CSS for a natural self-view, but we draw the raw (un-mirrored) + * video here so the model sees the scene in its true orientation. + * @returns {string | null} + */ +function captureSnapshot() { + if (!cameraStream || !camVideo.videoWidth) return null; + const vw = camVideo.videoWidth; + const vh = camVideo.videoHeight; + const scale = Math.min(1, SNAPSHOT_MAX_EDGE / Math.max(vw, vh)); + const w = Math.max(1, Math.round(vw * scale)); + const h = Math.max(1, Math.round(vh * scale)); + const canvas = document.createElement("canvas"); + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext("2d"); + if (!ctx) return null; + ctx.drawImage(camVideo, 0, 0, w, h); + return canvas.toDataURL("image/jpeg", SNAPSHOT_QUALITY); +} + +/** Brief shutter flash on the preview so the user sees a snapshot was taken. */ +function flashPreview() { + camPip.classList.remove("flash"); + void camPip.offsetWidth; // reflow so the animation restarts + camPip.classList.add("flash"); +} + +// ── Tool executor ───────────────────────────────────────────────────────── +// Runs the function the model called, returns the result, and asks for a +// response so the model speaks it. Errors come back as the tool output too, so +// the model can recover gracefully instead of the turn stalling. + +/** + * Run the function the model called, return its result to the backend, and ask + * for a follow-up response. We also hand the result back to the caller so it + * can be shown in the conversation once the tool has actually run. + * @param {string} name @param {string} argsJson @param {string} callId + * @returns {Promise<{ output: string, image?: string }>} + */ +async function runTool(name, argsJson, callId) { + if (!client) return { output: "" }; + let args = /** @type {Record} */ ({}); + try { args = JSON.parse(argsJson || "{}"); } catch { /* keep {} */ } + + if (DEBUG) console.debug(`[tool] run name=${name} callId=${JSON.stringify(callId)} args=${argsJson}`); + if (!callId) console.warn("[tool] empty call_id — the backend didn't tag the call, can't return a function_call_output"); + + /** @type {{ output: string, image?: string }} */ + let result = { output: "" }; + try { + if (name === "web_search") { + const query = typeof args.query === "string" ? args.query : ""; + result.output = await execWebSearch(query); + // Return the result and let the bare response.create (below) trigger the + // spoken answer. + client.sendToolOutput(callId, result.output); + } else if (name === "camera_snapshot") { + const dataUrl = captureSnapshot(); + if (dataUrl) { + if (DEBUG) console.debug(`[tool] camera_snapshot captured frame (${dataUrl.length} chars), sending image + output`); + result = { output: "Snapshot captured from the webcam and attached as an image.", image: dataUrl }; + // Return the tool output; the frame itself rides along with the + // response.create below (sent right before it), so the model sees the + // snapshot in the very response it's about to speak. + client.sendToolOutput(callId, result.output); + flashPreview(); + } else { + console.warn("[tool] camera_snapshot: no frame — camera off or not ready"); + result.output = "The camera is not available right now."; + client.sendToolOutput(callId, result.output); + } + } else { + result.output = `Unknown tool: ${name}`; + client.sendToolOutput(callId, result.output); + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + result.output = `Tool failed: ${msg}`; + client.sendToolOutput(callId, result.output); + } + if (DEBUG) console.debug(`[tool] requesting model response after ${name}`); + // Camera: the captured frame rides with the response.create (sent just before + // it) so it's in context for the reply. Other tools: a bare create. + client.requestResponse(result.image ? { image: result.image } : undefined); + return result; +} + +/** @param {string} query @returns {Promise} */ +async function execWebSearch(query) { + if (!query) return "No query provided."; + /** @type {Record} */ + const body = { query }; + // Only send a user key when there's no server key (server prefers its own). + if (!serverSearchKey && userSearchKey) body.key = userSearchKey; + + const res = await fetch("api/search", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) { + let detail = String(res.status); + try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {} + throw new Error(`search error (${detail})`); + } + const json = await res.json(); + // Date-stamp the header so the model treats these as fresh realtime facts + // rather than its (older) training knowledge. + const today = new Date().toISOString().slice(0, 10); + /** @type {string[]} */ + const lines = [`Google search result from ${today}:`]; + if (json.answer) lines.push(`Answer: ${json.answer}`); + for (const r of json.results || []) { + lines.push(`- ${r.title}: ${r.snippet} (${r.url})`); + } + return lines.length > 1 ? lines.join("\n") : `${lines[0]}\nNo results found.`; +} + +/** Learn server config (search key + connection target), then refresh the UI. */ +async function fetchConfig() { + try { + const res = await fetch("api/config"); + if (res.ok) { + const json = await res.json(); + serverSearchKey = !!json.search; + lbMode = !!json.lb; + // Lock to LB mode only when the deploy reports a load balancer. + allowDirect = json.allowDirect ?? !lbMode; + // Deploy-pinned direct URL (overrides the LB server-side already). + pinnedUrl = (json.s2sUrl || "").trim(); + // The conversation-time limiter rides on the LB being present. + limiterOn = lbMode; + } + // Non-OK response: leave the fail-open default (allowDirect = true). + } catch { + // Config endpoint unreachable (e.g. static hosting): keep direct entry. + } + if (DEBUG) console.debug(`[ui] config: allowDirect=${allowDirect} lbMode=${lbMode}`); + // Login chip + remaining-budget (no-op / hidden when the limiter is off). + void account.refresh(); + syncToolsUi(); + syncConnectionUi(); +} + +/** + * Resolve where to connect, per the deploy's mode: + * • LB mode -> `{ sessionUrl }`, the client POSTs the same-origin /api/session + * proxy and the server forwards to the LB (its address stays server-side). + * • direct -> `{ directUrl }`, connect straight to the s2s WebSocket. + * Throws a user-facing error if direct mode is on but no URL was entered. + * @returns {{ sessionUrl: string } | { directUrl: string }} + */ +function connectionTarget() { + if (!allowDirect) { + return { sessionUrl: "api/session" }; + } + const directUrl = buildDirectWsUrl(pinnedUrl || settings.directUrl); + if (!directUrl) { + throw new Error("Enter a speech-to-speech server URL in Settings."); + } + return { directUrl }; +} + +/** + * Normalise a user-typed server address into a realtime WebSocket URL. + * Accepts bare hosts (`localhost:8080`), http(s) URLs, or ws(s) URLs, and adds + * the `/v1/realtime` path when none is given. A full connect URL (with path + * and/or query) is preserved as-is. + * @param {string} raw @returns {string} + */ +function buildDirectWsUrl(raw) { + let s = (raw || "").trim(); + if (!s) return ""; + if (!/^wss?:\/\//i.test(s)) { + if (/^https?:\/\//i.test(s)) { + s = s.replace(/^http/i, "ws"); // http→ws, https→wss + } else { + const isLocal = /^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(s); + s = (isLocal ? "ws://" : "wss://") + s; + } + } + try { + const u = new URL(s); + if (u.pathname === "" || u.pathname === "/") u.pathname = "/v1/realtime"; + return u.toString(); + } catch { + return s; + } +} + +/** Create + resume an AudioContext synchronously (must run inside the user + * gesture so iOS lets it start). Returns null if construction fails. */ +function createResumedAudioContext() { + try { + const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext; + const ctx = new Ctx({ latencyHint: "interactive" }); + if (ctx.state === "suspended") void ctx.resume().catch(() => {}); + return /** @type {AudioContext} */ (ctx); + } catch (err) { + console.warn("[main] AudioContext init failed:", err); + return null; + } +} + +/** Read the editable settings out of the form. The URL field is only honoured + * in free direct mode — in LB mode it's hidden, and when the deploy pins a + * URL it's read-only, so the user's saved URL survives either way. */ +function readSettingsFromForm() { + return { + directUrl: allowDirect && !pinnedUrl ? inputLbUrl.value.trim() : settings.directUrl, + voice: inputVoice.value || DEFAULT_VOICE, + instructions: inputInstructions.value.trim() || DEFAULT_INSTRUCTIONS, + noiseGate: readGateThreshold(), + }; +} + +/** Gate threshold (dBFS) currently shown on the slider, clamped to range. */ +function readGateThreshold() { + const v = Math.round(Number(inputNoiseGate.value)); + if (!Number.isFinite(v)) return GATE_OFF_DB; + return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, v)); +} + +/** Adapt the connection field to the mode learned from /api/config. */ +function syncConnectionUi() { + if (pinnedUrl) { + // Deploy-pinned URL: show it, but locked — the deployment owns it. + connField.hidden = false; + inputLbUrl.value = pinnedUrl; + inputLbUrl.readOnly = true; + connHint.classList.remove("error"); + connHint.textContent = "Speech-to-speech server URL pinned by this deployment."; + } else if (allowDirect) { + // Direct mode: the user sets their own s2s server URL. + connField.hidden = false; + inputLbUrl.value = settings.directUrl; + inputLbUrl.readOnly = false; + inputLbUrl.placeholder = "http://localhost:port"; + connHint.classList.remove("error"); + connHint.textContent = + "URL of your speech-to-speech server, e.g. http://localhost:8080 (the app adds /v1/realtime)."; + } else { + // LB mode: the load balancer URL is deployment-owned — hide it entirely so + // its address is never exposed in Settings. + connField.hidden = true; + } +} + +/** True when the user must supply a server URL before connecting (direct mode + * with nothing set). */ +function missingServerUrl() { + return allowDirect && !pinnedUrl && !buildDirectWsUrl(settings.directUrl); +} + +/** Open Settings and point the user at the empty server-URL field. */ +function promptServerUrl() { + if (settingsModal.open) syncConnectionUi(); + else openSettings(); + connHint.textContent = "Set the speech-to-speech server URL to start."; + connHint.classList.add("error"); + inputLbUrl.focus(); +} + +settingsForm.addEventListener("submit", (event) => { + const submitter = /** @type {HTMLButtonElement | null} */ ((/** @type {SubmitEvent} */ (event)).submitter); + if (submitter?.value !== "save") return; + + settings = readSettingsFromForm(); + saveSettings(settings); + + // Voice + instructions can apply to a live session without reconnecting; a + // changed connection URL only takes effect on the next restart. + if (client && LIVE_STATES.has(currentState)) { + client.updateSession({ voice: settings.voice, instructions: effectiveInstructions() }); + } +}); + +// The noise gate applies live (worklet param), so tune it without a restart: +// update the label/marker, persist, and push straight to the running client. +inputNoiseGate.addEventListener("input", () => { + setGateThreshold(readGateThreshold()); +}); + +restartBtn.addEventListener("click", async () => { + if (currentState === "connecting") return; // a connect is already underway + settings = readSettingsFromForm(); + saveSettings(settings); + if (missingServerUrl()) { promptServerUrl(); return; } // keep settings open + settingsModal.close(); + // Grab the AudioContext NOW, inside the click gesture — teardown() awaits, and + // creating it afterwards would fall outside the gesture (silent on iOS). + const audioContext = createResumedAudioContext(); + try { + if (client) await teardown(); + await doStart(audioContext); + } catch (err) { + await handleStartError(err); + } +}); + +circleBtn.addEventListener("click", async () => { + try { + if (currentState === "idle" || currentState === "error") { + if (missingServerUrl()) { promptServerUrl(); return; } + await doStart(); + } + } catch (err) { + await handleStartError(err); + } +}); + +/** A failed start is either the daily limit (show the modal, return to idle) or + * a real fault (surface it). doStart already closed any orphan AudioContext. + * @param {any} err */ +async function handleStartError(err) { + if (err && err.code === "limit") { + await teardown(); + account.showLimit(err.tier); + return; + } + // The user left the queue (close() aborted the wait): teardown already reset + // the UI to idle, so there's nothing to report. + if (err && err.code === "aborted") return; + // The whole waiting line is full: a warm, reassuring modal rather than an error. + if (err && err.code === "queue-full") { + await teardown(); + account.showBusy(); + return; + } + // Our place lapsed (ticket reaped, or the join window ran out). Recoverable, not + // a fault: land on the retry state with a kind, plain-language reason. + if (err && (err.code === "queue-expired" || err.code === "join-expired")) { + await teardown(); + setState("error"); + setCaption( + err.code === "join-expired" + ? "Your spot expired. Tap to rejoin." + : "That took a while. Tap to rejoin.", + "error", + ); + return; + } + onFatalError(err); +} + +micBtn.addEventListener("click", () => { + if (!micStream || !client) return; + micMuted = !micMuted; + for (const track of micStream.getAudioTracks()) { + track.enabled = !micMuted; + } + client.setMuted(micMuted); + micBtn.classList.toggle("muted", micMuted); + micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute"); + micBtn.title = micMuted ? "Unmute" : "Mute"; +}); + +stopBtn.addEventListener("click", async () => { + await teardown(); +}); + +// "Leave queue": tear down the pending connect (aborts the poll wait) and drop +// our place in line. Same teardown path as stopping a live call. +leaveQueueBtn.addEventListener("click", async () => { + await teardown(); +}); + +// "Join now": accept the held slot. The click is a user gesture, so the client +// re-resumes the AudioContext here (iOS) before dialing. +joinQueueBtn.addEventListener("click", () => { + stopJoinCountdown(); + if (client) client.join(); +}); + +const MIC_CONSTRAINTS = { + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, +}; + +/** Prompt for mic permission up front, then immediately release the tracks so no + * recording indicator lingers during a queue wait. Throws a friendly error if the + * user denies. */ +async function primeMicPermission() { + try { + const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); + for (const track of s.getTracks()) track.stop(); + } catch (err) { + throw new Error( + `Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`, + ); + } +} + +/** Acquire the live capture stream once a slot is granted. Permission was primed + * in the tap gesture, so this is silent. Stored module-side for mute + teardown. */ +async function acquireMicStream() { + micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); + return micStream; +} + +/** @param {number} position Update the queued caption ("You're #N in line"). */ +function onQueuePosition(position) { + const n = Number(position) || 0; + setCaption(n > 0 ? `You're #${n} in line` : "Finding you a spot…", "muted"); +} + +// ── "Your turn" join countdown ────────────────────────────────────────────── +// While a slot is held for us, show how long is left to accept it. The client's +// join gate expires just before the load balancer reclaims the slot. +let joinCountdownTimer = 0; + +/** @param {number} sec */ +function startJoinCountdown(sec) { + stopJoinCountdown(); + let left = Math.max(0, Math.floor(sec)); + const paint = () => { + joinQueueBtn.textContent = left > 0 ? `Join now (${left}s)` : "Join now"; + }; + paint(); + joinCountdownTimer = window.setInterval(() => { + left -= 1; + if (left <= 0) { + stopJoinCountdown(); + joinQueueBtn.textContent = "Join now"; + return; + } + paint(); + }, 1000); +} + +function stopJoinCountdown() { + if (joinCountdownTimer) { + clearInterval(joinCountdownTimer); + joinCountdownTimer = 0; + } +} + +/** + * Start a conversation. Pass a pre-created AudioContext when the caller already + * made one inside the tap/click gesture (required on iOS); otherwise one is + * created here, which is still inside the gesture for a direct orb tap. + * @param {AudioContext | null} [audioContext] + */ +async function doStart(audioContext = null) { + // Resolve the target before touching mic/audio so a misconfiguration (e.g. + // direct mode with no URL) fails fast with a clear message. + const target = connectionTarget(); + + chat.clear(); + chat.reset(); + setState("connecting"); + setCaption("Asking for mic…", "muted"); + + // Create + resume the AudioContext SYNCHRONOUSLY, still inside the gesture. + // iOS Safari only starts an AudioContext from a user gesture; if we waited + // until after the getUserMedia / session-creation awaits below, it would stay + // suspended and the whole pipeline would be silent. + if (!audioContext) audioContext = createResumedAudioContext(); + + // Prime the mic permission now (get the prompt out of the way up front), then + // release it. The real capture stream is acquired only once a slot is granted + // (see acquireMicStream), so the mic 'in use' indicator never lights while we + // sit in the queue. Permission persists, so the later acquire is silent. + try { + await primeMicPermission(); + } catch (err) { + if (audioContext) void audioContext.close().catch(() => {}); + throw err; + } + + // The webcam is started on arrival (autoStartCamera), so nothing to do here; + // a still-pending grant just means the snapshot tool isn't ready yet. + + const c = new S2sWsRealtimeClient({ + ...target, + voice: settings.voice, + instructions: effectiveInstructions(), + acquireMic: acquireMicStream, + tools: activeToolDefs(), + noiseGate: gateParams(settings.noiseGate), + ...(audioContext ? { audioContext } : {}), + }); + client = c; + + c.addEventListener("queue", (e) => { + const { position, queueId } = /** @type {CustomEvent<{ position: number; queueId: string }>} */ (e).detail; + if (queueId) queuedTicketId = queueId; + onQueuePosition(position); + }); + + c.addEventListener("ready-to-join", (e) => { + const { info, expiresSec } = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo; expiresSec: number }>} */ (e).detail; + // A slot is held for us. We're out of the queue now, so drop the ticket ref. + // Track the granted session id already so that leaving (or letting the timer + // lapse) refunds the budget the server reserved at claim, even before we dial. + queuedTicketId = ""; + if (info?.sessionId) { + trackedSessionId = info.sessionId; + trackedTier = info.tier || "anon"; + } + startJoinCountdown(expiresSec); + }); + + c.addEventListener("status", (e) => { + const detail = /** @type {CustomEvent<{ status: string }>} */ (e).detail; + onClientStatus(detail.status); + }); + c.addEventListener("transcript", (e) => { + const d = /** @type {CustomEvent<{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }>} */ (e).detail; + chat.onTranscript(d); + }); + + c.addEventListener("response-finished", (e) => { + const detail = /** @type {CustomEvent<{ responseId: string; status: string; audible?: boolean; transcript?: string }>} */ (e).detail; + chat.onResponseFinished(detail); + }); + + c.addEventListener("toolcall", (e) => { + const { name, arguments: args, callId } = /** @type {CustomEvent<{ name: string; arguments: string; callId: string }>} */ (e).detail; + chat.onToolCall(name); + // Execute the tool, then push it to the conversation once the result is in, + // so the toggle shows both the call input and its output together. + void runTool(name, args, callId).then(({ output, image }) => { + chat.onToolResult(name, args, output, image); + }); + }); + c.addEventListener("error", (e) => { + const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail; + onFatalError(detail.error); + }); + c.addEventListener("server-error", (e) => { + // Non-fatal: the backend reported an error mid-session. Log it, keep the + // socket and the conversation alive (the model can recover on its own). + const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail; + const msg = detail.error instanceof Error ? detail.error.message : String(detail.error); + console.warn("[main] server error (non-fatal):", msg); + }); + c.addEventListener("session", (e) => { + const info = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo }>} */ (e).detail.info; + console.log("[ws] session created:", info.sessionId); + // A slot was granted — we're out of the queue; drop the ticket reference so + // teardown doesn't try to leave a line we already left. + queuedTicketId = ""; + // A metered tier (anon / free): heartbeat so the server can extend the + // reservation and tell us when the daily budget runs out. PRO isn't limited. + if (info.limited && info.sessionId) { + trackedSessionId = info.sessionId; + trackedTier = info.tier || "anon"; + startHeartbeat(info.heartbeatSec || 5); + } + }); + c.addEventListener("input-level", (e) => { + const { rms } = /** @type {CustomEvent<{ rms: number }>} */ (e).detail; + paintInputLevel(rms); + }); + + try { + await c.connect(); + } catch (err) { + // The grant can be refused (402 → limit) or the dial can fail. In LB mode + // the AudioContext hasn't been adopted by the client yet (the session POST + // runs first), so close the one we created here to avoid leaking it. + if (audioContext) void audioContext.close().catch(() => {}); + throw err; + } +} + +// ── Conversation-time heartbeat ───────────────────────────────────────────── + +/** Ping the server every `sec` seconds so it can meter the live session; when + * it reports the daily budget is spent, cut the call and show the limit modal. + * @param {number} sec */ +function startHeartbeat(sec) { + stopHeartbeat(); + heartbeatTimer = window.setInterval(async () => { + if (!trackedSessionId) return; + try { + const res = await fetch("api/session/heartbeat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionId: trackedSessionId }), + keepalive: true, + }); + const json = await res.json().catch(() => ({})); + if (json.expired) await onLimitReached(); + } catch (err) { + // A transient network blip shouldn't kill the call; the next tick retries. + if (DEBUG) console.debug("[ui] heartbeat failed:", err); + } + }, Math.max(1, sec) * 1000); +} + +function stopHeartbeat() { + if (heartbeatTimer) { + clearInterval(heartbeatTimer); + heartbeatTimer = 0; + } +} + +/** The server cut the live session: tear down and explain why. */ +async function onLimitReached() { + const tier = trackedTier; + stopHeartbeat(); + await teardown(); + account.showLimit(tier); +} + +/** Tell the server a session ended so it reconciles + refunds the unused chunk. + * Uses sendBeacon so it still fires when the tab is closing. */ +function endTrackedSession() { + if (!trackedSessionId) return; + const body = JSON.stringify({ sessionId: trackedSessionId }); + try { + const blob = new Blob([body], { type: "application/json" }); + if (!navigator.sendBeacon("api/session/end", blob)) { + void fetch("api/session/end", { + method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, + }).catch(() => {}); + } + } catch { + // Best-effort; the server sweep reaps the session anyway. + } + trackedSessionId = ""; + trackedTier = ""; +} + +/** Leave the waiting queue so the LB frees our place. sendBeacon so it still + * fires on tab close; the LB also reaps the ticket on TTL as a backstop. */ +function endQueueTicket() { + if (!queuedTicketId) return; + const body = JSON.stringify({ queueId: queuedTicketId }); + try { + const blob = new Blob([body], { type: "application/json" }); + if (!navigator.sendBeacon("api/queue/end", blob)) { + void fetch("api/queue/end", { + method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, + }).catch(() => {}); + } + } catch { + // Best-effort; the LB reaps the ticket on TTL anyway. + } + queuedTicketId = ""; +} + +/** @param {string} status */ +function onClientStatus(status) { + switch (status) { + case "creating-session": + case "connecting": + setState("connecting"); + break; + case "queued": + setState("queued"); + break; + case "your-turn": + setState("your-turn"); + break; + case "connected": + setState("listening"); + break; + case "user-speaking": + setState("user-speaking"); + break; + case "processing": + setState("processing"); + break; + case "ai-speaking": + setState("ai-speaking"); + break; + case "closed": + // teardown() will move us to idle + break; + case "error": + setState("error"); + break; + } +} + +async function teardown() { + stopHeartbeat(); + stopJoinCountdown(); + endTrackedSession(); + endQueueTicket(); + chat.reset({ dismiss: true }); + if (client) { + try { + await client.close(); + } catch (err) { + console.warn("[main] error closing client:", err); + } + client = null; + } + if (micStream) { + for (const track of micStream.getTracks()) track.stop(); + micStream = null; + } + // The webcam is independent of the call lifecycle (it runs while the user is + // on the page), so we leave it on here — only the camera toggle stops it. + micMuted = false; + micBtn.classList.remove("muted"); + setState("idle"); + // Refresh the chip's remaining-today after the budget moved. + if (limiterOn) void account.refresh(); +} + +/** @param {unknown} err */ +function onFatalError(err) { + console.error("[main] fatal:", err); + setState("error"); + const message = err instanceof Error ? err.message : String(err); + setCaption(truncateError(message), "error"); + void teardown().catch(() => { + setState("error"); + setCaption(truncateError(message), "error"); + }); +} + +setState("idle"); +chat.renderEmptyState(); +initGateArc(); +void fetchConfig(); +// Start the webcam as soon as the user lands (camera tool defaults on), and +// react to later permission changes (re-grant after a denial re-enables it). +void autoStartCamera(); +void watchCameraPermission(); + +// Reconcile a live session if the tab is closed/hidden mid-call (no teardown). +window.addEventListener("pagehide", () => { endTrackedSession(); endQueueTicket(); }); + +requestAnimationFrame(() => { + document.body.classList.remove("booting"); +}); diff --git a/demo/requirements.txt b/demo/requirements.txt new file mode 100644 index 0000000..fc64b9c --- /dev/null +++ b/demo/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +httpx==0.28.1 +# [oauth] extra pulls authlib + itsdangerous for attach_huggingface_oauth. +huggingface_hub[oauth]>=0.30,<1.0 diff --git a/demo/server.py b/demo/server.py new file mode 100644 index 0000000..ba6b45e --- /dev/null +++ b/demo/server.py @@ -0,0 +1,447 @@ +""" +Tiny server for the speech-to-speech demo. + +The demo used to ship as a `sdk: static` Space, but the web-search tool needs a +search key the browser must NOT see. A static Space has no runtime process, so it +can't hold a secret the front-end uses. This server fixes that: it serves the +unchanged front-end AND exposes a same-origin `/api/search` proxy that holds the +Serper key server-side (see docs/adr/0001). + +Everything lives in one container; the speech-to-speech backend stays a separate, +load-balanced service the browser talks to over WebSocket as before. The load +balancer's address is a secret too (like the Serper key): the browser never sees +it. `/api/session` proxies the session handshake server-side so only the +per-session compute URL the LB hands back (which the browser must dial) is exposed. + +On the deployed Space the server also meters conversation time by HF login tier +(anonymous / signed-in / PRO) — see `limiter.py` and `auth.py`. That whole feature +is off unless BOTH `LOAD_BALANCER_URL` and `SPACE_ID` are set, so it runs only on +the live Space, never locally (even with the LB exported for testing). + +`SPEECH_TO_SPEECH_URL` overrides everything: when set, the LB logic above is +disabled entirely (no session proxy, no queue, no metering, no sign-in) and the +browser connects directly to that URL, shown read-only in Settings. + +Endpoints: + GET /api/config -> { search, lb, allowDirect, s2sUrl, auth } + GET /api/me -> login + tier + remaining budget (LB mode only) + POST /api/search -> { results, answer } Google via Serper.dev + POST /api/session -> proxies /session: a grant, or a queue ticket + GET /api/queue/{id} -> proxies /queue/{id}: position, or a grant on claim + DELETE /api/queue/{id} -> leave the queue (explicit "Leave queue" button) + POST /api/queue/end -> leave the queue (sendBeacon on teardown) + POST /api/session/heartbeat-> extend the reservation; { expired } + POST /api/session/end -> reconcile + refund (sendBeacon on teardown) + /* -> static files (index.html, main.js, ...) + +When every compute slot is busy the load balancer hands back a queue ticket +instead of a grant; the browser polls /api/queue/{id} until it reaches the front +and a slot frees. Waiting reserves nothing — the daily budget is only reserved at +the moment a slot is actually claimed (a grant), never while queued. +""" + +import asyncio +import logging +import os + +import httpx +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel + +import auth +import limiter + +logger = logging.getLogger("s2s.search") + +SERPER_KEY = os.environ.get("SERPER_API_KEY", "").strip() +# Speech-to-speech load balancer URL. When set, the browser POSTs /api/session +# (which proxies /session here, server-side) and connects to the URL the LB +# returns (the original flow). The LB address itself is never sent to the browser. +# When empty, the user may instead set a direct s2s server URL in Settings and the +# browser connects to it straight (no load balancer). +LOAD_BALANCER_URL = os.environ.get("LOAD_BALANCER_URL", "").strip() +# Direct s2s server URL pinned by the deploy. Takes priority over the load +# balancer: when set, ALL LB logic is disabled (no /api/session proxy, no queue, +# no limiter, no sign-in) and the browser connects to this URL directly. Unlike +# the LB address it is NOT a secret — /api/config sends it to the client, which +# shows it read-only in Settings. +SPEECH_TO_SPEECH_URL = os.environ.get("SPEECH_TO_SPEECH_URL", "").strip() +if SPEECH_TO_SPEECH_URL: + LOAD_BALANCER_URL = "" +# HF injects SPACE_ID ("owner/space") into every Space runtime; it's absent +# locally and on a plain `docker run`. We meter conversation time ONLY on the +# deployed Space — i.e. when BOTH the LB is configured AND we're on a Space. +# Off-Space (local dev, even with the LB exported) the app still proxies the LB, +# but nothing is metered: no budget, no reservations, no sign-in gating. +SPACE_ID = os.environ.get("SPACE_ID", "").strip() +LIMITER_ENABLED = bool(LOAD_BALANCER_URL) and bool(SPACE_ID) +SERPER_URL = "https://google.serper.dev/search" +# Cap results so the tool output stays small enough to feed back to the model. +MAX_RESULTS = 5 +HERE = os.path.dirname(os.path.abspath(__file__)) + +app = FastAPI(title="s2s-demo") + +# Wire HF OAuth before the app serves (no-op unless the OAuth env is present). +# Sign-in only matters when we're metering (prod Space), so gate it on that. +AUTH_ENABLED = LIMITER_ENABLED and auth.attach(app) + + +@app.on_event("startup") +async def _startup(): + """Stand up the usage DB and a periodic sweeper — metered (prod Space) only.""" + if not LIMITER_ENABLED: + return + limiter.init() + asyncio.create_task(_sweeper()) + + +async def _sweeper(): + while True: + await asyncio.sleep(limiter.REAP_AFTER_SEC) + try: + await asyncio.to_thread(limiter.sweep) + except Exception as exc: # pragma: no cover - defensive + logger.warning("usage sweep failed: %r", exc) + + +class SearchRequest(BaseModel): + query: str + # Optional user-supplied key (fallback when the deploy has no server key). + # Used for this request only; never stored. + key: str | None = None + + +@app.get("/api/config") +def config(): + """Client bootstrap: whether web search is available, whether the deploy runs + behind a load balancer (so the browser uses the /api/session proxy + limiter), + whether HF sign-in is available, and whether the user may instead set a direct + s2s server URL. The LB address itself is intentionally NOT included.""" + return { + "search": bool(SERPER_KEY), + "lb": bool(LOAD_BALANCER_URL), + "allowDirect": not LOAD_BALANCER_URL, + # Deploy-pinned direct s2s URL (empty when unset). Not a secret: the + # browser dials it itself, and Settings shows it locked. + "s2sUrl": SPEECH_TO_SPEECH_URL, + "auth": AUTH_ENABLED, + } + + +@app.get("/api/me") +async def me(request: Request): + """Login state, tier, and remaining daily budget. Only meaningful in LB mode; + sets the anonymous tracking cookie when first seen.""" + if not LIMITER_ENABLED: + return {"enabled": False} + view = auth.user_view(request) + tier, keys, set_cookie = auth.resolve_identity(request) + unlimited = limiter.budget_for(tier) is None + rem = None if unlimited else await asyncio.to_thread(limiter.remaining, keys, tier) + out = { + "enabled": True, + "auth": AUTH_ENABLED, + **view, + "remainingSec": rem, + "limitSec": limiter.budget_for(tier), + "loginUrl": auth.OAUTH_LOGIN_PATH if AUTH_ENABLED else None, + "logoutUrl": auth.OAUTH_LOGOUT_PATH if AUTH_ENABLED else None, + } + resp = JSONResponse(out) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + +@app.post("/api/search") +async def search(req: SearchRequest): + """Proxy a Google search via Serper.dev. The key stays on the server unless + the user brought their own (then theirs is used for this request only).""" + query = (req.query or "").strip() + if not query: + raise HTTPException(status_code=400, detail="Empty query.") + + key = (req.key or "").strip() or SERPER_KEY + if not key: + # No server key and the user didn't supply one — search is unavailable. + raise HTTPException(status_code=503, detail="Search is not configured.") + + headers = {"X-API-KEY": key, "Content-Type": "application/json"} + payload = {"q": query, "num": MAX_RESULTS} + try: + async with httpx.AsyncClient(timeout=12.0) as http: + resp = await http.post(SERPER_URL, headers=headers, json=payload) + except httpx.RequestError as exc: + logger.warning("Serper unreachable: %r", exc) + raise HTTPException(status_code=502, detail="Search provider unreachable.") + + if resp.status_code != 200: + # Serper's error body carries the real reason (e.g. "Not enough + # credits") and contains no key, so it's safe to log and relay. + body = resp.text[:300] + logger.warning("Serper error %s: %s", resp.status_code, body) + msg = None + try: + msg = resp.json().get("message") + except Exception: + pass + detail = f"Search provider error ({resp.status_code})" + if msg: + detail += f": {msg}" + raise HTTPException(status_code=502, detail=detail) + + data = resp.json() + results = [] + for item in (data.get("organic") or [])[:MAX_RESULTS]: + results.append( + { + "title": item.get("title", ""), + "snippet": item.get("snippet", ""), + "url": item.get("link", ""), + } + ) + + # A direct answer when Google has one — saves the model a hop. + box = data.get("answerBox") or {} + answer = box.get("answer") or box.get("snippet") or None + if not answer: + kg = data.get("knowledgeGraph") or {} + answer = kg.get("description") or None + + return JSONResponse({"query": query, "answer": answer, "results": results}) + + +@app.post("/api/session") +async def session(request: Request): + """Proxy the session handshake to the load balancer, keeping its URL secret, + and meter conversation time by tier. + + The browser POSTs here (same-origin); we resolve the caller's tier, refuse if + today's budget is already spent (402), otherwise POST /session + and relay the JSON back. The LB body carries a per-session `connect_url` + (compute host + short-lived token) the browser must dial directly — that one + URL is unavoidably exposed, but the stable load-balancer address is not. On a + successful grant we reserve the first time chunk against the day's budget.""" + if not LOAD_BALANCER_URL: + # No LB configured — this deploy is direct-mode only; the browser should + # never call this. 404 so it's indistinguishable from a missing route. + raise HTTPException(status_code=404, detail="Not found.") + + tier, keys, set_cookie = auth.resolve_identity(request) + # Metering runs only on the deployed Space; off-Space the LB still proxies but + # nothing is tracked. Within metering, unlimited tiers (pro, org) aren't either. + tracked = LIMITER_ENABLED and limiter.budget_for(tier) is not None + + # Refuse before troubling the LB if the day's budget is already gone. Done + # here (at enqueue) so we never put a user who can't talk into the queue. + if tracked: + rem = await asyncio.to_thread(limiter.remaining, keys, tier) + if rem is not None and rem <= 0: + resp = JSONResponse( + {"tier": tier, "reason": "limit", "remainingSec": 0}, status_code=402 + ) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + url = f"{LOAD_BALANCER_URL.rstrip('/')}/session" + try: + async with httpx.AsyncClient(timeout=15.0) as http: + lb = await http.post(url, headers={"Content-Type": "application/json"}, content="{}") + except httpx.RequestError as exc: + logger.warning("Load balancer unreachable: %r", exc) + raise HTTPException(status_code=502, detail="Speech service unreachable.") + + # The queue is full: the LB replies 503 {state:"at_capacity"}. Relay it as-is + # so the client shows a soft "try again shortly", not a hard error. + if lb.status_code == 503: + body = _safe_json(lb) + if body.get("state") == "at_capacity": + resp = JSONResponse({"state": "at_capacity"}, status_code=503) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + if lb.status_code != 200: + # The LB's error body may name the reason (e.g. capacity); it carries no + # secret, so relay a trimmed copy. + logger.warning("Session handshake failed %s: %s", lb.status_code, lb.text[:300]) + raise HTTPException(status_code=502, detail=f"Session handshake failed ({lb.status_code}).") + + data = lb.json() + + # Busy pool: the LB queued us. Relay the ticket untouched — crucially with NO + # reservation, so waiting in line never costs the day's budget. + if data.get("state") == "queued": + data["tier"] = tier + resp = JSONResponse(data) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + # A slot was free: reserve the first chunk now and return the grant. + return await _finalize_grant(data, keys, tier, tracked, set_cookie) + + +@app.get("/api/queue/{queue_id}") +async def queue_status(queue_id: str, request: Request): + """Poll a waiting ticket: relay the position, or — when the head of the line + claims a freed slot — reserve the budget now and return the grant. Re-checks the + daily budget at claim, since a multi-minute wait could have spent it elsewhere.""" + if not LOAD_BALANCER_URL: + raise HTTPException(status_code=404, detail="Not found.") + + tier, keys, set_cookie = auth.resolve_identity(request) + tracked = LIMITER_ENABLED and limiter.budget_for(tier) is not None + + url = f"{LOAD_BALANCER_URL.rstrip('/')}/queue/{queue_id}" + try: + async with httpx.AsyncClient(timeout=15.0) as http: + lb = await http.get(url) + except httpx.RequestError as exc: + logger.warning("Load balancer unreachable: %r", exc) + raise HTTPException(status_code=502, detail="Speech service unreachable.") + + if lb.status_code == 404: + # Ticket unknown/expired (reaped after we stopped polling). Tell the client + # to start over rather than spin. + resp = JSONResponse({"state": "expired"}, status_code=404) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + if lb.status_code != 200: + logger.warning("Queue poll failed %s: %s", lb.status_code, lb.text[:300]) + raise HTTPException(status_code=502, detail=f"Queue poll failed ({lb.status_code}).") + + data = lb.json() + + if data.get("state") == "queued": + data["tier"] = tier + resp = JSONResponse(data) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + # Claimed a slot. Re-check the budget: it may have been spent in another tab + # during the wait. If so, refuse — the just-claimed slot is now a pending + # session on the LB and its pending-timeout reaper reclaims it shortly. + if tracked: + rem = await asyncio.to_thread(limiter.remaining, keys, tier) + if rem is not None and rem <= 0: + resp = JSONResponse( + {"tier": tier, "reason": "limit", "remainingSec": 0}, status_code=402 + ) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + return await _finalize_grant(data, keys, tier, tracked, set_cookie) + + +@app.delete("/api/queue/{queue_id}") +async def queue_leave(queue_id: str): + """Leave the queue from the explicit 'Leave queue' button (a real fetch).""" + if not LOAD_BALANCER_URL: + raise HTTPException(status_code=404, detail="Not found.") + await _lb_leave(queue_id) + return {"ok": True} + + +@app.post("/api/queue/end") +async def queue_end(request: Request): + """Leave the queue on teardown/tab-close (navigator.sendBeacon, which can only + POST). Body: { queueId }. Best-effort; the LB reaps the ticket on TTL anyway.""" + if not LOAD_BALANCER_URL: + raise HTTPException(status_code=404, detail="Not found.") + qid = await _queue_id(request) + if qid: + await _lb_leave(qid) + return {"ok": True} + + +async def _finalize_grant(data, keys, tier, tracked, set_cookie): + """Shared grant tail (fast path or queue claim): reserve the first chunk, attach + the metering fields the client needs, and set the anon cookie.""" + remaining = None + if tracked and data.get("session_id"): + await asyncio.to_thread(limiter.begin, data["session_id"], keys, tier) + remaining = await asyncio.to_thread(limiter.remaining, keys, tier) + + data.update({ + "tier": tier, + "limited": tracked, + "remainingSec": remaining, + "heartbeatSec": limiter.HEARTBEAT_SEC, + }) + resp = JSONResponse(data) + if set_cookie: + auth.set_anon_cookie(resp, set_cookie) + return resp + + +async def _lb_leave(queue_id: str) -> None: + """Best-effort: tell the LB to drop a waiting ticket.""" + url = f"{LOAD_BALANCER_URL.rstrip('/')}/queue/{queue_id}" + try: + async with httpx.AsyncClient(timeout=5.0) as http: + await http.delete(url) + except httpx.RequestError as exc: + logger.warning("Queue leave failed: %r", exc) + + +def _safe_json(response) -> dict: + try: + body = response.json() + except Exception: + return {} + return body if isinstance(body, dict) else {} + + +async def _queue_id(request: Request) -> str: + """Pull `queueId` from a JSON body, tolerating sendBeacon's blob posts.""" + try: + data = await request.json() + except Exception: + return "" + return (data or {}).get("queueId", "") if isinstance(data, dict) else "" + + +async def _session_id(request: Request) -> str: + """Pull `sessionId` from a JSON body, tolerating sendBeacon's blob posts.""" + try: + data = await request.json() + except Exception: + return "" + return (data or {}).get("sessionId", "") if isinstance(data, dict) else "" + + +@app.post("/api/session/heartbeat") +async def session_heartbeat(request: Request): + """Extend the live reservation one chunk at a time. `expired` once the day's + budget is spent — the client then tears down.""" + if not LIMITER_ENABLED: + raise HTTPException(status_code=404, detail="Not found.") + sid = await _session_id(request) + alive = bool(sid) and await asyncio.to_thread(limiter.heartbeat, sid) + return {"expired": not alive} + + +@app.post("/api/session/end") +async def session_end(request: Request): + """Clean teardown: reconcile to real elapsed time and refund the unused + chunk. Sent via navigator.sendBeacon, so it must succeed without a response.""" + if not LIMITER_ENABLED: + raise HTTPException(status_code=404, detail="Not found.") + sid = await _session_id(request) + if sid: + await asyncio.to_thread(limiter.end, sid) + return {"ok": True} + + +# Static front-end. Registered last so the /api routes win. `html=True` serves +# index.html at "/". The repo is public anyway, so serving the dir is fine. +app.mount("/", StaticFiles(directory=HERE, html=True), name="static") diff --git a/demo/style.css b/demo/style.css new file mode 100644 index 0000000..ae07a1b --- /dev/null +++ b/demo/style.css @@ -0,0 +1,2646 @@ +:root { + --bg: #0a0b10; + --bg-elev: #13151c; + --bg-elev-2: #1b1e29; + --border: rgba(255, 255, 255, 0.08); + --border-strong: rgba(255, 255, 255, 0.16); + --text: #f5f6fa; + --text-dim: rgba(245, 246, 250, 0.65); + --text-faint: rgba(245, 246, 250, 0.42); + + --accent: #8b7dff; + --accent-2: #22d3ee; + --listening: #22d3ee; + --speaking: #8b7dff; + --processing: #f59e0b; + --error: #ff6a75; + --success: #34d399; + + /* Mono is the "machine voice": reserved for system text the app emits — + * the orb's status caption, tool calls, the transport tag. Body/UI stays + * Inter. */ + --font-mono: "Geist Mono", ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace; + + /* Thesis: color belongs to the voice. The chrome is monochrome; saturated + * hue appears only on the orb and on these tiny role echoes, which mirror + * the orb's own state colors (you listening = cyan, assistant = violet, + * tool = amber) so the transcript reads in the same color language. */ + --voice-user: var(--accent-2); + --voice-assistant: var(--accent); + --voice-tool: var(--processing); + + /* Smoothed mic RMS in [0..1], updated every frame from JS while a session + * is active. Used by audio-reactive circle states. */ + --audio-level: 0; + /* Five log-spaced frequency bands extracted from the mic analyser; + * drive each bar's height independently for a real "spectrum" feel. */ + --bar0: 0; + --bar1: 0; + --bar2: 0; + --bar3: 0; + --bar4: 0; + + --radius-sm: 8px; + --radius-md: 14px; + --radius-lg: 22px; + + --shadow-soft: 0 10px 40px rgba(0, 0, 0, 0.35); + + color-scheme: dark; +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; + height: 100%; +} + +body { + font-family: "Inter", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + background: radial-gradient(ellipse at 50% 20%, #1a1c28 0%, var(--bg) 60%); + color: var(--text); + min-height: 100vh; + overflow-x: hidden; + -webkit-font-smoothing: antialiased; +} + +button { + font-family: inherit; +} + +a { + color: var(--text); + text-decoration: none; + border-bottom: 1px solid var(--border-strong); +} +a:hover { + border-bottom-color: var(--text); +} + +#app { + display: grid; + grid-template-rows: auto 1fr auto; + min-height: 100vh; +} + +.hidden { + display: none !important; +} + +/* ─── Topbar ──────────────────────────────────────────────────────────── */ + +/* Topbar is just a floating row of controls over the stage - no + * separator line, no background. Same story for the footer. Keeps the + * app feeling like one continuous canvas. */ +.topbar { + display: flex; + /* Top-align so the right control cluster sits up with the title instead of + * floating to the vertical middle of the tall identity block. */ + align-items: flex-start; + justify-content: space-between; + padding: 18px 28px; + /* Default stacking (below the conversation panel, z 200) so the panel drawer + * covers the controls when it's open. */ +} + +.brand { + display: flex; + align-items: center; + gap: 10px; + font-weight: 600; + font-size: 14px; + letter-spacing: 0.01em; + color: var(--text-dim); +} + +/* Transport tag rides the wordmark as a system label, not prose — mono, + * dimmed, the only mono in the topbar. */ +.brand-tag { + font-family: var(--font-mono); + font-weight: 500; + font-size: 11px; + letter-spacing: 0.02em; + opacity: 0.55; +} + +.brand .brand-logo { + width: 26px; + height: 26px; + object-fit: contain; + flex: none; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.45)); + transition: transform 0.2s ease; +} +.brand .brand-logo:hover { + transform: translateY(-1px) rotate(-2deg); +} + +/* Narrow viewports (typically the mobile shell iframe and phone-sized + browser windows): the full product name pushes the topbar's right + cluster off-screen. Keep the logo as the brand cue and drop the + wordmark - the orb already gives enough context. */ +@media (max-width: 600px) { + /* The full identity stack is too tall for a phone topbar. Show only the + * title + (i); the popup carries the blurb, credits and pipeline. + * `.brand` prefix raises specificity so this beats the later base + * `.ident-meta { display: flex }` rule regardless of source order. */ + .brand .ident-blurb, + .brand .ident-meta { + display: none; + } + .ident-title { + font-size: 18px; + } + /* (i) moves into the right-hand control cluster on phones, leaving the + * title alone on the left. */ + .brand .about-btn { + display: none; + } + .about-btn-mobile { + display: inline-flex; + } + /* Keep phone icons at the current compact size — the desktop bump above + * shouldn't leak into the mobile topbar. */ + .topbar-right .icon-btn { + width: 36px; + height: 36px; + } + .topbar-right .icon-btn svg { + width: 18px; + height: 18px; + } + /* Keep the account control compact on phones: avatar-only chip, shorter pill. */ + .account-chip { + height: 36px; + padding: 0 6px; + } + .account-handle { + display: none; + } + .signin-pill { + height: 36px; + padding: 0 12px; + } +} + +.topbar-right { + display: flex; + align-items: center; + gap: 10px; +} + +/* The HF user pill: compact avatar + handle. Rendered as a flex row so + * the avatar stays aligned with the text baseline even when the text + * wraps on narrow viewports. */ +.hf-user { + display: inline-flex; + align-items: center; + gap: 8px; + font-size: 13px; + color: var(--text-dim); + padding: 4px 10px 4px 4px; + border-radius: 999px; + background: var(--bg-elev); + border: 1px solid var(--border); + line-height: 1; +} + +.hf-avatar { + width: 24px; + height: 24px; + border-radius: 50%; + object-fit: cover; + background: color-mix(in srgb, var(--accent) 25%, var(--bg-elev-2)); + /* Hidden until an actual URL loads (see `setHfAvatar` in main.ts). + * Keeps the initial login flash from showing a broken image icon. */ + opacity: 0; + transition: opacity 0.25s ease; + flex: none; +} +.hf-avatar.loaded { + opacity: 1; +} + +.hf-user-name { + white-space: nowrap; +} + +/* ─── Transport pill ────────────────────────────────────────────── + * Surface the actual network path WebRTC picked for the robot peer + * connection (LAN, direct-through-NAT, or TURN-relayed). Lets the + * user spot at a glance when audio is going through the internet + * instead of staying on-prem. + */ +.transport-pill { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 12px; + font-weight: 500; + letter-spacing: 0.02em; + padding: 5px 10px; + border-radius: 999px; + background: var(--bg-elev); + border: 1px solid var(--border); + color: var(--text-dim); + user-select: none; + transition: background 0.2s ease, border-color 0.2s ease, color 0.2s ease; +} + +.transport-pill .transport-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: currentColor; + box-shadow: 0 0 0 3px color-mix(in srgb, currentColor 25%, transparent); + flex: none; +} + +/* Bitrate readout sits after the kind label. Dimmer + monospace so the + * kind (LAN / Direct / Relayed) stays the primary info and the numbers + * don't wiggle the layout as digits change. */ +.transport-pill .transport-bitrate { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + color: color-mix(in srgb, currentColor 75%, var(--text-dim)); + opacity: 0.85; + padding-left: 6px; + border-left: 1px solid color-mix(in srgb, currentColor 25%, transparent); +} +.transport-pill .transport-bitrate:empty { + display: none; +} + +.transport-pill.transport-checking { + color: var(--text-dim); +} +.transport-pill.transport-checking .transport-dot { + animation: transport-blink 1.2s ease-in-out infinite; +} + +.transport-pill.transport-lan { + color: #4ade80; + border-color: color-mix(in srgb, #4ade80 35%, var(--border)); + background: color-mix(in srgb, #4ade80 10%, var(--bg-elev)); +} + +.transport-pill.transport-direct { + color: #60a5fa; + border-color: color-mix(in srgb, #60a5fa 35%, var(--border)); + background: color-mix(in srgb, #60a5fa 10%, var(--bg-elev)); +} + +.transport-pill.transport-relay { + color: #fbbf24; + border-color: color-mix(in srgb, #fbbf24 35%, var(--border)); + background: color-mix(in srgb, #fbbf24 10%, var(--bg-elev)); +} + +@keyframes transport-blink { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.35; } +} + +.icon-btn { + background: var(--bg-elev); + border: 1px solid var(--border); + color: var(--text-dim); + width: 36px; + height: 36px; + border-radius: var(--radius-sm); + display: inline-flex; + align-items: center; + justify-content: center; + cursor: pointer; + touch-action: manipulation; + transition: background 0.15s, color 0.15s, border-color 0.15s; +} +.icon-btn:hover { + background: var(--bg-elev-2); + color: var(--text); + border-color: var(--border-strong); +} +/* Topbar control cluster reads a touch bigger on desktop. Phones keep the + * compact 36px (reset in the mobile media query). */ +.topbar-right .icon-btn { + width: 40px; + height: 40px; +} +.topbar-right .icon-btn svg { + width: 20px; + height: 20px; +} + +/* ─── Account (HF login chip + sign-in pill + popover) ─────────────────────── */ +.account { + position: relative; + display: inline-flex; + align-items: center; +} +/* Signed-out: a compact pill that reads as the primary affordance. */ +.signin-pill { + display: inline-flex; + align-items: center; + gap: 7px; + height: 40px; + padding: 0 14px; + border-radius: var(--radius-sm); + border: 1px solid var(--border-strong); + background: var(--bg-elev); + color: var(--text); + font-size: 13px; + font-weight: 600; + text-decoration: none; + transition: background 0.15s, border-color 0.15s; +} +.signin-pill svg { + width: 16px; + height: 16px; + flex: none; +} +.signin-pill:hover { + background: var(--bg-elev-2); + border-color: var(--text); +} +/* Narrow viewports: collapse the pill to an icon-only square. */ +@media (max-width: 800px) { + .signin-pill { + gap: 0; + width: 40px; + padding: 0; + justify-content: center; + } + .signin-pill span { + display: none; + } +} +/* Signed-in: avatar + handle chip. */ +.account-chip { + display: inline-flex; + align-items: center; + gap: 8px; + height: 40px; + padding: 0 10px 0 6px; + border-radius: var(--radius-sm); + border: 1px solid var(--border); + background: var(--bg-elev); + color: var(--text-dim); + cursor: pointer; + transition: background 0.15s, color 0.15s, border-color 0.15s; +} +.account-chip:hover { + background: var(--bg-elev-2); + color: var(--text); + border-color: var(--border-strong); +} +.account-avatar { + width: 26px; + height: 26px; + border-radius: 50%; + object-fit: cover; + flex: none; +} +.account-avatar-fallback { + display: inline-flex; + align-items: center; + justify-content: center; + background: var(--bg-elev-2); + color: var(--text); + font-size: 12px; + font-weight: 700; +} +.account-handle { + font-size: 13px; + font-weight: 600; + max-width: 12ch; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.account-pro { + font-size: 10px; + font-weight: 700; + letter-spacing: 0.04em; + padding: 2px 5px; + border-radius: 5px; + background: var(--text); + color: var(--bg); +} +/* Org "Team" badge: same pill as PRO, accent-coloured so it reads as + * unlimited without claiming a paid PRO subscription. */ +.account-team { + background: var(--accent); + color: var(--bg); +} +.account-pop { + position: absolute; + top: calc(100% + 8px); + right: 0; + min-width: 200px; + padding: 6px; + border-radius: var(--radius-md); + border: 1px solid var(--border-strong); + background: var(--bg-elev); + box-shadow: var(--shadow-soft); + z-index: 50; +} +.account-pop[hidden] { + display: none; +} +.account-pop-row { + padding: 8px 10px; +} +.account-pop-name { + font-weight: 600; + font-size: 13px; +} +.account-pop-meta { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + padding-top: 0; + font-size: 12px; + color: var(--text-dim); +} +.account-tier { + font-weight: 600; + color: var(--text); +} +.account-pop-link { + display: block; + padding: 8px 10px; + border-radius: var(--radius-sm); + color: var(--text-dim); + font-size: 13px; + text-decoration: none; + transition: background 0.15s, color 0.15s; +} +.account-pop-link:hover { + background: var(--bg-elev-2); + color: var(--text); +} +.account-signout { + border-top: 1px solid var(--border); + margin-top: 4px; + padding-top: 10px; + border-radius: 0 0 var(--radius-sm) var(--radius-sm); +} + +/* ─── Daily-limit modal ────────────────────────────────────────────────────── */ +.limit-modal { + width: min(400px, 92vw); +} +.limit-card { + position: relative; + align-items: center; + text-align: center; + gap: 14px; + padding: 34px 28px 26px; +} +.limit-close { + position: absolute; + top: 12px; + right: 12px; +} +/* HF smiling face (brand yellow) on a neutral badge — friendly, not a stop sign. + * The logo is the single pop of colour, so the badge itself stays quiet. */ +.limit-badge { + width: 66px; + height: 66px; + border-radius: 50%; + display: grid; + place-items: center; + margin: 2px auto 2px; + background: var(--bg-elev-2); + border: 1px solid var(--border-strong); + box-shadow: 0 0 0 6px rgba(255, 210, 30, 0.06); +} +.limit-badge .hf-logo { + width: 46px; + height: auto; +} +.limit-title { + margin: 0; + font-size: 20px; + font-weight: 700; + letter-spacing: 0; +} +.limit-msg { + color: var(--text-dim); + font-size: 14px; + line-height: 1.55; + margin: 0; + max-width: 30ch; +} +.limit-cta { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + margin-top: 4px; +} +.limit-cta .hf-logo { + width: 20px; + height: auto; + flex: none; +} +.limit-note { + margin: 0; + font-size: 12px; + color: var(--text-faint); +} +#limit-cta[hidden] { + display: none; +} + +/* ─── Stage ───────────────────────────────────────────────────────────── */ + +.stage { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 24px 24px 32px; + gap: 20px; + /* Let the stage shrink inside the grid's 1fr row so a height-capped orb can + * give back space rather than forcing the whole app to overflow. */ + min-height: 0; +} + +/* ─── Central circle ──────────────────────────────────────────────────── */ + +/* Wraps the orb and its two side controls so they stay aligned on one row. */ +.orb-wrap { + position: relative; + display: flex; + align-items: center; + justify-content: center; + gap: clamp(18px, 3vw, 32px); +} + +.circle { + position: relative; + /* Cap by height as well as width (min(vw, vh)-style): in a short container + * the orb shrinks instead of overflowing, so the page stays scroll-free even + * when embedded in a small box. On normal screens the 320px cap still wins, + * so the desktop size is unchanged. */ + width: clamp(140px, min(38vw, 46vh), 320px); + aspect-ratio: 1 / 1; + border-radius: 50%; + border: none; + background: transparent; + padding: 0; + cursor: pointer; + outline: none; + display: grid; + place-items: center; + color: var(--glow, var(--accent)); + transition: transform 0.18s ease, filter 0.18s ease; + -webkit-tap-highlight-color: transparent; + /* Treat taps as immediate clicks (no 300ms delay / double-tap-zoom). */ + touch-action: manipulation; +} +.circle:hover { + filter: brightness(1.08); +} +.circle:active { + transform: scale(0.97); + filter: brightness(0.92); +} +.circle:focus-visible .circle-core { + outline: 2px solid var(--glow, var(--accent)); + outline-offset: 6px; +} +.circle[disabled] { + cursor: default; + opacity: 0.75; +} + +.circle-glow { + position: absolute; + inset: 0; + border-radius: 50%; + background: radial-gradient(circle at center, var(--glow, var(--accent)) 0%, transparent 65%); + filter: blur(28px); + opacity: 0.5; + transform: scale(1); + transition: opacity 0.25s, background 0.25s, transform 1.4s ease-in-out; + pointer-events: none; +} + +/* Two nested circular rings: the inner tracks the core edge, the outer + * expands / fades to convey "audio radiating out" during speaking. */ +.circle-ring, +.circle-ring-outer { + position: absolute; + top: 50%; + left: 50%; + border-radius: 50%; + transform: translate(-50%, -50%); + pointer-events: none; + transition: opacity 0.4s ease, border-color 0.4s ease, transform 0.3s ease; +} +.circle-ring { + width: 82%; + height: 82%; + border: 1.5px solid color-mix(in srgb, var(--glow, var(--accent)) 35%, transparent); + opacity: 0.35; +} +.circle-ring-outer { + width: 94%; + height: 94%; + border: 1px solid color-mix(in srgb, var(--glow, var(--accent)) 22%, transparent); + opacity: 0; +} + +.circle-core { + position: relative; + width: 72%; + height: 72%; + border-radius: 50%; + background: radial-gradient( + circle at 35% 28%, + color-mix(in srgb, var(--glow, var(--accent)) 28%, transparent), + color-mix(in srgb, var(--glow, var(--accent)) 10%, transparent) 55%, + color-mix(in srgb, var(--glow, var(--accent)) 5%, transparent) + ); + border: 2px solid color-mix(in srgb, var(--glow, var(--accent)) 40%, transparent); + display: grid; + place-items: center; + box-shadow: + 0 0 32px color-mix(in srgb, var(--glow, var(--accent)) 22%, transparent), + inset 0 0 28px color-mix(in srgb, var(--glow, var(--accent)) 18%, transparent); + transition: background 0.4s ease, border-color 0.4s ease, box-shadow 0.4s ease, transform 0.4s ease; +} + +/* Indicator slot: a single SVG / spinner / bar group is visible at a time, + * driven by the state class on `.circle`. */ +.circle-indicator { + position: relative; + width: 44%; + height: 44%; + display: grid; + place-items: center; + color: var(--glow, var(--accent)); +} +.circle-indicator > .ind { + grid-area: 1 / 1; + opacity: 0; + transform: scale(0.85); + transition: opacity 0.25s ease, transform 0.25s ease; + pointer-events: none; +} +.circle-indicator > svg.ind { + width: 60%; + height: 60%; +} + +/* Spinner: a rotating ring gap, CSS-only. + * + * Note: the base `.circle-indicator > .ind` rule forces `transform: + * scale(.85)` / `scale(1)` on every indicator to drive the show/hide + * transition. If we only rotate here, the browser has to interpolate + * between `scale(1)` and `rotate(360deg)` (two different transform + * functions), which produces a broken, barely-moving animation. So we + * include the scale explicitly in the keyframes and bump specificity + * with `!important` so the spinner always wins over the state rule. */ +.ind-spinner { + width: 48%; + height: 48%; + border: 3px solid currentColor; + border-right-color: transparent; + border-radius: 50%; + opacity: 0; + animation: ind-spin 0.9s linear infinite; +} + +/* Thinking dots: 3 soft pulsing dots while the model is composing a + * response. Apple-style cadence: each dot scales up and brightens in + * turn, staggered by ~160 ms. Per-dot animation is on the child, so + * the parent's scale(.85 → 1) show/hide transform composes cleanly. */ +.ind-thinking { + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + width: 60%; + height: 60%; +} +.ind-thinking .dot { + width: 10px; + height: 10px; + border-radius: 50%; + background: currentColor; + opacity: 0.3; + animation: thinking-dot 1.25s ease-in-out infinite; +} +.ind-thinking .dot:nth-child(1) { animation-delay: 0s; } +.ind-thinking .dot:nth-child(2) { animation-delay: 0.16s; } +.ind-thinking .dot:nth-child(3) { animation-delay: 0.32s; } + +/* Bars: 5 vertical pills driven by --bar0..--bar4 CSS vars. */ +.ind-bars { + display: flex; + align-items: center; + gap: 5px; + height: 42%; +} +.ind-bars .bar { + width: 4px; + min-height: 4px; + border-radius: 3px; + background: currentColor; + opacity: 0.7; + --h: var(--bar0, 0); + height: calc(4px + var(--h) * 36px); + transition: height 0.08s ease-out, opacity 0.08s ease-out; +} +.ind-bars .bar:nth-child(1) { --h: var(--bar0); } +.ind-bars .bar:nth-child(2) { --h: var(--bar1); } +.ind-bars .bar:nth-child(3) { --h: var(--bar2); } +.ind-bars .bar:nth-child(4) { --h: var(--bar3); } +.ind-bars .bar:nth-child(5) { --h: var(--bar4); } +.ind-bars .bar { + opacity: calc(0.55 + 0.45 * var(--h)); +} + +/* Active indicator per state. + * + * Note: `ai-speaking` deliberately has NO indicator here. The orb + * itself becomes the indicator by pulsing on Reachy's voice (see the + * `--ai-audio-level` rules further down), which is a lot clearer and + * less confusing than reusing the mic-bars (which viewers would read + * as "you are speaking"). */ +.state-signed-out .ind-connect, +.state-authenticated .ind-mic, +.state-ready .ind-mic, +.state-connecting .ind-spinner, +.state-connected .ind-spinner, +.state-auto-selecting .ind-spinner, +.state-starting .ind-spinner, +.state-queued .ind-spinner, +.state-processing .ind-thinking, +.state-listening .ind-bars, +.state-user-speaking .ind-bars, +.state-ai-speaking .ind-voice, +.state-error .ind-error { + opacity: 1; + transform: scale(1); +} + +/* AI speaking indicator: speaker + two sound waves. + * + * Each wave pulses outward (opacity + stroke grow) with a quarter- + * beat offset so it reads as sound radiating out. Kept as a pure + * CSS animation so the icon is always visually alive even between + * syllables when --ai-audio-level momentarily dips. */ +.ind-voice .wave { + transform-origin: 50% 50%; + animation: voice-wave 1.35s ease-out infinite; +} +.ind-voice .wave-1 { animation-delay: 0s; } +.ind-voice .wave-2 { animation-delay: 0.35s; } + +/* Idle indicators: a chain-link "connect" glyph for the signed-out + * step (invites the user to authenticate with HF) and a microphone + * once a session is ready. Both picked up by the generic + * `.circle-indicator > svg.ind` sizing rule (60% × 60% of the slot) + * and rely on per-state opacity transitions for show / hide. */ +.ind-connect, +.ind-mic { + color: color-mix(in srgb, var(--glow, var(--accent)) 85%, white); +} + +/* ─── Caption below the circle ────────────────────────────────────────── */ + +/* The caption under the orb is meant to whisper, not shout: micro-label + * vibe, uppercase, letter-spaced, muted. Only appears for actionable / + * transitional states (see STATE_VIEWS). During a live conversation the + * orb alone carries the state so we collapse this row entirely. */ +.circle-caption { + margin: 0; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--text-faint); + min-height: 1.2em; + text-align: center; + opacity: 0.75; + transition: opacity 0.25s ease, color 0.25s ease, transform 0.25s ease, + min-height 0.25s ease; +} +.circle-caption.empty { + opacity: 0; + min-height: 0; + transform: translateY(-4px); + pointer-events: none; +} +.circle-caption.muted { + color: var(--text-faint); + opacity: 0.65; +} +.circle-caption.error { + color: var(--error); + opacity: 1; + letter-spacing: 0.08em; +} + +/* Warm, human line under the mono caption — sentence case, only while queued. + * The caption keeps the terse position; this reassures. */ +.circle-subcaption { + margin: 8px 0 0; + max-width: 32ch; + font-size: 13.5px; + line-height: 1.5; + color: var(--text-dim); + text-align: center; + text-wrap: balance; + transition: opacity 0.25s ease; +} +.circle-subcaption[hidden] { display: none; } + +/* Queue actions sit under the caption: "Join now" (primary, only when it's your + * turn) stacked above the quiet "Leave queue" escape hatch. */ +.queue-actions { + margin-top: 16px; + display: flex; + flex-direction: column; + align-items: center; + gap: 10px; +} +.queue-actions[hidden] { display: none; } + +/* "Join now": the one call to action in the queue flow, so it reads as inviting + * (filled accent) while everything around it stays quiet. */ +.join-queue-btn { + padding: 10px 26px; + font-family: var(--font-mono); + font-size: 12px; + font-weight: 600; + letter-spacing: 0.1em; + text-transform: uppercase; + color: #0b0b10; + background: var(--accent); + border: none; + border-radius: 999px; + cursor: pointer; + font-variant-numeric: tabular-nums; + transition: transform 0.12s ease, filter 0.2s ease; +} +.join-queue-btn:hover { filter: brightness(1.08); } +.join-queue-btn:active { transform: scale(0.97); } +.join-queue-btn:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 3px; +} +.join-queue-btn[hidden] { display: none; } + +/* "Leave queue": a quiet outlined pill, understated so it reads as an escape + * hatch, not a CTA. */ +.leave-queue-btn { + padding: 7px 16px; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--text-faint); + background: transparent; + border: 1px solid color-mix(in srgb, var(--text-faint) 35%, transparent); + border-radius: 999px; + cursor: pointer; + transition: color 0.2s ease, border-color 0.2s ease, background 0.2s ease; +} +.leave-queue-btn:hover { + color: var(--text); + border-color: color-mix(in srgb, var(--text-faint) 60%, transparent); + background: color-mix(in srgb, var(--text-faint) 8%, transparent); +} +.leave-queue-btn:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} +.leave-queue-btn[hidden] { display: none; } + +/* ─── Tool-call toaster ───────────────────────────────────────────────── */ + +/* A small, non-interactive pill that appears below the circle when the + * model invokes a tool (move_head, play_move). Sits just under the + * state caption, collapses to zero height when no toast is active. */ +.tool-toast { + display: inline-flex; + align-items: center; + gap: 8px; + margin-top: 10px; + padding: 6px 12px; + border-radius: 999px; + border: 1px solid var(--border-strong); + background: color-mix(in srgb, var(--bg-elev-2) 78%, transparent); + color: var(--text-dim); + font-size: 12px; + font-weight: 500; + letter-spacing: 0.01em; + line-height: 1; + white-space: nowrap; + max-width: 80vw; + overflow: hidden; + text-overflow: ellipsis; + + opacity: 0; + transform: translateY(-4px) scale(0.96); + pointer-events: none; + transition: opacity 0.22s ease, transform 0.22s ease; +} +.tool-toast.visible { + opacity: 1; + transform: translateY(0) scale(1); +} +.tool-toast-icon { + width: 14px; + height: 14px; + flex: none; + color: color-mix(in srgb, var(--voice-tool) 80%, white); + animation: tool-toast-spin 3.2s linear infinite; + animation-play-state: paused; +} +.tool-toast.visible .tool-toast-icon { + animation-play-state: running; +} +.tool-toast-text { + display: inline-block; + overflow: hidden; + text-overflow: ellipsis; +} +@keyframes tool-toast-spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +/* ─── Side controls (mic / stop) ──────────────────────────────────────── */ + +/* Mic button + its radial noise-gate arc. The wrapper centres the button; the + arc SVG is an absolute overlay larger than the button, revealed with the + live session (it has no layout footprint, so the idle collapse is unaffected). */ +.mic-gate { + position: relative; + flex: none; + display: inline-flex; + align-items: center; + justify-content: center; +} +.mic-gate-arc { + position: absolute; + left: 50%; + top: 50%; + width: 80px; + height: 80px; + transform: translate(-50%, -50%); + pointer-events: none; /* only the hit-path below catches drags */ + opacity: 0; + transition: opacity 0.25s ease; +} +.orb-wrap.live .mic-gate-arc { opacity: 1; } +.mga-track { + stroke: rgba(255, 255, 255, 0.1); /* hairline; recedes until needed */ + stroke-width: 1.5; + stroke-linecap: round; +} +.mga-fill { + stroke: var(--accent-2); + stroke-width: 2; + stroke-linecap: round; + opacity: 0.85; + transition: stroke-dashoffset 0.06s linear; +} +/* Threshold setpoint: a bead riding the ring. White with a thin dark outline so + it stays legible over the moving fill; it recolors to cyan with a soft glow + the moment the live level crosses it (the gate opening). */ +.mga-handle { + fill: var(--text); + stroke: none; + transition: fill 0.2s ease, filter 0.2s ease; +} +.mic-gate.gate-open .mga-handle { + fill: var(--accent-2); + filter: drop-shadow(0 0 3px var(--accent-2)); +} +.mga-hit { + stroke: transparent; + stroke-width: 16; + pointer-events: none; /* enabled only while live (below) */ + cursor: pointer; + touch-action: none; +} +/* Only catch drags during a live call; when idle the arc is hidden and must + not steal clicks near the collapsed mic button / orb. */ +.orb-wrap.live .mga-hit { pointer-events: stroke; } + +.side-btn { + flex: none; + width: 52px; + height: 52px; + border-radius: 50%; + border: 1px solid var(--border-strong); + /* Brighter background so the buttons actually stand out against the + * deep-blue stage gradient; previous var(--bg-elev) was too close to + * the page bg to be readable. */ + background: color-mix(in srgb, var(--bg-elev-2) 80%, #2a2e3c); + color: var(--text); + display: inline-flex; + align-items: center; + justify-content: center; + cursor: pointer; + touch-action: manipulation; + box-shadow: 0 6px 16px rgba(0, 0, 0, 0.35), + inset 0 1px 0 rgba(255, 255, 255, 0.06); + /* Hidden by default: take up no space until the session is live. The + * `width: 0` collapse keeps the orb centered on the idle screen. */ + opacity: 0; + transform: scale(0.55); + width: 0; + padding: 0; + pointer-events: none; + overflow: hidden; + transition: opacity 0.25s ease, transform 0.25s ease, width 0.25s ease, + background 0.15s, color 0.15s, border-color 0.15s; +} +.side-btn:hover { + background: color-mix(in srgb, var(--bg-elev-2) 60%, #323746); + border-color: var(--text-dim); +} +.side-btn svg { + width: 22px; + height: 22px; + flex: none; +} +.side-btn .mic-off { display: none; } +.side-btn.muted { + color: #fff; + border-color: var(--error); + background: color-mix(in srgb, var(--error) 70%, #1a0e13); +} +.side-btn.muted .mic-on { display: none; } +.side-btn.muted .mic-off { display: block; } + +/* Stop button: subtle warm tint so "end" reads as destructive. */ +#stop-btn:hover { + color: #fff; + border-color: color-mix(in srgb, var(--error) 60%, var(--border-strong)); + background: color-mix(in srgb, var(--error) 25%, var(--bg-elev-2)); +} + +/* Reveal when the session is live: buttons flank the orb on a flex row. */ +.orb-wrap.live .side-btn { + opacity: 1; + transform: scale(1); + width: 52px; + pointer-events: auto; +} + +/* Disable every transition / animation during the first paint so the + * orb doesn't fade-and-scale in when the page loads. `main.ts` removes + * the class after one animation frame. */ +body.booting, +body.booting *, +body.booting *::before, +body.booting *::after { + transition: none !important; + animation-duration: 0s !important; + animation-delay: 0s !important; +} + +/* ─── Circle animation keyframes ──────────────────────────────────────── */ + +/* Slow, subtle breathing for "warm idle" states. */ +@keyframes breathe { + 0%, 100% { transform: translate(-50%, -50%) scale(1); opacity: 0.4; } + 50% { transform: translate(-50%, -50%) scale(1.06); opacity: 0.15; } +} + +/* Outer ring expanding and fading - conveys "I am producing audio". */ +@keyframes ring-out { + 0% { transform: translate(-50%, -50%) scale(1); opacity: 0.35; } + 100% { transform: translate(-50%, -50%) scale(1.18); opacity: 0; } +} + +/* Soft inner scale for the core while talking. */ +@keyframes core-breathe { + 0%, 100% { transform: scale(1); } + 50% { transform: scale(1.04); } +} + +/* Subtle glow throb used for "thinking" — dimmer than speaking. */ +@keyframes thinking { + 0%, 100% { transform: scale(1); opacity: 0.7; } + 50% { transform: scale(0.96); opacity: 0.45; } +} + +/* Individual dot pulse for the 3-dot processing indicator. */ +@keyframes thinking-dot { + 0%, 60%, 100% { transform: scale(0.7); opacity: 0.3; } + 30% { transform: scale(1.15); opacity: 1; } +} + +/* Sound-wave pulse: arc fades in, scales up slightly, fades out. + * The transform-origin is the speaker's center (roughly x=12), so + * the waves feel like they're emanating from the cone. */ +@keyframes voice-wave { + 0% { opacity: 0; transform: scale(0.7); } + 30% { opacity: 1; transform: scale(1); } + 70% { opacity: 0.2; transform: scale(1.12); } + 100% { opacity: 0; transform: scale(0.7); } +} + +@keyframes ind-spin { + /* Scale kept at 1 so we don't fight with the indicator's base + * show/hide transform (see `.ind-spinner` for the rationale). */ + from { transform: scale(1) rotate(0deg); } + to { transform: scale(1) rotate(360deg); } +} + +/* ─── State-specific colors ──────────────────────────────────────────── */ + +.circle.state-signed-out { --glow: #8b7dff; } +.circle.state-authenticated, +.circle.state-ready { --glow: #34d399; } +.circle.state-connecting, +.circle.state-connected, +.circle.state-auto-selecting, +.circle.state-starting { --glow: #facc15; } +/* Queued: a calm slate glow, distinct from connecting's active yellow — this is + * waiting, not working. The spinner turns slowly and the ring breathes. */ +.circle.state-queued { --glow: #94a3b8; } +.circle.state-queued .ind-spinner { animation-duration: 2.4s; } +.circle.state-queued .circle-ring { animation: breathe 2.6s ease-in-out infinite; } +/* Your turn: a slot is held for you — the orb warms to the accent and breathes a + * little quicker, an invitation to join. */ +.circle.state-your-turn { --glow: var(--accent); } +.circle.state-your-turn .circle-ring { animation: breathe 1.6s ease-in-out infinite; } +.circle.state-listening, +.circle.state-user-speaking { --glow: var(--listening); } +.circle.state-processing { --glow: var(--processing); } +.circle.state-ai-speaking { --glow: var(--speaking); } +.circle.state-error { --glow: var(--error); } + +/* Idle / ready: gentle breathing of the inner ring. Kept out of the + * `signed-out` state so the very first paint on page load stays quiet + * (the orb now shows the Reachy head silhouette, no need to also pulse). */ +.circle.state-authenticated .circle-ring, +.circle.state-ready .circle-ring { + animation: breathe 2.4s ease-in-out infinite; +} + +/* Connecting flows: subtle glow throb so the orb feels thoughtful + * while the session is being negotiated. Kept off `processing` on + * purpose - the 3 thinking dots already pulse, and layering another + * throb on top competes with them for attention. */ +.circle.state-connecting .circle-core, +.circle.state-connected .circle-core, +.circle.state-auto-selecting .circle-core, +.circle.state-starting .circle-core { + animation: thinking 1.4s ease-in-out infinite; +} + +/* User is speaking: the mic RMS drives scale + opacity via --audio-level. */ +.circle.state-user-speaking .circle-ring { + animation: none; + opacity: calc(0.25 + 0.55 * var(--audio-level)); + transform: translate(-50%, -50%) scale(calc(1 + 0.08 * var(--audio-level))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-user-speaking .circle-ring-outer { + opacity: calc(0.1 + 0.35 * var(--audio-level)); + transform: translate(-50%, -50%) scale(calc(1 + 0.12 * var(--audio-level))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-listening .circle-ring { + animation: breathe 2s ease-in-out infinite; +} + +/* Assistant is speaking: the whole orb breathes in sync with Reachy's + * voice instead of running a fixed timer. `--ai-audio-level` (0-1) is + * updated at display rate by AiLevelMonitor from the OpenAI output + * track, so every syllable visibly moves the core + outer ring. This + * reads instantly as "the orb is the voice" and completely avoids the + * ambiguity of bars-vs-mic the user flagged. */ +.circle.state-ai-speaking .circle-core { + animation: none; + transform: scale(calc(1 + 0.09 * var(--ai-audio-level, 0))); + transition: transform 0.08s linear; +} +.circle.state-ai-speaking .circle-ring { + animation: none; + opacity: calc(0.3 + 0.5 * var(--ai-audio-level, 0)); + transform: translate(-50%, -50%) scale(calc(1 + 0.05 * var(--ai-audio-level, 0))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-ai-speaking .circle-ring-outer { + animation: none; + opacity: calc(0.15 + 0.55 * var(--ai-audio-level, 0)); + transform: translate(-50%, -50%) scale(calc(1 + 0.18 * var(--ai-audio-level, 0))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-ai-speaking .circle-glow { + opacity: calc(0.35 + 0.45 * var(--ai-audio-level, 0)); + transition: opacity 0.08s linear; +} + +/* ─── Robot picker ────────────────────────────────────────────────────── */ + +.robot-picker { + width: min(420px, 92vw); + background: var(--bg-elev); + border: 1px solid var(--border); + border-radius: var(--radius-md); + padding: 14px 16px; + box-shadow: var(--shadow-soft); +} + +.picker-title { + margin: 0 0 10px; + font-size: 11px; + font-weight: 600; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); +} + +.robot-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.robot-card { + display: flex; + justify-content: space-between; + align-items: center; + padding: 12px 14px; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + background: var(--bg-elev-2); + cursor: pointer; + transition: border-color 0.15s, background 0.15s; +} +.robot-card:hover { + border-color: var(--border-strong); +} +.robot-card.selected { + border-color: var(--border-strong); + background: var(--bg-elev); +} + +.robot-card .name { + font-weight: 600; + font-size: 14px; +} +.robot-card .id { + font-family: var(--font-mono); + font-size: 12px; + color: var(--text-faint); +} + +.robot-empty { + padding: 14px; + text-align: center; + color: var(--text-faint); + font-size: 13px; +} + +/* ─── Footer ──────────────────────────────────────────────────────────── */ + +.footer { + padding: 12px 28px 18px; + font-size: 11px; + letter-spacing: 0.02em; + color: var(--text-faint); + display: flex; + justify-content: center; + opacity: 0.55; + transition: opacity 0.25s ease; +} +.footer:hover { + opacity: 0.9; +} +.footer a { + color: inherit; + text-decoration: none; + border-bottom: 1px dotted currentColor; +} +.footer a:hover { + color: var(--text-dim); +} +/* While the webcam preview sits bottom-left, push the credit to the + * bottom-right so the two don't collide. */ +body.cam-on .footer { + justify-content: flex-end; +} + +/* ─── Modal ───────────────────────────────────────────────────────────── */ + +.modal { + border: 1px solid var(--border); + border-radius: var(--radius-md); + padding: 0; + background: var(--bg-elev); + color: var(--text); + width: min(440px, 92vw); + box-shadow: var(--shadow-soft); +} +.modal::backdrop { + background: rgba(8, 9, 13, 0.65); + backdrop-filter: blur(4px); +} + +.modal-content { + display: flex; + flex-direction: column; + gap: 16px; + padding: 22px 24px 20px; +} + +.modal-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 4px; +} +.modal-header h2 { + margin: 0; + font-size: 16px; + font-weight: 600; + letter-spacing: 0.02em; +} + +.field[hidden] { display: none; } +.field { + display: flex; + flex-direction: column; + gap: 6px; + font-size: 13px; + font-weight: 500; + color: var(--text-dim); +} +.field > span { + color: var(--text); + font-weight: 600; + font-size: 12px; + letter-spacing: 0.04em; + text-transform: uppercase; +} +.field input, +.field select, +.field textarea { + font-family: inherit; + font-size: 14px; + color: var(--text); + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius-sm); + padding: 10px 12px; + outline: none; + transition: border-color 0.15s; + resize: vertical; +} +.field textarea { + min-height: 84px; +} +.field input:focus, +.field select:focus, +.field textarea:focus { + border-color: var(--text-dim); +} +/* Deploy-pinned server URL: shown for transparency, not editable. */ +.field input[readonly] { + color: var(--text-dim); + border-style: dashed; + cursor: default; +} +.field input[readonly]:focus { + border-color: var(--border); +} +.field small { + color: var(--text-faint); + font-size: 12px; + line-height: 1.4; +} +.field small.error { color: var(--error); } +.field small code { + background: var(--bg); + padding: 1px 5px; + border-radius: 4px; + border: 1px solid var(--border); +} + +/* Noise gate: a header with a live value, a level meter, and a range slider + that shares the meter's horizontal (dB) axis. */ +.field-head { + display: flex; + align-items: baseline; + justify-content: space-between; +} +.field-value { + font-weight: 500; + font-size: 12px; + letter-spacing: 0; + text-transform: none; + color: var(--text-dim); +} +/* The slider and the level meter are one widget: the range input is overlaid + on the meter track (its native track is transparent), so the live-level fill + shows through behind the thumb and the thumb itself is the threshold. */ +.gate { + display: flex; + flex-direction: column; + gap: 4px; +} +.gate-track { + position: relative; + height: 14px; + display: flex; + align-items: center; +} +.gate-track::before { + /* the visible meter track */ + content: ""; + position: absolute; + left: 0; + right: 0; + height: 8px; + border-radius: 999px; + background: var(--bg); + border: 1px solid var(--border); +} +.gate-meter-fill { + position: absolute; + left: 1px; + top: 50%; + transform: translateY(-50%); + height: 6px; + width: 0; + border-radius: 999px; + background: var(--accent-2); + transition: width 0.06s linear; + pointer-events: none; +} +.gate-ends { + display: flex; + justify-content: space-between; + font-size: 11px; + color: var(--text-faint); + letter-spacing: 0; + text-transform: none; +} +/* The range input sits transparently on top of the meter track. */ +.gate-track input[type="range"] { + position: relative; + z-index: 1; + width: 100%; + margin: 0; + -webkit-appearance: none; + appearance: none; + padding: 0; + border: none; + background: transparent; + height: 14px; + cursor: pointer; +} +.gate-track input[type="range"]::-webkit-slider-runnable-track { + height: 14px; + background: transparent; +} +.gate-track input[type="range"]::-moz-range-track { + height: 14px; + background: transparent; +} +.gate-track input[type="range"]::-webkit-slider-thumb { + -webkit-appearance: none; + appearance: none; + width: 6px; + height: 18px; + border-radius: 3px; + background: var(--text); + border: 2px solid var(--bg-elev); + box-shadow: 0 0 0 1px var(--border-strong); +} +.gate-track input[type="range"]::-moz-range-thumb { + width: 6px; + height: 18px; + border-radius: 3px; + background: var(--text); + border: 2px solid var(--bg-elev); + box-shadow: 0 0 0 1px var(--border-strong); +} +.gate-track input[type="range"]:focus { border: none; } + +.modal-footer { + display: flex; + justify-content: space-between; + gap: 12px; + padding-top: 6px; +} + +.btn { + padding: 10px 16px; + border-radius: var(--radius-sm); + border: 1px solid var(--border-strong); + background: var(--bg-elev-2); + color: var(--text); + font-weight: 600; + font-size: 13px; + cursor: pointer; + transition: background 0.15s, border-color 0.15s, transform 0.05s; +} +.btn:hover { + border-color: var(--text); +} +.btn:active { + transform: translateY(1px); +} +.btn.primary { + background: var(--text); + border-color: var(--text); + color: var(--bg); +} +.btn.primary:hover { + background: #fff; + border-color: #fff; +} +.btn.ghost { + background: transparent; + border-color: var(--border); + color: var(--text-dim); +} +.btn.wide { + width: 100%; + padding: 12px 16px; +} +.btn[disabled] { + opacity: 0.45; + cursor: not-allowed; +} +.btn[disabled]:hover { + border-color: var(--border-strong); +} + +/* ─── Settings tabs ───────────────────────────────────────────────────── */ + +.tabs { + display: flex; + gap: 4px; + padding: 4px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius-sm); +} +.tab { + flex: 1; + padding: 8px 12px; + border: none; + background: transparent; + color: var(--text-dim); + font-family: inherit; + font-size: 13px; + font-weight: 600; + letter-spacing: 0.02em; + border-radius: calc(var(--radius-sm) - 3px); + cursor: pointer; + transition: background 0.15s, color 0.15s; +} +.tab:hover { + color: var(--text); +} +.tab.active { + background: var(--bg-elev-2); + color: var(--text); + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.3); +} + +.tab-panels { + display: flex; + flex-direction: column; +} +.tab-panel { + display: flex; + flex-direction: column; + gap: 16px; +} +.tab-panel[hidden] { + display: none; +} + +/* Horizontal layout for Voice + Model. */ +.field-row { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 12px; +} + +/* ─── Chat button + badge ────────────────────────────────────────────── */ + +#chat-btn { + position: relative; +} + +.chat-badge { + position: absolute; + top: 6px; + right: 6px; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--text); + border: 1.5px solid var(--bg-elev); + opacity: 0; + transform: scale(0); + transition: opacity 0.2s ease, transform 0.2s ease; + pointer-events: none; +} +.chat-badge.visible { + opacity: 1; + transform: scale(1); +} + +/* ─── Ephemeral bubble stack ─────────────────────────────────────────── */ + +.bubble-stack { + position: fixed; + /* Clear the topbar control row so the first bubble doesn't sit at the same + * height as the buttons. */ + top: 96px; + right: 28px; + width: min(300px, calc(100vw - 56px)); + display: flex; + flex-direction: column; + gap: 8px; + pointer-events: none; + z-index: 100; +} + +.bubble { + --bubble-dx: 10px; + pointer-events: auto; + padding: 10px 14px; + border-radius: var(--radius-md); + font-size: 13px; + line-height: 1.5; + border: 1px solid var(--border); + background: var(--bg-elev); + box-shadow: 0 4px 18px rgba(0, 0, 0, 0.32); + max-width: 100%; + word-break: break-word; + opacity: 0; + transform: translateX(var(--bubble-dx)); + transition: opacity 0.22s ease, transform 0.22s ease; +} +.bubble.in { + opacity: 1; + transform: translateX(0); +} +.bubble.out { + opacity: 0; + transform: translateX(var(--bubble-dx)); + pointer-events: none; + transition: opacity 0.3s ease, transform 0.3s ease; +} + +/* Surfaces stay neutral; the side they sit on plus the mono role label + * carry the distinction. No tinted fills — color is the orb's job. */ +.bubble.user { + --bubble-dx: -10px; + align-self: flex-start; +} +.bubble.assistant { + --bubble-dx: 10px; + align-self: flex-end; +} +.bubble.tool { + --bubble-dx: -10px; + align-self: flex-start; + display: flex; + align-items: center; + gap: 8px; + color: var(--text-dim); + font-family: var(--font-mono); + font-size: 12px; +} +.bubble.tool .bubble-tool-icon { + width: 13px; + height: 13px; + flex: none; + color: var(--voice-tool); +} + +.bubble-role { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + margin-bottom: 4px; + opacity: 0.7; +} +.bubble.user .bubble-role { color: var(--voice-user); } +.bubble.assistant .bubble-role { color: var(--voice-assistant); } + +/* ─── Conversation history panel ─────────────────────────────────────── */ + +.chat-panel { + position: fixed; + inset: 0; + z-index: 200; + pointer-events: none; +} +.chat-panel-backdrop { + position: absolute; + inset: 0; + background: rgba(8, 9, 13, 0.45); + backdrop-filter: blur(6px); + -webkit-backdrop-filter: blur(6px); + opacity: 0; + transition: opacity 0.25s ease; +} +.chat-panel.open .chat-panel-backdrop { + opacity: 1; + pointer-events: auto; +} +.chat-panel-inner { + position: absolute; + top: 0; + right: 0; + bottom: 0; + width: min(360px, 90vw); + background: var(--bg-elev); + border-left: 1px solid var(--border); + display: flex; + flex-direction: column; + transform: translateX(100%); + transition: transform 0.28s cubic-bezier(0.32, 0.72, 0, 1); + pointer-events: auto; + box-shadow: -8px 0 32px rgba(0, 0, 0, 0.35); +} +.chat-panel.open .chat-panel-inner { + transform: translateX(0); +} +.chat-panel-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 18px 20px; + border-bottom: 1px solid var(--border); + flex: none; +} +.chat-panel-header h3 { + margin: 0; + font-size: 14px; + font-weight: 600; + letter-spacing: 0.01em; +} +.chat-history { + flex: 1; + overflow-y: auto; + padding: 16px 20px; + display: flex; + flex-direction: column; + gap: 10px; + scroll-behavior: smooth; +} +.chat-history::-webkit-scrollbar { width: 3px; } +.chat-history::-webkit-scrollbar-track { background: transparent; } +.chat-history::-webkit-scrollbar-thumb { background: var(--border-strong); border-radius: 3px; } + +.chat-empty { + flex: 1; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 6px; + color: var(--text-faint); + padding: 48px 0; + opacity: 0.65; +} +.chat-empty svg { + margin-bottom: 6px; + opacity: 0.8; +} +.chat-empty-title { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.14em; + text-transform: uppercase; +} +.chat-empty-hint { + font-size: 12px; + color: var(--text-faint); + opacity: 0.7; +} + +/* ─── History messages ───────────────────────────────────────────────── */ + +.hist-msg { + display: flex; + flex-direction: column; + gap: 3px; + max-width: 88%; +} +.hist-msg.user { align-self: flex-start; } +.hist-msg.assistant { align-self: flex-end; } +.hist-msg.tool { align-self: flex-start; max-width: 100%; } + +.hist-role { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); + padding: 0 2px; +} +.hist-msg.user .hist-role { color: color-mix(in srgb, var(--voice-user) 75%, var(--text-faint)); } +.hist-msg.assistant .hist-role { color: color-mix(in srgb, var(--voice-assistant) 75%, var(--text-faint)); } +.hist-msg.tool .hist-role { color: color-mix(in srgb, var(--voice-tool) 75%, var(--text-faint)); } + +.hist-body { + padding: 9px 12px; + border-radius: var(--radius-md); + font-size: 13px; + line-height: 1.5; + border: 1px solid var(--border); + background: var(--bg-elev-2); + word-break: break-word; +} +/* Bodies share one neutral surface; alignment + the mono role label do the + * distinguishing, so the panel reads as one quiet column. */ +.hist-msg.user .hist-body.partial { opacity: 0.65; } + +/* ─── Tool call history item ─────────────────────────────────────────── */ + +.hist-tool-header { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + border-radius: var(--radius-md) var(--radius-md) 0 0; + background: color-mix(in srgb, var(--processing) 10%, var(--bg-elev-2)); + border: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); + border-bottom: 1px solid color-mix(in srgb, var(--processing) 15%, var(--border)); + cursor: pointer; + color: var(--text); + font-family: inherit; + font-size: 13px; + font-weight: 500; + width: 100%; + text-align: left; + transition: background 0.15s; +} +.hist-tool-header:only-child { + border-radius: var(--radius-md); + border-bottom: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); +} +.hist-tool-header:hover { + background: color-mix(in srgb, var(--processing) 16%, var(--bg-elev-2)); +} +.hist-tool-icon { + width: 13px; + height: 13px; + flex: none; + color: var(--processing); +} +.hist-tool-name { + font-family: var(--font-mono); + font-size: 12px; + color: var(--voice-tool); + font-weight: 600; +} +.hist-tool-chevron { + margin-left: auto; + width: 13px; + height: 13px; + color: var(--text-faint); + transition: transform 0.2s ease; + flex: none; +} +.hist-tool-header[aria-expanded="true"] .hist-tool-chevron { + transform: rotate(180deg); +} +.hist-tool-body { + padding: 10px 12px 12px; + border-radius: 0 0 var(--radius-md) var(--radius-md); + background: var(--bg); + border: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); + border-top: none; + display: none; +} +.hist-tool-body.open { display: block; } +.hist-tool-label { + font-family: var(--font-mono); + font-size: 10px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--text-faint); + margin: 8px 0 4px; +} +.hist-tool-label:first-child { margin-top: 0; } +.hist-tool-block { + font-family: var(--font-mono); + font-size: 11px; + line-height: 1.65; + color: var(--text-dim); + white-space: pre-wrap; + word-break: break-word; + overflow-x: auto; +} +.hist-tool-output { color: var(--text); } + +/* ─── Phone layout ──────────────────────────────────────────────────────── + * On phones the floating bubble stream overlaps the orb and there isn't room + * for it, so we drop it entirely and rely on the conversation panel (opened + * from the top-right chat button) as the single place to read the transcript. + * The badge still pulses there when new messages arrive while it's closed. + * We also stack the mic / stop controls above and below the orb (instead of + * left/right) so the wide live row never overflows, and let the panel take + * the full width. */ +@media (max-width: 600px) { + .bubble-stack { + display: none; + } + + .topbar { + padding: 14px 16px; + } + + .stage { + padding: 16px 12px 24px; + } + + /* Stack vertically: mic above the orb, stop below it (DOM order is + * mic → circle → stop). */ + .orb-wrap { + flex-direction: column; + gap: 14px; + } + + .circle { + /* Height-capped like the base rule so a narrow AND short box still fits. */ + width: clamp(130px, min(56vw, 52vh), 240px); + } + + /* In the column layout the side controls must collapse by HEIGHT, not + * width, so they take no vertical space until the session is live. */ + .side-btn { + width: 44px; + height: 0; + transition: opacity 0.25s ease, transform 0.25s ease, height 0.25s ease, + background 0.15s, color 0.15s, border-color 0.15s; + } + .side-btn svg { + width: 19px; + height: 19px; + } + .orb-wrap.live .side-btn { + width: 44px; + height: 44px; + } + .mic-gate-arc { width: 70px; height: 70px; } + + /* Full-screen conversation on phones — feels more deliberate than a + * narrow slide-over. It already spans top-to-bottom (inset 0); make it + * span edge-to-edge too and drop the now-pointless border/shadow. */ + .chat-panel-inner { + width: 100vw; + border-left: none; + box-shadow: none; + } +} + +/* ─── About panel ───────────────────────────────────────────────────────── + * Opened from the (i) by the wordmark. Reuses the .modal shell. + * Strictly monochrome per DESIGN.md — the only "color" is the orb, never + * here. Machine identifiers (model IDs, role tags, usernames) ride Geist + * Mono; everything human stays Inter. */ +.about-modal { + /* Roomier on tablet/desktop; the 92vw cap keeps phones full-width. */ + width: min(680px, 92vw); + max-height: 88vh; +} +.about-modal .modal-content { + max-height: 88vh; + overflow-y: auto; + gap: 20px; + padding: 26px 30px 24px; +} +.about-modal .modal-content::-webkit-scrollbar { width: 3px; } +.about-modal .modal-content::-webkit-scrollbar-track { background: transparent; } +.about-modal .modal-content::-webkit-scrollbar-thumb { background: var(--border-strong); border-radius: 3px; } + +/* Links here are quiet: no permanent underline (the global dotted rule is + * too loud for a dense credit block), brighten + underline on hover only. */ +.about-modal a { + color: var(--text); + border-bottom: none; + text-decoration: none; + transition: color 0.15s ease; +} +.about-modal a:hover { + color: #fff; +} +.about-intro a:hover { + text-decoration: underline; + text-underline-offset: 2px; +} +.about-modal .ext { + width: 12px; + height: 12px; + flex: none; + opacity: 0.5; +} + +/* Title row: the demo name and the (i) read as one unit. */ +.ident-head { + display: flex; + align-items: center; + gap: 9px; +} +/* (i) trigger sits right after the title. A faint outline keeps it + * catchable without turning into a card; it fills in on hover. */ +.about-btn { + flex: none; + width: 34px; + height: 34px; + border-radius: 50%; + background: transparent; + border: 1px solid var(--border-strong); + color: var(--text-dim); + display: inline-flex; + align-items: center; + justify-content: center; + cursor: pointer; + touch-action: manipulation; + transition: background 0.15s ease, color 0.15s ease, border-color 0.15s ease; +} +.about-btn:hover { + background: var(--bg-elev); + border-color: var(--text-dim); + color: var(--text); +} +.about-btn svg { + width: 20px; + height: 20px; +} + +/* The mobile twin of (i) lives in the right-hand control cluster and is + * hidden on desktop (the in-title one shows there instead). */ +.about-btn-mobile { + display: none; +} + +/* ── Popup intro: a plain paragraph on the project + a repo link ── */ +.about-intro p { + margin: 0; + font-size: 14px; + line-height: 1.6; + color: var(--text-dim); +} +.about-repo { + display: inline-flex; + align-items: center; + gap: 5px; + margin-top: 12px; + font-size: 14px; + font-weight: 500; +} + +/* ── Corner identity (replaces the wordmark) ── + * A compact stack in the topbar: title, one-line blurb, two meta rows. + * Monochrome; only the role/name identifiers ride the machine typeface. */ +.ident { + display: flex; + flex-direction: column; + gap: 10px; + min-width: 0; +} +/* Reset the global underlined-anchor styling inside the identity block. */ +.ident a { + border-bottom: none; + color: inherit; + transition: color 0.15s ease; +} +.ident a:hover { + color: var(--text); + text-decoration: underline; + text-underline-offset: 2px; +} +.ident-title { + font-size: 22px; + font-weight: 600; + letter-spacing: 0.005em; + line-height: 1.1; + color: var(--text); +} +.ident-blurb { + margin: 0; + max-width: 48ch; + font-size: 14.5px; + line-height: 1.5; + font-weight: 400; + color: var(--text-dim); +} +.ident-meta { + display: flex; + flex-direction: column; + gap: 7px; + font-size: 14px; + font-weight: 400; + color: var(--text-dim); +} +.ident-row { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 4px 8px; +} +.ident-label { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); +} + +/* Shared credit bits — now used in the corner identity block. */ +.sep { + color: var(--text-faint); + opacity: 0.6; +} +.hf-credit { + display: inline-flex; + align-items: center; + gap: 5px; +} +/* Brand marks keep their own color — a deliberate exception to the + * monochrome rule, for the HF and Cerebras logos only. */ +.hf-mark { + width: 14px; + height: 14px; + flex: none; + color: #ffd21e; +} +.cerebras-credit { + display: inline-flex; + align-items: center; + gap: 5px; +} +.cerebras-mark { + width: 14px; + height: 14px; + flex: none; +} +/* Usernames are identifiers, so they ride the machine typeface. */ +.handle { + font-family: var(--font-mono); + font-size: 13.5px; +} + +/* ── Pipeline (the signal flow) ── */ +.about-pipeline { + border-top: 1px solid var(--border); + padding-top: 16px; +} +.pipeline-title { + margin: 0 0 14px; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--text-faint); +} +.pipeline { + list-style: none; + margin: 0; + padding: 0; + position: relative; +} +/* One continuous rail threading every node, dot centers at x=8.5px. */ +.pipeline::before { + content: ""; + position: absolute; + left: 8px; + top: 7px; + bottom: 7px; + width: 1px; + background: var(--border-strong); +} +.pipeline > li { + position: relative; + padding-left: 30px; +} +/* Stages: solid node. Endpoints (you / orb): hollow node. */ +.pipe-stage::before { + content: ""; + position: absolute; + left: 5px; + top: 4px; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--text-dim); +} +.pipe-endpoint::before { + content: ""; + position: absolute; + left: 5px; + top: 3px; + width: 7px; + height: 7px; + border-radius: 50%; + border: 1px solid var(--text-faint); + background: var(--bg-elev); +} +.pipe-endpoint { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--text-faint); + padding-bottom: 12px; +} +.pipeline > li.pipe-endpoint:last-child { + padding-bottom: 0; +} +.pipe-stage { + padding-bottom: 16px; +} +.pipe-tag { + font-family: var(--font-mono); + font-size: 12px; + font-weight: 600; + letter-spacing: 0.08em; + color: var(--text); + margin-right: 9px; +} +.pipe-job { + font-size: 14px; + color: var(--text-dim); +} +/* Middot between the job and its model link, matching the separators + * used elsewhere. */ +.pipe-job::after { + content: "·"; + margin: 0 7px; + color: var(--text-faint); +} +.pipe-note { + color: var(--text-faint); +} +/* The Cerebras link stays as quiet as the note; brightens + underlines on hover. */ +.pipe-note a { + color: inherit; +} +.pipe-note a:hover { + color: var(--text); + text-decoration: underline; + text-underline-offset: 2px; +} +.pipe-model { + display: inline-flex; + align-items: center; + gap: 4px; + margin-top: 4px; + font-family: var(--font-mono); + font-size: 12px; + color: var(--text-dim); + word-break: break-all; +} +.pipe-model:hover { + color: #fff; + text-decoration: underline; + text-underline-offset: 2px; +} +.pipe-model .ext { + width: 11px; + height: 11px; +} + +/* "Interrupted" tag on an assistant reply the user barged in on. */ +.hist-note { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); + padding: 0 2px; +} +.hist-msg.assistant .hist-note { + align-self: flex-end; +} +.hist-msg.interrupted .hist-body { + opacity: 0.7; +} + +/* Captured webcam frame shown in the transcript (camera tool result). */ +.hist-image { + display: block; + width: 100%; + max-width: 240px; + border-radius: var(--radius-sm); + border: 1px solid var(--border); + margin-top: 2px; +} + +/* ─── Tools panel ────────────────────────────────────────────────────────── + * Reuses the modal/field shell. Switches are monochrome (checked = near-white, + * the same high-contrast treatment as the primary button): color belongs to + * the voice, not the chrome. */ +.tools-intro { + margin: 0; + font-size: 13px; + line-height: 1.5; + color: var(--text-dim); +} +.tool-list { + display: flex; + flex-direction: column; +} +.tool-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + padding: 12px 0; +} +.tool-row-sep { + border-top: 1px solid var(--border); + margin-top: 4px; +} +.tool-info { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; +} +.tool-name { + font-size: 14px; + font-weight: 600; + color: var(--text); +} +.tool-desc { + font-size: 12.5px; + color: var(--text-dim); +} +.tool-row.disabled .tool-name, +.tool-row.disabled .tool-desc { + opacity: 0.5; +} +.tool-hint { + display: block; + font-size: 12px; + line-height: 1.4; + color: var(--text-faint); +} +.tools-key { + margin: 0 0 4px; +} + +/* Toggle switch */ +.switch { + position: relative; + display: inline-flex; + flex: none; + width: 40px; + height: 24px; + cursor: pointer; +} +.switch input { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + margin: 0; + opacity: 0; + cursor: pointer; +} +.switch-track { + position: absolute; + inset: 0; + border-radius: 999px; + background: var(--bg); + border: 1px solid var(--border-strong); + transition: background 0.15s, border-color 0.15s; +} +.switch-track::after { + content: ""; + position: absolute; + top: 50%; + left: 3px; + transform: translateY(-50%); + width: 16px; + height: 16px; + border-radius: 50%; + background: var(--text-dim); + transition: transform 0.18s ease, background 0.15s; +} +.switch input:checked + .switch-track { + background: var(--text); + border-color: var(--text); +} +.switch input:checked + .switch-track::after { + transform: translate(16px, -50%); + background: var(--bg); +} +.switch input:focus-visible + .switch-track { + outline: 2px solid var(--text-dim); + outline-offset: 2px; +} +.switch input:disabled { + cursor: not-allowed; +} +.switch input:disabled + .switch-track { + opacity: 0.5; +} + +/* ─── Webcam preview (camera tool) ────────────────────────────────────────── + * Floating self-view, bottom-left. Mirrored for the user; the frame sent to + * the model is drawn un-mirrored (see captureSnapshot). */ +.cam-pip { + position: fixed; + left: 20px; + bottom: 20px; + width: 280px; + aspect-ratio: 4 / 3; + border-radius: var(--radius-md); + overflow: hidden; + border: 1px solid var(--border-strong); + background: var(--bg-elev); + box-shadow: var(--shadow-soft); + z-index: 90; + opacity: 0; + transform: translateY(8px) scale(0.96); + pointer-events: none; + transition: opacity 0.22s ease, transform 0.22s ease; +} +.cam-pip.visible { + opacity: 1; + transform: none; +} +.cam-video { + display: block; + width: 100%; + height: 100%; + object-fit: cover; + transform: scaleX(-1); /* mirror the self-view only */ + background: var(--bg); +} +.cam-label { + position: absolute; + left: 8px; + bottom: 6px; + font-family: var(--font-mono); + font-size: 9.5px; + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--text); + opacity: 0.8; + text-shadow: 0 1px 3px rgba(0, 0, 0, 0.7); +} +.cam-flash { + position: absolute; + inset: 0; + background: #fff; + opacity: 0; + pointer-events: none; +} +.cam-pip.flash .cam-flash { + animation: cam-flash 0.4s ease; +} +@keyframes cam-flash { + 0% { opacity: 0; } + 12% { opacity: 0.85; } + 100% { opacity: 0; } +} +@media (max-width: 600px) { + /* Bottom-centred on phones. Auto margins centre it without touching the + * transform, so the slide/scale-in animation still works. */ + .cam-pip { + left: 0; + right: 0; + margin-inline: auto; + bottom: 16px; + width: min(188px, 52vw); + } + /* The credit would sit under the centred preview, so drop it while the + * camera is on. */ + body.cam-on .footer { + display: none; + } +} +@media (prefers-reduced-motion: reduce) { + .cam-pip { + transition: opacity 0.22s ease; + transform: none; + } + .cam-pip.flash .cam-flash { + animation: none; + } +} + +/* ─── Desktop type scale ──────────────────────────────────────────────────── + * A uniform step up (~+1px, title +2) for all UI text on larger screens. + * Scoped to min-width: 601px so it can't reach phones — the ≤600px layout + * keeps every size exactly as it was. */ +@media (min-width: 601px) { + .cam-label { font-size: 10.5px; } + + .bubble-role, + .hist-role { font-size: 11px; } + + .circle-caption, + .footer, + .chat-empty-title, + .hist-tool-block, + .ident-label, + .pipeline-title, + .pipe-endpoint { font-size: 12px; } + + .field > span, + .field small, + .bubble.tool, + .chat-empty-hint, + .hist-tool-name, + .pipe-tag, + .pipe-model, + .tool-hint { font-size: 13px; } + + .tool-desc { font-size: 13.5px; } + + .btn, + .bubble, + .field, + .hist-body, + .hist-tool-header, + .tools-intro { font-size: 14px; } + + .handle { font-size: 14.5px; } + + .about-intro p, + .about-repo, + .brand, + .chat-panel-header h3, + .field textarea, + .ident-meta, + .pipe-job, + .tool-name { font-size: 15px; } + + .ident-blurb { font-size: 15.5px; } + + .modal-header h2 { font-size: 17px; } + + .ident-title { font-size: 24px; } +} + +/* ─── Short viewports: fit without a vertical scrollbar ───────────────────── + * The base media queries only react to WIDTH, so a short container (e.g. the + * app embedded in a small box, or a laptop window once the browser chrome eats + * into the viewport) used to overflow vertically and get clipped. We KEEP the + * identity header, orb, caption and credits — we just undo the desktop type + * bump, tighten the vertical rhythm and let the height-capped orb shrink, so + * everything fits with comfortable margin. No opt-in needed. + * + * The single biggest space eater is the identity block (~180px at the desktop + * type scale), so this tier compacts it first. */ +@media (max-height: 780px) { + .topbar { padding: 14px 24px; } + .ident { gap: 5px; } + .ident-title { font-size: 20px; } + .ident-blurb { font-size: 13.5px; line-height: 1.4; } + .ident-meta { gap: 5px; font-size: 13px; } + .stage { padding: 12px 20px 16px; gap: 12px; } + .footer { padding: 8px 24px 10px; } + .circle { width: clamp(140px, min(38vw, 44vh), 300px); } +} + +@media (max-height: 560px) { + .topbar { padding: 8px 16px; } + .ident-title { font-size: 17px; } + /* Below this, room is tight enough that the one-line blurb and the footer + * (a verbatim duplicate of the header's "Powered by" credit) are dropped so + * the title + credits still read; the rest of the header stays. */ + .ident-blurb { display: none; } + .stage { padding: 8px 14px 10px; gap: 8px; } + .footer { display: none; } + .circle { width: clamp(120px, min(50vw, 46vh), 240px); } +} diff --git a/demo/ui/account.js b/demo/ui/account.js new file mode 100644 index 0000000..dc2d5b2 --- /dev/null +++ b/demo/ui/account.js @@ -0,0 +1,189 @@ +// @ts-check +/** + * Account — the HF login chip and the daily-limit modal. + * + * Reads `/api/me` to learn the current tier (anonymous / signed-in / PRO) and + * remaining daily talk-time, renders a sign-in pill or a signed-in chip with a + * small popover (tier, remaining, sign out, upgrade), and shows the limit modal + * when a conversation is refused or cut. The time metering itself lives in + * main.js (heartbeat loop) + the server; this module is just the surface. + * + * Inert unless the deploy is in LB mode (`/api/me` → `{enabled:true}`). + */ + +import { $, escHtml } from "./dom.js"; + +const PRO_URL = "https://huggingface.co/subscribe/pro"; + +// Official multi-color Hugging Face logo, used in the badge + sign-in CTA. +const HF_MARK = ``; + +/** @param {number} sec @returns {string} "m:ss" */ +function fmt(sec) { + const s = Math.max(0, Math.round(sec)); + return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`; +} + +export class Account { + constructor() { + /** @type {HTMLElement} */ + this._root = $("#account"); + /** @type {HTMLDialogElement} */ + this._modal = $("#limit-modal"); + this._modalTitle = $("#limit-title"); + this._modalMsg = $("#limit-msg"); + this._modalNote = $("#limit-note"); + /** @type {HTMLAnchorElement} */ + this._modalCta = /** @type {any} */ ($("#limit-cta")); + + /** @type {{enabled:boolean, auth?:boolean, loggedIn?:boolean, username?:string, avatar?:string, tier?:string, remainingSec?:number|null, limitSec?:number|null, loginUrl?:string|null, logoutUrl?:string|null}} */ + this._me = { enabled: false }; + this._popoverOpen = false; + + $("#limit-close").addEventListener("click", () => this._modal.close()); + this._modal.addEventListener("click", (e) => { + if (e.target === this._modal) this._modal.close(); + }); + // Close the popover on an outside click. + document.addEventListener("click", (e) => { + if (this._popoverOpen && !this._root.contains(/** @type {Node} */ (e.target))) { + this._closePopover(); + } + }); + } + + get tier() { + return this._me.tier || "anon"; + } + + /** Fetch `/api/me` and (re)render the chip. Safe to call repeatedly (load, + * after the OAuth redirect, after a conversation ends). */ + async refresh() { + try { + const res = await fetch("api/me"); + this._me = res.ok ? await res.json() : { enabled: false }; + } catch { + this._me = { enabled: false }; + } + this._render(); + } + + _render() { + const me = this._me; + if (!me.enabled) { + this._root.hidden = true; + this._root.innerHTML = ""; + return; + } + this._root.hidden = false; + + if (!me.loggedIn) { + // Signed-out: a sign-in pill (only when OAuth is actually available). + if (me.auth && me.loginUrl) { + this._root.innerHTML = ``; + } else { + this._root.innerHTML = ""; + this._root.hidden = true; + } + return; + } + + // Signed-in: avatar + handle chip that toggles a popover. + const isPro = me.tier === "pro"; + // Org members get unlimited usage too, but aren't PRO — don't brand them so. + const isUnlimited = isPro || me.tier === "org"; + const avatar = me.avatar + ? `` + : ``; + const remaining = + isUnlimited || me.remainingSec == null + ? "Unlimited" + : `${fmt(me.remainingSec)} left today`; + const tierLabel = isPro ? "PRO" : isUnlimited ? "Team" : "Free"; + + this._root.innerHTML = ` + + `; + + const chip = $("#account-chip"); + chip.addEventListener("click", (e) => { + e.stopPropagation(); + this._popoverOpen ? this._closePopover() : this._openPopover(); + }); + } + + _openPopover() { + const pop = document.getElementById("account-pop"); + const chip = document.getElementById("account-chip"); + if (!pop || !chip) return; + pop.hidden = false; + chip.setAttribute("aria-expanded", "true"); + this._popoverOpen = true; + } + + _closePopover() { + const pop = document.getElementById("account-pop"); + const chip = document.getElementById("account-chip"); + if (pop) pop.hidden = true; + if (chip) chip.setAttribute("aria-expanded", "false"); + this._popoverOpen = false; + } + + /** + * Show the limit modal for a tier — used both when a conversation is refused + * at start (402) and when a live one is cut (heartbeat `expired`). + * @param {string} [tier] + */ + showLimit(tier = this.tier) { + const canSignIn = this._me.auth && this._me.loginUrl; + this._modalTitle.textContent = "Thanks for chatting!"; + if (tier === "anon") { + this._modalMsg.textContent = + "Guest conversations run for 5 minutes. Sign in with Hugging Face to get 10 minutes a day for free, and PRO members chat with no limit at all."; + this._modalNote.textContent = "Your free minutes refresh tomorrow."; + if (canSignIn) { + this._modalCta.innerHTML = `${HF_MARK}Sign in with Hugging Face`; + this._modalCta.href = /** @type {string} */ (this._me.loginUrl); + this._modalCta.hidden = false; + } else { + this._modalCta.hidden = true; + } + } else { + // Signed-in, non-PRO. + this._modalMsg.textContent = + "You've enjoyed your 10 minutes for today. Go PRO for unlimited conversations and to support open source AI."; + this._modalNote.textContent = "Or come back tomorrow. Your minutes reset daily."; + this._modalCta.innerHTML = "Upgrade to PRO"; + this._modalCta.href = PRO_URL; + this._modalCta.hidden = false; + } + if (!this._modal.open) this._modal.showModal(); + } + + /** Show a warm "we're at capacity" message when even the waiting line is full. + * Reuses the limit modal shell; no call-to-action, just reassurance. */ + showBusy() { + this._modalTitle.textContent = "Hugged to the limit 🤗"; + this._modalMsg.textContent = + "Every slot and the whole line are full right now. Too much love! Grab a coffee and pop back in a minute."; + this._modalNote.textContent = "A spot usually opens up within a few minutes."; + this._modalCta.hidden = true; + if (!this._modal.open) this._modal.showModal(); + } +} diff --git a/demo/ui/chat.js b/demo/ui/chat.js new file mode 100644 index 0000000..84292ab --- /dev/null +++ b/demo/ui/chat.js @@ -0,0 +1,425 @@ +// @ts-check +/** + * ChatView — owns the whole conversation surface: the slide-in history panel, + * the ephemeral on-orb bubbles, and all the transcript/tool/streaming + * bookkeeping. main.js wires the realtime client's events straight to the + * `on*` methods here and otherwise doesn't touch chat state. + * + * Two parallel surfaces share one shape (see `_buildMessageEl`): + * - ephemeral bubbles (`.bubble` / `.bubble-*`) fade on a timer + * - persistent history (`.hist-msg` / `.hist-*`) the durable panel log + * + * Keying: + * - user transcripts by the server's `item_id` — a speculative continuation + * REUSES it, so both segments land in one row/bubble; deltas are CUMULATIVE + * (each carries the full sentence so far), so we replace text wholesale. + * - assistant transcripts by `response_id`, so a cancelled speculative reply + * can be marked interrupted without erasing what was already shown. + */ + +import { $, escHtml, DEBUG } from "./dom.js"; + +const WRENCH_PATH = ``; +const CHAT_BUBBLE_SVG = ``; +const EMPTY_STATE_HTML = `
${CHAT_BUBBLE_SVG}No messages yetTap the orb and start talking
`; + +export class ChatView { + constructor() { + /** @type {HTMLButtonElement} */ + this._chatBtn = $("#chat-btn"); + /** @type {HTMLSpanElement} */ + this._chatBadge = $("#chat-badge"); + /** @type {HTMLDivElement} */ + this._chatPanel = $("#chat-panel"); + /** @type {HTMLDivElement} */ + this._chatPanelBackdrop = $("#chat-panel-backdrop"); + /** @type {HTMLButtonElement} */ + this._chatPanelClose = $("#chat-panel-close"); + /** @type {HTMLDivElement} */ + this._chatHistory = $("#chat-history"); + /** @type {HTMLDivElement} */ + this._bubbleStack = $("#bubble-stack"); + + this._panelOpen = false; + this._scrollQueued = false; + + // ── User transcript state (keyed by item_id) ─────────────────────────── + /** @type {Map} */ + this._userHistByItem = new Map(); + /** @type {HTMLElement | null} */ + this._activeUserBubble = null; + this._activeUserItemId = ""; + // Monotonic counter for synthesizing unique keys when the server omits an + // item_id / response_id, so id-less messages never collapse onto each other. + this._anonSeq = 0; + + // ── Assistant transcript state (keyed by response_id) ────────────────── + /** @type {Map} */ + this._asstByResp = new Map(); + + // ── Ephemeral bubble auto-dismiss ────────────────────────────────────── + // Per-element expiry (epoch ms). A bubble fades once its expiry passes — + // but only in stack order (see _reapBubbles). Refreshing the expiry keeps a + // bubble alive while it updates (e.g. the live user utterance). + /** @type {WeakMap} */ + this._bubbleExpiry = new WeakMap(); + // Single pending reaper handle: one timer for the whole stack (not one per + // bubble) so dismissal is strictly oldest-first regardless of per-bubble delays. + this._reaperHandle = 0; + + this._chatBtn.addEventListener("click", () => (this._panelOpen ? this._closePanel() : this._openPanel())); + this._chatPanelClose.addEventListener("click", () => this._closePanel()); + this._chatPanelBackdrop.addEventListener("click", () => this._closePanel()); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && this._panelOpen) this._closePanel(); + }); + } + + // ── Panel ─────────────────────────────────────────────────────────────── + + _openPanel() { + this._panelOpen = true; + this._chatPanel.classList.add("open"); + this._chatBadge.classList.remove("visible"); + this._scrollToBottom(); + } + + _closePanel() { + this._panelOpen = false; + this._chatPanel.classList.remove("open"); + } + + // Coalesce scroll-to-bottom: a burst of cumulative transcript deltas would + // otherwise queue one rAF per delta, all writing the same scrollTop. + _scrollToBottom() { + if (!this._panelOpen || this._scrollQueued) return; + this._scrollQueued = true; + requestAnimationFrame(() => { + this._scrollQueued = false; + this._chatHistory.scrollTop = this._chatHistory.scrollHeight; + }); + } + + _markUnread() { + if (this._panelOpen) { + this._scrollToBottom(); + return; + } + this._chatBadge.classList.add("visible"); + } + + // ── Shared rendering ────────────────────────────────────────────────────── + + /** + * Build a role-labelled message element. Ephemeral bubbles and persistent + * history rows share the same shape and differ only in their class prefix + * (`bubble`/`bubble-*` vs `hist-msg`/`hist-*`). + * @param {{ container: string, prefix: string, role: "user"|"assistant", text: string, partial?: boolean }} o + * @returns {HTMLElement} + */ + _buildMessageEl({ container, prefix, role, text, partial = false }) { + const el = document.createElement("div"); + el.className = `${container} ${role}`; + const label = role === "user" ? "You" : "Assistant"; + el.innerHTML = `
${label}
${escHtml(text)}
`; + return el; + } + + // ── Ephemeral bubbles ─────────────────────────────────────────────────── + + /** @param {"user"|"assistant"|"tool"} role @param {string} text @returns {HTMLElement} */ + _spawnBubble(role, text) { + let el; + if (role === "tool") { + el = document.createElement("div"); + el.className = "bubble tool"; + el.innerHTML = `${WRENCH_PATH}${escHtml(text)}`; + } else { + el = this._buildMessageEl({ container: "bubble", prefix: "bubble", role, text }); + } + this._bubbleStack.appendChild(el); + // Cap the stack at 3, but never evict the bubble the caller is still + // actively updating (the live user bubble) — drop the next-oldest instead. + const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]); + if (visible.length > 3) { + this._dismissBubble(visible.find((b) => b !== this._activeUserBubble) ?? visible[0]); + } + requestAnimationFrame(() => el.classList.add("in")); + return el; + } + + /** @param {HTMLElement} el @param {string} text */ + _updateBubbleText(el, text) { + const t = el.querySelector(".bubble-body"); + if (t) t.textContent = text; + } + + /** @param {HTMLElement} el */ + _dismissBubble(el) { + if (!el || el.classList.contains("out")) return; // idempotent + this._bubbleExpiry.delete(el); + el.classList.remove("in"); + el.classList.add("out"); + const remove = () => el.remove(); + el.addEventListener("transitionend", remove, { once: true }); + // Fallback: transitionend never fires if the bubble's visual state didn't + // change (dismissed pre-paint) or the tab is backgrounded. The transition + // is 0.3s, so force removal a little after. + setTimeout(remove, 400); + } + + /** + * Fade bubbles whose expiry has passed — strictly oldest-first. We walk the + * stack top (oldest) to bottom (newest) and stop at the first bubble still + * alive: nothing newer may leave while an older bubble is still on screen. A + * bubble that keeps updating pushes its own expiry forward, so it (and + * everything behind it) stays put until it finally goes quiet. + */ + _reapBubbles() { + this._reaperHandle = 0; + const now = Date.now(); + const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]); + let nextWake = Infinity; + for (const el of visible) { + const exp = this._bubbleExpiry.get(el) ?? now; // no expiry recorded → treat as due + if (exp <= now) { + this._dismissBubble(el); + } else { + // Oldest survivor isn't due yet; stop so nothing newer leaves before it. + nextWake = exp; + break; + } + } + if (nextWake !== Infinity) { + this._reaperHandle = setTimeout(() => this._reapBubbles(), Math.max(50, nextWake - Date.now())); + } + } + + /** + * (Re)arm a bubble's auto-dismiss by pushing its expiry out by `delay`. + * Calling it again resets the countdown — so a bubble that keeps updating + * stays on screen and only fades once it goes quiet. Removal is ordered by the + * shared reaper, so the oldest bubble always disappears first. + * @param {HTMLElement} el @param {number} [delay] + */ + _bumpDismiss(el, delay = 4000) { + this._bubbleExpiry.set(el, Date.now() + delay); + if (!this._reaperHandle) this._reaperHandle = setTimeout(() => this._reapBubbles(), delay); + } + + // ── History ─────────────────────────────────────────────────────────────── + + /** Render the empty-state placeholder into the history panel. */ + renderEmptyState() { + this._chatHistory.innerHTML = EMPTY_STATE_HTML; + } + + /** Reset the panel to the empty state and clear the unread badge. */ + clear() { + this.renderEmptyState(); + this._chatBadge.classList.remove("visible"); + } + + /** @param {"user"|"assistant"} role @param {string} text @param {boolean} partial @returns {HTMLElement} */ + _appendHistMsg(role, text, partial) { + const empty = this._chatHistory.querySelector(".chat-empty"); + if (empty) empty.remove(); + const el = this._buildMessageEl({ container: "hist-msg", prefix: "hist", role, text, partial }); + this._chatHistory.appendChild(el); + this._scrollToBottom(); + return el; + } + + /** @param {HTMLElement | null} el @param {string} text @param {boolean} partial */ + _updateHistMsg(el, text, partial) { + if (!el) return; + const body = /** @type {HTMLElement | null} */ (el.querySelector(".hist-body")); + if (!body) return; + body.textContent = text; + body.classList.toggle("partial", partial); + this._scrollToBottom(); + } + + /** + * Append a tool-call row to the conversation. We only add it once the tool + * has run, so the expandable toggle carries BOTH the call input and its result. + * @param {string} name @param {string} argsJson @param {string} output + */ + _appendHistTool(name, argsJson, output) { + const empty = this._chatHistory.querySelector(".chat-empty"); + if (empty) empty.remove(); + let pretty = argsJson; + try { pretty = JSON.stringify(JSON.parse(argsJson), null, 2); } catch {} + const el = document.createElement("div"); + el.className = "hist-msg tool"; + el.innerHTML = ` +
Tool call
+ +
+
Input
+
${escHtml(pretty)}
+
Output
+
${escHtml(output || "(no output)")}
+
+ `; + const header = /** @type {HTMLButtonElement} */ (el.querySelector(".hist-tool-header")); + const body = /** @type {HTMLDivElement} */ (el.querySelector(".hist-tool-body")); + header.addEventListener("click", () => { + const expanded = header.getAttribute("aria-expanded") === "true"; + header.setAttribute("aria-expanded", String(!expanded)); + body.classList.toggle("open", !expanded); + }); + this._chatHistory.appendChild(el); + this._scrollToBottom(); + } + + /** Tag an assistant history row as interrupted (user barged in mid-reply). + * @param {HTMLElement | null} hist */ + _markHistInterrupted(hist) { + if (!hist || hist.querySelector(".hist-note")) return; + hist.classList.add("interrupted"); + const note = document.createElement("div"); + note.className = "hist-note"; + note.textContent = "Interrupted"; + hist.appendChild(note); + } + + /** Render a captured webcam frame in the transcript (the camera tool result). + * @param {string} dataUrl */ + _appendHistImage(dataUrl) { + const empty = this._chatHistory.querySelector(".chat-empty"); + if (empty) empty.remove(); + const el = document.createElement("div"); + el.className = "hist-msg tool"; + el.innerHTML = `
Snapshot
Webcam snapshot sent to the model`; + const img = /** @type {HTMLImageElement} */ (el.querySelector("img")); + img.src = dataUrl; + this._chatHistory.appendChild(el); + this._scrollToBottom(); + } + + /** + * Reset all streaming bookkeeping for session start / teardown. Pass + * `dismiss` to also fade any bubbles still on screen. + * @param {{ dismiss?: boolean }} [opts] + */ + reset(opts) { + if (opts?.dismiss) { + if (this._activeUserBubble) this._dismissBubble(this._activeUserBubble); + for (const { bubble } of this._asstByResp.values()) this._dismissBubble(bubble); + } + this._userHistByItem.clear(); + this._activeUserBubble = null; + this._activeUserItemId = ""; + this._asstByResp.clear(); + } + + // ── Client event handlers ───────────────────────────────────────────────── + + /** + * A streamed transcript delta (user or assistant). + * @param {{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }} d + */ + onTranscript(d) { + if (DEBUG) console.debug(`[ui] transcript role=${d.role} partial=${d.partial} item=${d.itemId} resp=${d.responseId} text=${JSON.stringify(d.text)}`); + + if (d.role === "user") { + // Group by item_id: a speculative continuation reuses the same id, so it + // updates the same row/bubble. A missing id falls back to the active item + // (same utterance) or a fresh unique key, never a shared sentinel that + // would collapse distinct turns into one row. + const id = d.itemId || this._activeUserItemId || `_u${++this._anonSeq}`; + const text = d.text; + + let hist = this._userHistByItem.get(id); + if (!hist) { + hist = this._appendHistMsg("user", text, d.partial); + this._userHistByItem.set(id, hist); + } else { + this._updateHistMsg(hist, text, d.partial); + } + + // One ephemeral bubble per active item. Purely timer-based: the timer is + // refreshed on every delta, so it stays while the user keeps talking and + // fades a few seconds after they stop — no dependency on a response ever + // arriving, so it can never get stuck. + if (this._activeUserItemId !== id || !this._activeUserBubble) { + this._activeUserBubble = this._spawnBubble("user", text); + this._activeUserItemId = id; + } else { + this._updateBubbleText(this._activeUserBubble, text); + } + this._bumpDismiss(this._activeUserBubble, 6000); + this._markUnread(); + } else if (d.role === "assistant") { + // Assistant transcript arrives once, as the full text, keyed by + // response_id so a cancelled speculative response can be removed later. A + // missing id gets a unique key so two id-less replies never collide. + const rid = d.responseId || `_a${++this._anonSeq}`; + const entry = this._asstByResp.get(rid); + if (!entry) { + const bubble = this._spawnBubble("assistant", d.text); + this._asstByResp.set(rid, { bubble, hist: this._appendHistMsg("assistant", d.text, false) }); + this._bumpDismiss(bubble); + } else { + this._updateBubbleText(entry.bubble, d.text); + this._updateHistMsg(entry.hist, d.text, false); + this._bumpDismiss(entry.bubble); + } + this._markUnread(); + } + } + + /** + * A response closed (completed or cancelled). + * @param {{ responseId: string; status: string; audible?: boolean; transcript?: string }} detail + */ + onResponseFinished(detail) { + const { responseId, status, audible, transcript } = detail; + if (DEBUG) console.debug(`[ui] response-finished resp=${responseId} status=${status} audible=${audible} known=${this._asstByResp.has(responseId)}`); + // Without an id we can't target a specific response; the bubble will + // auto-dismiss on its own timer regardless. + if (!responseId) return; + const entry = this._asstByResp.get(responseId); + + if (status === "cancelled") { + // Keep every transcript that was received — mark it interrupted rather + // than erasing it. If the `*.transcript.done` never fired, build the row + // from the text carried in response.done. + let hist = entry?.hist ?? null; + if (!hist && transcript) { + hist = this._appendHistMsg("assistant", transcript, false); + } else if (hist && transcript) { + this._updateHistMsg(hist, transcript, false); + } + if (hist) this._markHistInterrupted(hist); + this._asstByResp.delete(responseId); + return; + } + + // Any other terminal close (completed / failed / incomplete / …): just + // release the map entry. The bubble already auto-dismisses on its timer and + // the history row persists as the conversation log. Crucially we do NOT + // touch user state here — that lifecycle is fully independent. + this._asstByResp.delete(responseId); + } + + /** The model called a tool — show an ephemeral "running" bubble. + * @param {string} name */ + onToolCall(name) { + this._bumpDismiss(this._spawnBubble("tool", name)); + this._markUnread(); + } + + /** The tool finished — append its call+result row (and any captured image). + * @param {string} name @param {string} argsJson @param {string} output @param {string} [image] */ + onToolResult(name, argsJson, output, image) { + this._appendHistTool(name, argsJson, output); + if (image) this._appendHistImage(image); // show the captured frame below the call + this._markUnread(); + } +} diff --git a/demo/ui/dom.js b/demo/ui/dom.js new file mode 100644 index 0000000..d5a645a --- /dev/null +++ b/demo/ui/dom.js @@ -0,0 +1,37 @@ +// @ts-check +/** Small shared helpers used across the UI modules: a strict query selector, + * HTML escaping for text we drop into innerHTML, error-string trimming, and + * the opt-in debug flag. */ + +/** Opt-in tracing: `localStorage.setItem("s2s.debug", "1")` then reload. */ +export const DEBUG = (() => { + try { + return localStorage.getItem("s2s.debug") === "1"; + } catch { + return false; + } +})(); + +/** + * Query a single element, throwing if it's missing (so a broken selector fails + * loudly at startup rather than as a later null-deref). + * @template {HTMLElement} T + * @param {string} selector + * @returns {T} + */ +export function $(selector) { + const el = document.querySelector(selector); + if (!el) throw new Error(`Missing element: ${selector}`); + return /** @type {T} */ (el); +} + +/** @param {string} s @returns {string} */ +export function escHtml(s) { + return s.replace(/&/g, "&").replace(//g, ">"); +} + +/** Trim a long error message to fit the orb caption. @param {string} text */ +export function truncateError(text) { + if (text.length <= 90) return text; + return text.slice(0, 87) + "…"; +} diff --git a/demo/worklets/audio-playback.js b/demo/worklets/audio-playback.js new file mode 100644 index 0000000..474020f --- /dev/null +++ b/demo/worklets/audio-playback.js @@ -0,0 +1,171 @@ +// @ts-check +/** + * AudioWorkletProcessor that plays back Float32 mono samples received from + * the main thread, upsampling whatever incoming rate the server uses + * (typically 24 kHz PCM16) to the AudioContext rate (typically 48 kHz). + * + * Lifecycle / messaging: + * + * main -> worklet: + * { kind: "config", inputRate: 24000 } one-shot at startup + * { kind: "audio", samples: Float32Array } (transferable) per chunk + * { kind: "clear" } wipe queue (barge-in) + * + * worklet -> main: + * { kind: "stats", queuedMs, played } every ~250 ms + * { kind: "underrun" } every time the queue + * runs dry mid-playback + * + * Underrun strategy: output silence. We do NOT hold the last sample (that + * tends to produce audible clicks/buzzes when long gaps appear between + * TTS chunks). A short ramp-out + ramp-in at boundaries would be nicer but + * the server's 30 ms cadence makes underruns visible only at end of turn. + */ + +const STATS_INTERVAL_FRAMES = 12000; +const FADE_FRAMES = 32; + +class AudioPlaybackProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this._inputRate = 24000; + this._stepRatio = this._inputRate / sampleRate; + this._queue = []; + this._readIdx = 0; + this._fracPos = 0; + this._playing = false; + this._framesSinceStats = 0; + this._totalPlayed = 0; + this._fadeIn = 0; + this._fadeOut = 0; + this._lastSample = 0; + + this.port.onmessage = (e) => { + const data = e.data; + if (!data || typeof data !== "object") return; + switch (data.kind) { + case "config": + if (typeof data.inputRate === "number" && data.inputRate > 0) { + this._inputRate = data.inputRate; + this._stepRatio = this._inputRate / sampleRate; + } + break; + case "audio": + if (data.samples instanceof Float32Array && data.samples.length > 0) { + this._queue.push(data.samples); + if (!this._playing) { + this._playing = true; + this._fadeIn = FADE_FRAMES; + this._fadeOut = 0; + } + } + break; + case "clear": + this._queue.length = 0; + this._readIdx = 0; + this._fracPos = 0; + this._fadeOut = FADE_FRAMES; + break; + } + }; + } + + _queuedSamples() { + let total = -this._readIdx; + for (const buf of this._queue) total += buf.length; + return Math.max(0, total); + } + + /** Linear-interp read at the current fractional position. */ + _readInterpolated() { + if (this._queue.length === 0) return null; + const head = this._queue[0]; + const idx = this._readIdx; + const frac = this._fracPos; + + let a = head[idx]; + let b; + if (idx + 1 < head.length) { + b = head[idx + 1]; + } else if (this._queue.length > 1) { + b = this._queue[1][0]; + } else { + b = a; + } + return a + (b - a) * frac; + } + + /** Advance the read position by `stepRatio`; pop consumed buffers. */ + _advance() { + this._fracPos += this._stepRatio; + while (this._fracPos >= 1) { + this._fracPos -= 1; + this._readIdx += 1; + } + while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) { + this._readIdx -= this._queue[0].length; + this._queue.shift(); + } + } + + process(_, outputs) { + const channels = outputs[0]; + if (!channels || channels.length === 0) return true; + const out = channels[0]; + const stereo = channels.length > 1 ? channels[1] : null; + + for (let i = 0; i < out.length; i++) { + let sample = 0; + + if (this._playing) { + const v = this._readInterpolated(); + if (v === null) { + // Underrun: try to ramp out cleanly to avoid clicks. + sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES); + this._lastSample = sample; + if (Math.abs(sample) < 1e-4) { + this._playing = false; + this._lastSample = 0; + this.port.postMessage({ kind: "underrun" }); + } + } else { + sample = v; + this._lastSample = v; + this._advance(); + } + + if (this._fadeIn > 0) { + const gain = 1 - this._fadeIn / FADE_FRAMES; + sample *= gain; + this._fadeIn -= 1; + } + if (this._fadeOut > 0) { + const gain = this._fadeOut / FADE_FRAMES; + sample *= gain; + this._fadeOut -= 1; + if (this._fadeOut === 0) { + this._playing = false; + this._lastSample = 0; + } + } + + this._totalPlayed += 1; + } + + out[i] = sample; + if (stereo) stereo[i] = sample; + } + + this._framesSinceStats += out.length; + if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) { + this._framesSinceStats = 0; + const queuedSamples = this._queuedSamples(); + const queuedMs = (queuedSamples / this._inputRate) * 1000; + this.port.postMessage({ kind: "stats", queuedMs, played: this._totalPlayed }); + } + + return true; + } +} + +registerProcessor("audio-playback", AudioPlaybackProcessor); diff --git a/demo/worklets/mic-capture.js b/demo/worklets/mic-capture.js new file mode 100644 index 0000000..a48860b --- /dev/null +++ b/demo/worklets/mic-capture.js @@ -0,0 +1,159 @@ +// @ts-check +/** + * AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz) + * down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it + * back to the main thread in fixed-size chunks. + * + * The Hugging Face speech-to-speech WebSocket route expects the + * `input_audio_buffer.append` payload at 16 kHz PCM16 mono. + * + * Design notes: + * - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a + * cheap low-pass before decimating. Good enough for voice STT; we lose + * a tiny bit of >8 kHz content which the pipeline discards anyway. + * - Output frames are emitted at the cadence dictated by `chunkMs` + * (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime + * server batches incoming audio so the cadence is flexible; 20-100 ms + * is the sweet spot. + * - Float -> Int16 saturates to [-1, 1] before scaling. + * - Optional noise gate: per-chunk RMS decides open/closed against a + * threshold; the gain ramps (fast attack, hold, slow release) so word + * onsets aren't clipped and quiet tails don't click. The gate only + * affects the audio we SEND; the main-thread visualiser taps the raw + * mic separately. We post the chunk RMS up every frame so the Settings + * mic meter can show the live level against the threshold. + */ + +const TARGET_RATE = 16000; +const DEFAULT_CHUNK_MS = 40; +// Gate envelope timing (fixed; only the threshold is user-tunable). +const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive +const GATE_HOLD_MS = 250; // stay open this long after the level drops back under +const GATE_RELEASE_MS = 80; // then fade closed over this long (no click) + +class MicCaptureProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS; + this._inputRate = sampleRate; + this._ratio = this._inputRate / TARGET_RATE; + this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000); + this._scratch = new Float32Array(0); + this._decimated = new Float32Array(this._chunkSamples16k); + this._enabled = true; + + // Noise gate state. Disabled by default (pure passthrough). + this._gateEnabled = false; + this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open + this._gateGain = 1; // smoothed gain currently applied + this._holdRemaining = 0; // samples left before the gate may start closing + this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE)); + this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE)); + this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE); + + this.port.onmessage = (e) => { + const data = e.data; + if (data?.kind === "enable") this._enabled = !!data.value; + else if (data?.kind === "gate") { + this._gateEnabled = !!data.enabled; + // dB -> linear amplitude. When off, threshold 0 keeps the gate open. + this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0; + } + }; + } + + /** + * Append `incoming` to the internal scratch buffer, then emit as many + * full output chunks as we have material for. + * @param {Float32Array} incoming + */ + _ingest(incoming) { + if (incoming.length === 0) return; + const next = new Float32Array(this._scratch.length + incoming.length); + next.set(this._scratch, 0); + next.set(incoming, this._scratch.length); + this._scratch = next; + this._maybeEmit(); + } + + _maybeEmit() { + const r = this._ratio; + const n = this._chunkSamples16k; + const needIn = Math.ceil(n * r); + const dec = this._decimated; + while (this._scratch.length >= needIn) { + // 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter. + let sumSq = 0; + if (Math.abs(r - 3) < 1e-6) { + // 48 kHz -> 16 kHz fast path with boxcar lowpass. + for (let i = 0; i < n; i++) { + const idx = i * 3; + const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3; + dec[i] = s; + sumSq += s * s; + } + } else { + // Generic path: linear interpolation. Slower but works at any rate + // (e.g. some Windows boxes report sampleRate=44100). + for (let i = 0; i < n; i++) { + const srcPos = i * r; + const idx = Math.floor(srcPos); + const frac = srcPos - idx; + const a = this._scratch[idx]; + const b = this._scratch[idx + 1] ?? a; + const s = a + (b - a) * frac; + dec[i] = s; + sumSq += s * s; + } + } + const rms = Math.sqrt(sumSq / n); + + // 2. Decide the gate target for this chunk, then ramp sample-by-sample. + let target = 1; + if (this._gateEnabled) { + if (rms >= this._thresholdLin) { + this._holdRemaining = this._holdSamples; // re-arm the hold + } else if (this._holdRemaining > 0) { + this._holdRemaining -= n; // coasting through the hold window + } else { + target = 0; + } + } + + // 3. Apply the (smoothed) gain and pack to Int16. + const out = new Int16Array(n); + let gain = this._gateGain; + for (let i = 0; i < n; i++) { + const coef = target > gain ? this._attackCoef : this._releaseCoef; + gain = target + (gain - target) * coef; + const s = dec[i] * gain; + const clamped = s < -1 ? -1 : s > 1 ? 1 : s; + out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + } + this._gateGain = gain; + + // Shift the scratch buffer to keep only the trailing unused samples. + const consumed = Math.floor(n * r); + this._scratch = this._scratch.slice(consumed); + + // Live input level for the Settings meter (raw RMS, pre-gate). + this.port.postMessage({ kind: "level", rms }); + + if (this._enabled) { + this.port.postMessage(out.buffer, [out.buffer]); + } + // When disabled (mic muted) we silently consume input so the worklet + // stays alive and the buffer never grows unbounded. + } + } + + process(inputs) { + const input = inputs[0]; + if (!input || input.length === 0 || !input[0]) return true; + const mono = input[0]; + if (mono.length > 0) this._ingest(mono); + return true; + } +} + +registerProcessor("mic-capture", MicCaptureProcessor); diff --git a/demo/ws/codec.js b/demo/ws/codec.js new file mode 100644 index 0000000..511ad9e --- /dev/null +++ b/demo/ws/codec.js @@ -0,0 +1,57 @@ +// @ts-check +/** + * Pure, stateless helpers for the WebSocket realtime client: base64 <-> PCM + * conversion for the audio frames on the wire, transcript extraction from a + * `response.done` payload, and a tiny URL helper. Kept separate from the client + * so the protocol/state logic stays readable. + */ + +/** @param {string} url */ +export function trimTrailingSlash(url) { + return url.endsWith("/") ? url.slice(0, -1) : url; +} + +/** + * Pull the assistant transcript out of a `response.done` payload. The text + * lives in `response.output[].content[].transcript` (audio) or `.text`. Used as + * the source of truth for interrupted replies, where the dedicated + * `*.transcript.done` event may never arrive. + * @param {any} response + * @returns {string} + */ +export function extractResponseTranscript(response) { + const output = response?.output; + if (!Array.isArray(output)) return ""; + /** @type {string[]} */ + const parts = []; + for (const item of output) { + for (const part of item?.content ?? []) { + const text = part?.transcript ?? part?.text; + if (typeof text === "string" && text.trim()) parts.push(text.trim()); + } + } + return parts.join(" ").trim(); +} + +/** @param {ArrayBuffer} buf */ +export function base64FromArrayBuffer(buf) { + const bytes = new Uint8Array(buf); + // Chunked encoding so we don't blow up the call stack on long buffers. + let binary = ""; + const chunk = 0x8000; + for (let i = 0; i < bytes.length; i += chunk) { + binary += String.fromCharCode.apply(null, /** @type {number[]} */ ( + /** @type {unknown} */ (bytes.subarray(i, i + chunk)) + )); + } + return btoa(binary); +} + +/** @param {string} b64 */ +export function base64ToBytes(b64) { + const binary = atob(b64); + const len = binary.length; + const out = new Uint8Array(len); + for (let i = 0; i < len; i++) out[i] = binary.charCodeAt(i); + return out; +} diff --git a/demo/ws/orb-visualizer.js b/demo/ws/orb-visualizer.js new file mode 100644 index 0000000..082f601 --- /dev/null +++ b/demo/ws/orb-visualizer.js @@ -0,0 +1,98 @@ +// @ts-check +/** + * Orb spectrum visualiser. Each animation frame it reads two AnalyserNodes (the + * mic input and the TTS output) and maps the low-frequency speech energy onto + * the orb's CSS custom properties: + * - `--bar0`..`--bar4` the 5-band level meter + * - `--ai-audio-level` the global "Reachy talks" glow / scale pulse + * + * The bottom of the FFT is where speech energy lives, so the band edges stay + * low — that keeps the bars dancing on voice rather than on noise. While the AI + * is speaking we source the bars from the OUTPUT analyser so the orb pulses with + * Reachy's voice instead of sitting dead while the user is silent. + */ + +// Exported so the client can size its AnalyserNodes to match our buffer. +export const VIS_FFT_SIZE = 256; +const VIS_BAND_COUNT = 5; +const VIS_BAND_EDGES = [2, 5, 9, 16, 28, 52]; +const VIS_ATTACK = 0.6; // weight for new sample on upswing (snappy) +const VIS_RELEASE = 0.18; // weight for new sample on decay (gentle fade) + +export class OrbVisualiser { + /** + * @param {AnalyserNode} micAnalyser + * @param {AnalyserNode} outAnalyser + * @param {() => boolean} isAiSpeaking Source the bars from the AI output when + * true, otherwise from the mic. + */ + constructor(micAnalyser, outAnalyser, isAiSpeaking) { + this._mic = micAnalyser; + this._out = outAnalyser; + this._isAiSpeaking = isAiSpeaking; + this._buf = new Uint8Array(micAnalyser.frequencyBinCount); + this._bands = new Float32Array(VIS_BAND_COUNT); + this._aiLevel = 0; + /** @type {number | null} */ + this._frame = null; + } + + /** Begin the rAF loop (idempotent). */ + start() { + if (this._frame !== null) return; + const root = document.documentElement; + const tick = () => { + this._frame = requestAnimationFrame(tick); + this._update(root); + }; + this._frame = requestAnimationFrame(tick); + } + + /** Stop the loop and clear the CSS vars so the orb returns to rest. */ + stop() { + if (this._frame !== null) { + cancelAnimationFrame(this._frame); + this._frame = null; + } + const root = document.documentElement; + for (let i = 0; i < VIS_BAND_COUNT; i++) root.style.removeProperty(`--bar${i}`); + root.style.removeProperty("--ai-audio-level"); + } + + /** @param {HTMLElement} root */ + _update(root) { + // Mic bars: split FFT into 5 log-ish bands, smooth, write CSS vars. + const source = this._isAiSpeaking() ? this._out : this._mic; + source.getByteFrequencyData(this._buf); + + for (let b = 0; b < VIS_BAND_COUNT; b++) { + const lo = VIS_BAND_EDGES[b]; + const hi = VIS_BAND_EDGES[b + 1]; + let sum = 0; + let n = 0; + for (let i = lo; i < hi && i < this._buf.length; i++) { + sum += this._buf[i]; + n += 1; + } + const target = n > 0 ? sum / (n * 255) : 0; + const prev = this._bands[b]; + const k = target > prev ? VIS_ATTACK : VIS_RELEASE; + const next = prev + (target - prev) * k; + this._bands[b] = next; + root.style.setProperty(`--bar${b}`, next.toFixed(3)); + } + + // Global AI audio level: peak of the output analyser, used by the CSS to + // make the orb's glow / scale react to Reachy's voice. + this._out.getByteFrequencyData(this._buf); + let peak = 0; + const limit = Math.min(this._buf.length, VIS_BAND_EDGES[VIS_BAND_COUNT]); + for (let i = 0; i < limit; i++) { + if (this._buf[i] > peak) peak = this._buf[i]; + } + const aiTarget = peak / 255; + const k = aiTarget > this._aiLevel ? VIS_ATTACK : VIS_RELEASE; + this._aiLevel = this._aiLevel + (aiTarget - this._aiLevel) * k; + root.style.setProperty("--ai-audio-level", this._aiLevel.toFixed(3)); + } +} diff --git a/demo/ws/s2s-ws-client.js b/demo/ws/s2s-ws-client.js new file mode 100644 index 0000000..aa6eaec --- /dev/null +++ b/demo/ws/s2s-ws-client.js @@ -0,0 +1,1108 @@ +// @ts-check +/** + * Minimal WebSocket client for the Hugging Face speech-to-speech load balancer. + * + * Two-step handshake (same /session route as the WebRTC client): + * + * 1. POST `/session` -> JSON `{ connect_url: wss:///v1/realtime?session_token=, ... }` + * 2. Open a WebSocket directly on `connect_url` (no rewrite, unlike the WebRTC client). + * + * Once the socket is open we follow the OpenAI Realtime GA WebSocket + * protocol: + * + * - Server pushes `session.created` immediately after upgrade. + * - We send `session.update` (GA schema: `session.audio.{input,output}`, + * `session.output_modalities`, ...). + * - We stream mic audio as PCM16 16 kHz mono base64 chunks via + * `input_audio_buffer.append`. + * - The server pushes `response.output_audio.delta` (PCM16 24 kHz mono + * base64) and transcript deltas. + * + * Audio is handled internally via two AudioWorklet processors so the + * client owns the full mic-in / speaker-out pipeline. The main app only + * sees high-level lifecycle events (`status`, `transcript`, `error`, + * `session`), the same shape as the WebRTC client. + * + * @typedef {"idle" | "creating-session" | "queued" | "your-turn" | "connecting" | + * "connected" | "user-speaking" | "processing" | "ai-speaking" | + * "closed" | "error" + * } WsStatus + * + * @typedef {Object} WsSessionInfo + * @property {string} sessionId + * @property {string} connectUrl + * @property {string} websocketUrl + * @property {string} sessionToken + * @property {number} pendingTimeoutS + * @property {string} [tier] Login tier from the session proxy ("anon"|"free"|"pro"). + * @property {boolean} [limited] Whether this session is metered (heartbeat needed). + * @property {number} [heartbeatSec] Suggested heartbeat cadence in seconds. + * @property {number} [remainingSec] Daily budget left after this grant (display). + * + * @typedef {Object} WsClientOptions + * @property {string} [sessionUrl] URL to POST for the session handshake (returns + * `{ connect_url, ... }`). Usually a same-origin proxy like `api/session` so the + * load-balancer address stays server-side. Provide this OR `directUrl`. + * @property {string} [loadBalancerUrl] Load-balancer base URL. Legacy/direct + * alternative to `sessionUrl`: the client POSTs `/session` itself. Prefer + * `sessionUrl` so the LB address isn't exposed to the browser. + * @property {string} [directUrl] Full WebSocket URL of an s2s realtime endpoint + * (e.g. `ws://localhost:8080/v1/realtime`). When set, the client skips the + * session POST and dials it directly — no load balancer in between. + * @property {string} voice + * @property {string} instructions + * @property {MediaStream} [micStream] Live mic stream. Provide this OR `acquireMic`. + * @property {() => Promise} [acquireMic] Lazily obtain the mic stream, + * called only once a session is actually granted (after any queue wait). Lets the + * caller prime mic permission up front but not hold the mic 'in use' indicator on + * while waiting in line. Ignored if `micStream` is already set. + * @property {AudioContext} [audioContext] Pre-created (and resumed) context. + * iOS Safari only lets an AudioContext start from within a user gesture, so + * the caller creates/resumes it synchronously on the orb tap and hands it + * here; otherwise it stays suspended (silent) after the mic/session awaits. + * @property {ToolDef[]} [tools] Function tools declared to the backend in the + * initial `session.update`. The model decides when to call them; the caller + * executes and replies via `sendToolOutput` + `requestResponse`. + * @property {NoiseGate} [noiseGate] Client-side noise gate applied to the mic + * before it's sent. Tunable live via `setNoiseGate`. + * + * @typedef {Object} NoiseGate + * @property {boolean} enabled + * @property {number} thresholdDb Open threshold in dBFS (e.g. -45). + * + * @typedef {Object} ToolDef + * @property {"function"} type + * @property {string} name + * @property {string} description + * @property {object} parameters JSON Schema for the call arguments. + * + * @typedef {Object} TranscriptEvent + * @property {"user" | "assistant"} role + * @property {string} text + * @property {boolean} partial + */ + +import { + base64FromArrayBuffer, + base64ToBytes, + extractResponseTranscript, + trimTrailingSlash, +} from "./codec.js"; +import { OrbVisualiser, VIS_FFT_SIZE } from "./orb-visualizer.js"; + +/** Build an Error carrying a `code` (and optional extra fields) so callers can + * branch on the failure kind: "limit" | "queue-full" | "queue-expired" | "aborted". + * @param {string} message @param {string} code @param {object} [extra] */ +function _codedError(message, code, extra) { + const err = /** @type {Error & { code?: string }} */ (new Error(message)); + err.code = code; + if (extra) Object.assign(err, extra); + return err; +} + +// The s2s pipeline runs internally at 16 kHz mono PCM. The WebRTC transport +// resamples to 48 kHz for Opus, but the WebSocket transport emits the +// native pipeline rate. We don't (can't) override it via `audio.output.format` +// because the server's pydantic validator rejects the whole `session.update` +// as soon as a sub-field shape it doesn't know about appears. +const OUTPUT_SAMPLE_RATE = 16000; +const MIC_CHUNK_MS = 40; + +export class S2sWsRealtimeClient extends EventTarget { + /** @param {WsClientOptions} options */ + constructor(options) { + super(); + /** @type {WsClientOptions} */ + this.options = options; + /** @type {ToolDef[]} Function tools declared to the backend. */ + this._tools = options.tools ?? []; + /** @type {string} Direct realtime WS URL (set => skip the LB session POST). */ + this._directUrl = options.directUrl ?? ""; + /** @type {string} Where to POST for the session handshake. Prefer the + * explicit `sessionUrl`; fall back to `/session` for callers + * that still pass the LB address directly. */ + this._sessionUrl = options.sessionUrl + ? options.sessionUrl + : options.loadBalancerUrl + ? `${trimTrailingSlash(options.loadBalancerUrl)}/session` + : ""; + /** @type {(() => Promise) | null} Lazy mic acquisition (post-grant). */ + this._acquireMic = options.acquireMic ?? null; + /** @type {boolean} Set by close() to abort a queue wait in progress. */ + this._closed = false; + /** @type {string} The active queue ticket id while waiting (else ""). */ + this._queueId = ""; + /** @type {(() => void) | null} Wakes the queue poll sleep early on close(). */ + this._queueWake = null; + /** @type {ReturnType | 0} */ + this._queueTimer = 0; + // Join gate: after waiting in line the caller must explicitly `join()` before + // we dial, so a slot isn't spent on someone who walked away. Resolved by + // join(), rejected on timeout (the LB reclaims the slot) or close(). + /** @type {(() => void) | null} */ + this._joinResolve = null; + /** @type {((err: Error) => void) | null} */ + this._joinReject = null; + /** @type {ReturnType | 0} */ + this._joinTimer = 0; + /** @type {NoiseGate} Mic noise gate; off by default. */ + this._noiseGate = options.noiseGate ?? { enabled: false, thresholdDb: -45 }; + /** @type {WebSocket | null} */ + this._ws = null; + /** @type {AudioContext | null} */ + this._ctx = null; + /** @type {MediaStreamAudioSourceNode | null} */ + this._micSrc = null; + /** @type {AudioWorkletNode | null} */ + this._captureNode = null; + /** @type {AudioWorkletNode | null} */ + this._playbackNode = null; + /** @type {GainNode | null} */ + this._captureSink = null; + /** @type {AnalyserNode | null} */ + this._micAnalyser = null; + /** @type {AnalyserNode | null} */ + this._outAnalyser = null; + /** @type {OrbVisualiser | null} */ + this._visualiser = null; + /** @type {WsStatus} */ + this._status = "idle"; + this._aiSpeaking = false; + /** @type {Set} response_ids that have actually played audio, so the + * UI can tell a barge-in cut (keep it) from a never-heard speculative + * response (drop it). */ + this._audibleResponses = new Set(); + /** @type {Map} The CURRENT assistant transcript segment per + * response, accumulated from streamed deltas (reset on each segment's done). */ + this._asstTranscriptByResp = new Map(); + /** @type {Map} Completed assistant transcript segments per + * response, space-joined. A single response can emit several + * `*.transcript.done` events; we concatenate them until response.done. */ + this._asstFullByResp = new Map(); + this._muted = false; + // ── Response lock ──────────────────────────────────────────────────── + // The backend allows only ONE response in flight: creating a second while + // one is active fails with `conversation_already_has_active_response`. So + // we serialize response.create. `_openResponses` counts responses the + // server has confirmed (response.created) but not yet finished + // (response.done) — it's cumulative, so every create maps to one done. + // `_createInFlight` covers the window after we send a create but before its + // response.created echo. Any requestResponse() made while locked is queued + // and replayed, one at a time, as each response.done frees the slot. + this._openResponses = 0; + this._createInFlight = false; + /** @type {{ image?: string }[]} Pending response.create payloads, one per + * queued requestResponse(). A payload may carry an image to send just + * before its create (so the frame travels with the create, not eagerly). */ + this._createQueue = []; + /** @type {Promise | null} */ + this._readyPromise = null; + this._sessionConfigured = false; + this._debug = (() => { try { return localStorage.getItem("s2s.debug") === "1"; } catch { return false; } })(); + } + + get status() { + return this._status; + } + + /** @param {WsStatus} status */ + _setStatus(status) { + if (this._status === status) return; + this._status = status; + this.dispatchEvent(new CustomEvent("status", { detail: { status } })); + } + + /** Full assistant transcript so far for a response: the completed segments + * plus the in-progress one, all space-joined. + * @param {string} rid @returns {string} */ + _asstDisplay(rid) { + const full = this._asstFullByResp.get(rid) || ""; + const seg = this._asstTranscriptByResp.get(rid) || ""; + if (!seg) return full; + return full ? `${full} ${seg}` : seg; + } + + _markAudible() { + if (this._status === "ai-speaking") return; + if (this._status === "closed" || this._status === "error") return; + this._setStatus("ai-speaking"); + } + + /** + * Full handshake. Resolves once the WS is open AND the audio pipeline is + * ready to send/receive samples. + * @returns {Promise} + */ + async connect() { + if (this._ws) throw new Error("Already connected"); + + let connectUrl; + if (this._directUrl) { + // Direct mode: no load balancer, no /session POST — dial the realtime + // endpoint straight away (e.g. a local s2s server). + connectUrl = this._directUrl; + this._setStatus("connecting"); + } else { + if (!this._sessionUrl) { + throw new Error("No session endpoint or direct URL configured"); + } + this._setStatus("creating-session"); + const { grant, waited } = await this._createSessionOrQueue(); + if (this._closed) throw _codedError("connect aborted", "aborted"); + // If we waited in line, don't dial until the user explicitly joins — this + // keeps a freed slot from being spent on someone who stepped away, and the + // click is a fresh gesture (re-arms the AudioContext on iOS). + if (waited) { + await this._awaitJoin(grant); + if (this._closed) throw _codedError("connect aborted", "aborted"); + } + this.dispatchEvent(new CustomEvent("session", { detail: { info: grant } })); + connectUrl = grant.connectUrl; + this._setStatus("connecting"); + } + + // Acquire the mic now — only once a slot is actually ours. The caller primed + // permission up front, so this is silent and the 'in use' indicator lights + // only for a real, connecting session (never during a queue wait). + if (!this.options.micStream && this._acquireMic) { + this.options.micStream = await this._acquireMic(); + } + + // Spin up the AudioContext + worklets in parallel with the WS dial. + const audioReady = this._setupAudio(); + const wsReady = this._openWebSocket(connectUrl); + await Promise.all([audioReady, wsReady]); + } + + /** + * POST the session handshake; if the pool is busy, wait in the queue (polling + * position) until a slot is claimed. Resolves to a grant plus whether we had to + * wait (which decides if an explicit join is required before dialing). + * @returns {Promise<{ grant: WsSessionInfo, waited: boolean }>} + */ + async _createSessionOrQueue() { + const first = await this._postSession(); + if (first.state === "queued") { + this._setStatus("queued"); + const grant = await this._pollQueue(first); + return { grant, waited: true }; + } + return { grant: first.grant, waited: false }; + } + + /** + * Hold at the front of the line until the user clicks join (resolves the gate) + * or the grant lapses. Announces "your-turn" + a deadline the UI counts down. + * @param {WsSessionInfo} grant + * @returns {Promise} + */ + _awaitJoin(grant) { + // The LB reclaims an unclaimed slot at its pending timeout; expire the gate a + // touch earlier so we never dial a session the LB just reaped. + const windowS = Math.max(3, (grant.pendingTimeoutS || 60) - 3); + this._setStatus("your-turn"); + this.dispatchEvent( + new CustomEvent("ready-to-join", { detail: { info: grant, expiresSec: windowS } }), + ); + return new Promise((resolve, reject) => { + this._joinResolve = resolve; + this._joinReject = reject; + this._joinTimer = setTimeout(() => { + this._joinResolve = null; + this._joinReject = null; + reject(_codedError("Your spot expired", "join-expired")); + }, windowS * 1000); + }); + } + + /** Accept the held slot and let connect() proceed to dial. Called from the + * "Join now" click, so it's a user gesture: re-resume the AudioContext, which + * iOS may have suspended while we waited. */ + join() { + if (this._joinTimer) { + clearTimeout(this._joinTimer); + this._joinTimer = 0; + } + try { + void this.options.audioContext?.resume(); + } catch { + // best-effort; _setupAudio resumes again + } + const resolve = this._joinResolve; + this._joinResolve = null; + this._joinReject = null; + resolve?.(); + } + + /** + * POST /session once. Returns either a granted session or a queue ticket. + * @returns {Promise<{ state: "granted", grant: WsSessionInfo } | { state: "queued", queueId: string, position: number, pollIntervalS: number }>} + */ + async _postSession() { + const url = this._sessionUrl; + console.log("[ws] POST", url); + const response = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + if (response.status === 402) { + // The session proxy refused: today's per-tier time budget is spent. Surface + // it as a typed error so the UI shows the limit modal, not a crash. + const body = await response.json().catch(() => ({})); + throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier }); + } + if (response.status === 503) { + const body = await response.json().catch(() => ({})); + if (body?.state === "at_capacity") { + throw _codedError("The queue is full — try again shortly.", "queue-full"); + } + throw new Error("/session failed (503)"); + } + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`/session failed (${response.status}): ${text}`); + } + const json = await response.json(); + if (json.state === "queued") { + return { + state: "queued", + queueId: json.queue_id, + position: json.position, + pollIntervalS: json.poll_interval_s, + }; + } + return { state: "granted", grant: this._parseGrant(json) }; + } + + /** + * Poll the waiting queue until this ticket claims a slot. Emits `queue` events + * ({ position }) as the line advances. Throws on limit (402), expiry (404), or + * close(). Transient network/5xx blips are ignored and retried next tick. + * @param {{ queueId: string, position: number, pollIntervalS: number }} ticket + * @returns {Promise} + */ + async _pollQueue(ticket) { + const intervalMs = Math.max(1, ticket.pollIntervalS || 2) * 1000; + this._queueId = ticket.queueId; + this._emitQueue(ticket.position); + + while (true) { + await this._queueSleep(intervalMs); + if (this._closed) throw _codedError("queue wait aborted", "aborted"); + + let response; + try { + response = await fetch(`api/queue/${encodeURIComponent(this._queueId)}`, { + headers: { "Content-Type": "application/json" }, + }); + } catch { + continue; // network blip — keep our place, retry next tick + } + + if (response.status === 402) { + const body = await response.json().catch(() => ({})); + throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier }); + } + if (response.status === 404) { + throw _codedError("Queue timed out", "queue-expired"); + } + if (!response.ok) continue; // 502/503 — transient, retry + + const json = await response.json().catch(() => null); + if (!json) continue; + if (json.state === "queued") { + this._emitQueue(json.position); + continue; + } + // Reached the front and claimed a slot. + this._queueId = ""; + return this._parseGrant(json); + } + } + + /** @param {number} position */ + _emitQueue(position) { + this.dispatchEvent( + new CustomEvent("queue", { detail: { position, queueId: this._queueId } }), + ); + } + + /** A sleep that close() can cut short so a queued client tears down promptly. + * @param {number} ms */ + _queueSleep(ms) { + return new Promise((resolve) => { + this._queueWake = resolve; + this._queueTimer = setTimeout(() => { + this._queueWake = null; + resolve(); + }, ms); + }); + } + + /** @param {any} json @returns {WsSessionInfo} */ + _parseGrant(json) { + return { + sessionId: json.session_id, + connectUrl: json.connect_url, + websocketUrl: json.websocket_url, + sessionToken: json.session_token, + pendingTimeoutS: json.pending_timeout_s, + tier: json.tier, + limited: json.limited, + heartbeatSec: json.heartbeatSec, + remainingSec: json.remainingSec, + }; + } + + async _setupAudio() { + // Prefer a context the caller already created + resumed inside the tap + // gesture (required on iOS). Fall back to creating one here for callers + // that don't (desktop is lenient about the gesture timing). + // Most desktops give us 48 kHz, mobiles can give 44.1/24/16 kHz; the + // capture worklet handles any rate (linear interp fallback). + const ctx = this.options.audioContext ?? new AudioContext({ latencyHint: "interactive" }); + this._ctx = ctx; + + // Resume if still suspended. This is best-effort here — on iOS the resume + // that actually counts is the one the caller did synchronously on tap. + if (ctx.state === "suspended") { + try { + await ctx.resume(); + } catch (err) { + console.warn("[ws] AudioContext resume failed:", err); + } + } + + // The worklets live at the repo root, one level up from this module. + const base = new URL("../worklets/", import.meta.url); + await ctx.audioWorklet.addModule(new URL("mic-capture.js", base).href); + await ctx.audioWorklet.addModule(new URL("audio-playback.js", base).href); + + const captureNode = new AudioWorkletNode(ctx, "mic-capture", { + numberOfInputs: 1, + numberOfOutputs: 0, + processorOptions: { chunkMs: MIC_CHUNK_MS }, + }); + captureNode.port.onmessage = (e) => { + const data = e.data; + if (data instanceof ArrayBuffer) { + this._onMicChunk(data); + } else if (data?.kind === "level") { + // Raw pre-gate mic RMS for the Settings meter. + this.dispatchEvent(new CustomEvent("input-level", { detail: { rms: data.rms } })); + } + }; + // Push the initial gate config now that the worklet exists. + captureNode.port.postMessage({ kind: "gate", ...this._noiseGate }); + this._captureNode = captureNode; + + const micSrc = ctx.createMediaStreamSource(this.options.micStream); + micSrc.connect(captureNode); + this._micSrc = micSrc; + + // Mic analyser: tap the mic in parallel with the worklet so we get the + // raw (un-resampled, un-clipped) signal for the visualiser. + const micAnalyser = ctx.createAnalyser(); + micAnalyser.fftSize = VIS_FFT_SIZE; + micAnalyser.smoothingTimeConstant = 0; + micSrc.connect(micAnalyser); + this._micAnalyser = micAnalyser; + + const playbackNode = new AudioWorkletNode(ctx, "audio-playback", { + numberOfInputs: 0, + numberOfOutputs: 1, + outputChannelCount: [1], + }); + playbackNode.port.postMessage({ kind: "config", inputRate: OUTPUT_SAMPLE_RATE }); + playbackNode.port.onmessage = (e) => this._onPlaybackMessage(e.data); + + // Output analyser sits between the playback worklet and the speakers. + const outAnalyser = ctx.createAnalyser(); + outAnalyser.fftSize = VIS_FFT_SIZE; + outAnalyser.smoothingTimeConstant = 0.3; + playbackNode.connect(outAnalyser); + outAnalyser.connect(ctx.destination); + this._outAnalyser = outAnalyser; + this._playbackNode = playbackNode; + + this._visualiser = new OrbVisualiser(micAnalyser, outAnalyser, () => this._aiSpeaking); + this._visualiser.start(); + } + + /** @param {string} connectUrl */ + _openWebSocket(connectUrl) { + return new Promise((resolve, reject) => { + const ws = new WebSocket(connectUrl); + ws.binaryType = "arraybuffer"; + this._ws = ws; + + const onceOpen = () => { + ws.removeEventListener("open", onceOpen); + ws.removeEventListener("error", onceErr); + resolve(); + }; + const onceErr = (e) => { + ws.removeEventListener("open", onceOpen); + ws.removeEventListener("error", onceErr); + reject(new Error(`WebSocket failed to open: ${e?.type ?? "error"}`)); + }; + ws.addEventListener("open", onceOpen); + ws.addEventListener("error", onceErr); + + ws.addEventListener("message", (e) => this._onWsMessage(e.data)); + ws.addEventListener("close", (e) => this._onWsClose(e)); + ws.addEventListener("error", (e) => { + console.error("[ws] socket error", e); + }); + }); + } + + /** + * @param {{ kind: string; queuedMs?: number; played?: number }} data + */ + _onPlaybackMessage(data) { + if (data?.kind === "underrun") { + // Server stopped sending audio mid-response. Most likely the turn + // ended cleanly (a response.done usually arrives just before/after + // this). We let the state machine fall back to "connected" via the + // response.done event handler. + } + } + + /** + * Mic worklet just sent us a ~40 ms PCM16 16 kHz mono chunk. + * Base64-encode and forward via the WS. + * @param {ArrayBuffer} pcm16Buffer + */ + _onMicChunk(pcm16Buffer) { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; + if (!this._sessionConfigured) return; // Server rejects audio before session.update. + if (this._muted) return; + const b64 = base64FromArrayBuffer(pcm16Buffer); + this._send({ type: "input_audio_buffer.append", audio: b64 }); + } + + /** + * @param {string | ArrayBuffer | Blob} raw + */ + async _onWsMessage(raw) { + let text; + if (typeof raw === "string") { + text = raw; + } else if (raw instanceof ArrayBuffer) { + text = new TextDecoder("utf-8").decode(raw); + } else if (raw instanceof Blob) { + text = await raw.text(); + } else { + return; + } + + let event; + try { + event = JSON.parse(text); + } catch { + return; + } + + const type = event?.type; + if (typeof type !== "string") return; + // Opt-in event tracing for diagnosing turn/transcript issues. Enable with + // `localStorage.setItem("s2s.debug", "1")` in the browser console. + if (this._debug) { + const extra = type.startsWith("conversation.item.input_audio_transcription") + ? ` item=${event.item_id} ci=${event.content_index} ${event.delta ?? event.transcript ?? ""}` + : type.startsWith("response.") + ? ` resp=${event.response_id ?? event.response?.id ?? ""} status=${event.response?.status ?? ""} ${event.transcript ?? ""}` + : ""; + console.debug(`[ws] ${type}${extra}`); + } + + switch (type) { + case "session.created": + // Server-side defaults for the s2s pipeline are already what we + // want (server_vad, whisper-1 transcription, PCM16 16k in / 24k + // out). We only push the user-tunable bits: voice + instructions. + this._sendSessionUpdate(); + this._sessionConfigured = true; + if (this._status === "connecting") this._setStatus("connected"); + break; + + case "session.updated": + // Acknowledged by server, nothing to do. + break; + + case "input_audio_buffer.speech_started": + // User started speaking — stop any audio still playing OR queued, every + // time. We clear unconditionally (not just when `_aiSpeaking`): after a + // reply or a tool result the worklet's ring buffer can still be draining + // even though we already flipped `_aiSpeaking` off, and that tail would + // otherwise keep playing over the user's barge-in. + this._playbackNode?.port.postMessage({ kind: "clear" }); + this._aiSpeaking = false; + this._setStatus("user-speaking"); + break; + + case "input_audio_buffer.speech_stopped": + if (this._status === "user-speaking") this._setStatus("processing"); + break; + + case "response.created": + // A response now owns the slot — count it and clear our create guard + // (this confirms either our create or a server-initiated one). + this._openResponses++; + this._createInFlight = false; + if (this._status === "connected" || this._status === "user-speaking") { + this._setStatus("processing"); + } + break; + + case "response.output_item.added": + if (this._status === "connected" || this._status === "user-speaking") { + this._setStatus("processing"); + } + break; + + case "response.audio.delta": + case "response.output_audio.delta": { + this._pushAudioDelta(event.delta); + const rid = event.response_id ?? event.response?.id; + if (rid) this._audibleResponses.add(rid); + if (!this._aiSpeaking) { + this._aiSpeaking = true; + this._markAudible(); + } + break; + } + + case "response.content_part.added": { + const part = event.part; + if (part?.type === "audio" || part?.type === "output_audio") { + this._markAudible(); + } + break; + } + + case "response.done": { + this._aiSpeaking = false; + // This response freed the slot (completion OR cancellation both arrive + // as response.done). Decrement and, if a create was waiting, replay it. + this._openResponses = Math.max(0, this._openResponses - 1); + if (this._status === "ai-speaking" || this._status === "processing") { + this._setStatus("connected"); + } + // A response closes here for BOTH normal completion and cancellation + // (the s2s server signals a speculative-turn interrupt as + // `response.done` with status "cancelled" — there is no separate + // `response.cancelled` event). Surface the id + status so the UI can + // drop a cancelled response's transcript and commit a completed one. + const status = event.response?.status ?? "completed"; + const responseId = event.response?.id ?? ""; + // Did this response ever play audio? Distinguishes a barge-in cut (the + // user heard part of it) from a speculative response that never played. + const audible = responseId ? this._audibleResponses.has(responseId) : false; + this._audibleResponses.delete(responseId); + // Pull whatever transcript the response carries, falling back to the + // segments we concatenated from the `*.transcript.done` events (plus any + // in-progress delta). For an interrupted reply the response payload may + // be empty, so this is the last chance to capture the text. + const transcript = + extractResponseTranscript(event.response) || + this._asstDisplay(responseId) || + ""; + // Response finished — clear both transcript accumulators for it. + this._asstTranscriptByResp.delete(responseId); + this._asstFullByResp.delete(responseId); + this.dispatchEvent(new CustomEvent("response-finished", { + detail: { responseId, status, audible, transcript }, + })); + // The slot is free now — replay a queued create (e.g. a tool follow-up + // that arrived while this response was still running). + this._flushQueuedCreate(); + break; + } + + case "response.function_call_arguments.done": { + const name = typeof event.name === "string" ? event.name : ""; + const args = typeof event.arguments === "string" ? event.arguments : "{}"; + const callId = typeof event.call_id === "string" ? event.call_id : ""; + if (name) { + this.dispatchEvent(new CustomEvent("toolcall", { + detail: { name, arguments: args, callId }, + })); + } else { + // A nameless call can't be executed, so no function_call_output is + // ever sent and the model would wait forever for a result. The + // backend shouldn't emit these; warn loudly rather than stall silently. + console.warn(`[ws] function_call_arguments.done with no name (call_id=${callId}); cannot run tool — turn may stall`); + } + break; + } + + case "conversation.item.input_audio_transcription.delta": { + const delta = typeof event.delta === "string" ? event.delta : ""; + if (delta) { + // `itemId` is REUSED across a speculative continuation, so the UI + // groups both segments into one message. The delta carries the full + // cumulative transcript so far (not an increment). + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { + role: "user", + text: delta, + partial: true, + itemId: typeof event.item_id === "string" ? event.item_id : "", + }, + }), + ); + } + break; + } + + case "conversation.item.input_audio_transcription.completed": { + const transcript = typeof event.transcript === "string" ? event.transcript : ""; + if (transcript) { + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { + role: "user", + text: transcript, + partial: false, + itemId: typeof event.item_id === "string" ? event.item_id : "", + }, + }), + ); + } + break; + } + + case "response.audio_transcript.delta": + case "response.output_audio_transcript.delta": { + // Stream the assistant transcript live: accumulate the incremental + // deltas and push the running text to the UI. Every transcribe event we + // receive reaches the conversation, so an interrupted reply already has + // its partial text even if the `.done` never fires. + this._markAudible(); + const rid = typeof event.response_id === "string" ? event.response_id : ""; + const delta = typeof event.delta === "string" ? event.delta : ""; + if (delta) { + this._asstTranscriptByResp.set(rid, (this._asstTranscriptByResp.get(rid) || "") + delta); + // Show completed segments + the segment streaming in right now. + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { role: "assistant", text: this._asstDisplay(rid), partial: true, responseId: rid }, + }), + ); + } + break; + } + + case "response.audio_transcript.done": + case "response.output_audio_transcript.done": { + const rid = typeof event.response_id === "string" ? event.response_id : ""; + // This is ONE completed segment. A response can emit several; concatenate + // them, space-separated, until response.done clears the accumulator. + const segment = + (typeof event.transcript === "string" && event.transcript) || + this._asstTranscriptByResp.get(rid) || + ""; + this._asstTranscriptByResp.delete(rid); // segment finished; next one starts fresh + if (segment) { + const prev = this._asstFullByResp.get(rid) || ""; + this._asstFullByResp.set(rid, prev ? `${prev} ${segment}` : segment); + } + const full = this._asstFullByResp.get(rid) || ""; + if (full) { + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { role: "assistant", text: full, partial: false, responseId: rid }, + }), + ); + } + break; + } + + case "error": { + const err = event.error; + console.error("[ws] server error:", err); + // The "another response is already active" race: our optimistic create + // collided with a still-running response. Don't surface it — clear the + // in-flight guard and re-queue, so the create replays on the next + // response.done (never retried immediately, which would just collide + // again). + if (err?.type === "conversation_already_has_active_response" || + err?.code === "conversation_already_has_active_response") { + if (this._createInFlight) { + this._createInFlight = false; + // Re-queue a BARE create: any image on the original payload was + // already sent before this (rejected) create, so don't resend it. + this._createQueue.push({}); + } + break; + } + // Every other server error is non-fatal: surface it for logging but + // NEVER tear the socket down. Only transport failures (close / failed + // open) are fatal, and those come through their own paths. + this.dispatchEvent( + new CustomEvent("server-error", { detail: { error: new Error(err?.message ?? "Server error") } }), + ); + break; + } + } + } + + /** @param {string} b64 */ + _pushAudioDelta(b64) { + if (!this._playbackNode) return; + if (!b64) return; + const bytes = base64ToBytes(b64); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const samples = new Float32Array(bytes.byteLength / 2); + for (let i = 0; i < samples.length; i++) { + const s = view.getInt16(i * 2, true); + samples[i] = s < 0 ? s / 0x8000 : s / 0x7fff; + } + this._playbackNode.port.postMessage({ kind: "audio", samples }, [samples.buffer]); + } + + /** @param {CloseEvent} ev */ + _onWsClose(ev) { + console.log("[ws] socket closed:", ev.code, ev.reason); + if (this._status === "closed" || this._status === "error") return; + if (ev.code === 1000) { + this._setStatus("closed"); + } else { + this.dispatchEvent( + new CustomEvent("error", { + detail: { error: new Error(`WebSocket closed (${ev.code}) ${ev.reason || ""}`.trim()) }, + }), + ); + this._setStatus("error"); + } + } + + _sendSessionUpdate() { + // Minimal payload: only the bits the user is allowed to configure. + // The s2s server already defaults to server_vad, whisper-1 + // transcription, 16 kHz PCM input and 24 kHz PCM output, so we don't + // need (and must not send) `audio.input.format`, `audio.input.transcription`, + // `audio.input.turn_detection` or `audio.output.format`: the pydantic + // validator on the server rejects the whole event if any unknown or + // future-shaped sub-field shows up. + /** @type {Record} */ + const session = { + type: "realtime", + instructions: this.options.instructions, + audio: { + output: { voice: this.options.voice }, + }, + }; + // Tools are declared here; the backend already accepts them in + // session.update and emits response.function_call_arguments.done when the + // model decides to call one. Only include the keys when we actually have + // tools — the server's pydantic validator is strict about shapes. + if (this._tools.length) { + session.tools = this._tools; + session.tool_choice = "auto"; + } + this._send({ type: "session.update", session }); + } + + /** Update voice/instructions on a live session without tearing down. */ + /** @param {{ voice?: string; instructions?: string }} patch */ + updateSession(patch) { + /** @type {Record} */ + const session = { type: "realtime" }; + if (patch.instructions) session.instructions = patch.instructions; + if (patch.voice) session.audio = { output: { voice: patch.voice } }; + if (Object.keys(session).length > 1) { + this._send({ type: "session.update", session }); + } + } + + /** + * Replace the declared tool set on a live session (e.g. the user flipped a + * tool switch mid-conversation). Always sends `tools` — an empty array + * clears them — so toggling the last tool off actually removes it. + * @param {ToolDef[]} tools + */ + setTools(tools) { + this._tools = tools; + this._send({ + type: "session.update", + session: { type: "realtime", tools, tool_choice: tools.length ? "auto" : "none" }, + }); + } + + /** + * Return a tool's result to the model. Pairs with the `toolcall` event's + * `callId`. Caller follows this with `requestResponse()` so the model speaks. + * @param {string} callId + * @param {string} output Plain text / JSON string the model will read. + */ + sendToolOutput(callId, output) { + if (!callId) return; // Can't target a result without the call id. + this._send({ + type: "conversation.item.create", + item: { type: "function_call_output", call_id: callId, output }, + }); + } + + /** + * Add an image to the conversation as user content, so the vision-language + * model can see it (used by the camera tool). `dataUrl` is a + * `data:image/jpeg;base64,...` string. + * @param {string} dataUrl + */ + sendUserImage(dataUrl) { + this._send({ + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: dataUrl }], + }, + }); + } + + /** + * Ask the model to generate a response now (after feeding tool results). + * Serialized: if a response is already in flight we queue this request and + * replay it once the active response finishes, so we never trip the + * backend's `conversation_already_has_active_response` guard. + * + * @param {{ image?: string }} [opts] Optional `image` (a data URL) sent as a + * user `input_image` immediately before this response.create — so the frame + * travels with the create (and is deferred together with it if queued), + * rather than being added to the conversation eagerly. Used by the camera + * tool so the model sees the snapshot in the response it's about to speak. + */ + requestResponse(opts = {}) { + if (this._responseActive()) { + this._createQueue.push(opts); + if (this._debug) console.debug(`[ws] response.create queued (a response is active); pending=${this._createQueue.length}`); + return; + } + this._createResponseNow(opts); + } + + /** True while a response occupies the single backend slot. */ + _responseActive() { + return this._openResponses > 0 || this._createInFlight; + } + + /** Send a response.create immediately and arm the in-flight guard. Any image + * on the payload is added as user content right before the create. + * @param {{ image?: string }} [opts] */ + _createResponseNow(opts = {}) { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; + if (opts.image) this.sendUserImage(opts.image); + this._createInFlight = true; + this._send({ type: "response.create" }); + } + + /** Replay one queued response.create if the slot is now free. Called on every + * response.done, so queued creates drain one-per-completion. */ + _flushQueuedCreate() { + if (this._createQueue.length > 0 && !this._responseActive()) { + const opts = this._createQueue.shift(); + if (this._debug) console.debug(`[ws] replaying queued response.create; remaining=${this._createQueue.length}`); + this._createResponseNow(opts); + } + } + + /** @param {boolean} muted */ + setMuted(muted) { + this._muted = muted; + } + + /** + * Update the mic noise gate live (the user moved the Settings cursor). + * @param {NoiseGate} gate + */ + setNoiseGate(gate) { + this._noiseGate = gate; + this._captureNode?.port.postMessage({ kind: "gate", ...gate }); + } + + /** @param {Record} event */ + _send(event) { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; + this._ws.send(JSON.stringify(event)); + } + + async close() { + // Abort a queue wait in progress: flag it and wake the poll sleep so + // `_pollQueue` throws "aborted" and connect() unwinds cleanly. + this._closed = true; + if (this._queueWake) { + clearTimeout(this._queueTimer); + const wake = this._queueWake; + this._queueWake = null; + wake(); + } + if (this._joinTimer) { + clearTimeout(this._joinTimer); + this._joinTimer = 0; + } + if (this._joinReject) { + const reject = this._joinReject; + this._joinResolve = null; + this._joinReject = null; + reject(_codedError("join aborted", "aborted")); + } + this._visualiser?.stop(); + this._visualiser = null; + try { + if (this._ws && this._ws.readyState <= WebSocket.OPEN) { + this._ws.close(1000, "client closed"); + } + } catch { + // ignored + } + this._ws = null; + + try { + this._captureNode?.port.close?.(); + } catch { + // ignored + } + try { + this._micSrc?.disconnect(); + } catch { + // ignored + } + try { + this._captureNode?.disconnect(); + } catch { + // ignored + } + try { + this._micAnalyser?.disconnect(); + } catch { + // ignored + } + try { + this._outAnalyser?.disconnect(); + } catch { + // ignored + } + try { + this._playbackNode?.disconnect(); + } catch { + // ignored + } + try { + await this._ctx?.close(); + } catch { + // ignored + } + this._ctx = null; + this._captureNode = null; + this._playbackNode = null; + this._micSrc = null; + this._micAnalyser = null; + this._outAnalyser = null; + this._setStatus("closed"); + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5d88660 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,72 @@ +--- +services: + + llama: + image: ghcr.io/ggml-org/llama.cpp:server-cuda + command: + - -hf + - ggml-org/gemma-4-E4B-it-GGUF + - -np + - "2" + - -c + - "65536" + - -fa + - "on" + - --swa-full + - --host + - 0.0.0.0 + - --port + - "8080" + ports: + - 8080:8080/tcp + volumes: + - ./cache/:/root/.cache/ + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['0'] + capabilities: [gpu] + + pipeline: + depends_on: + - llama + build: + context: . + dockerfile: ${DOCKERFILE:-Dockerfile} + command: + - speech-to-speech + - --mode + - socket + - --recv_host + - 0.0.0.0 + - --send_host + - 0.0.0.0 + - --llm_backend + - responses-api + - --model_name + - ggml-org/gemma-4-E4B-it-GGUF + - --responses_api_base_url + - http://llama:8080/v1 + - --responses_api_api_key + - "" + - --init_chat_role + - system + - --init_chat_prompt + - "You are a helpful assistant" + expose: + - 12345/tcp + - 12346/tcp + ports: + - 12345:12345/tcp + - 12346:12346/tcp + volumes: + - ./cache/:/root/.cache/ + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['0'] + capabilities: [gpu] diff --git a/docs/PROJECT_OVERVIEW.es.md b/docs/PROJECT_OVERVIEW.es.md new file mode 100644 index 0000000..9fc09a4 --- /dev/null +++ b/docs/PROJECT_OVERVIEW.es.md @@ -0,0 +1,339 @@ +# speech-to-speech — Descripción del Proyecto + +> **Paquete**: `speech-to-speech` +> **Versión**: `0.2.11` +> **Autor**: Hugging Face +> **Licencia**: Apache-2.0 +> **Python**: 3.10 – 3.12 +> **Lema**: Pipeline Speech-to-Speech de baja latencia end-to-end para construir agentes de voz en tiempo real. + +--- + +## 1. ¿Qué es `speech-to-speech`? + +`speech-to-speech` es una **pipeline de audio modular y de baja latencia** que convierte la entrada hablada del usuario en una respuesta hablada encadenando cuatro etapas de IA: +**VAD → STT → LLM → TTS**. + +De fábrica expone un **endpoint WebSocket compatible con el protocolo OpenAI Realtime** (`ws://host:port/v1/realtime`), +por lo que cualquier navegador o SDK que sepa hablar el protocolo Realtime de OpenAI puede conectarse y empezar a tener +conversaciones naturales en cuestión de segundos — sin escribir una sola línea de código en el servidor. + +La pipeline está diseñada para ser **agnóstica del backend en cada etapa**: puedes intercambiar distintos modelos +de STT / LLM / TTS según el hardware disponible (NVIDIA CUDA, Apple Silicon MLX o solo CPU), los idiomas de tus usuarios +y el compromiso entre latencia y calidad que necesites. + +Un subsistema opcional integrado de **RAG (Retrieval Augmented Generation) del lado servidor** permite que las +respuestas del LLM se basen en tu propia base de conocimientos privada (documentos Markdown / TXT / JSONL), +mientras permanece 100 % transparente para el cliente. Las actualizaciones dinámicas de la base de conocimientos +se exponen en el mismo servidor HTTP mediante una API REST de 11 endpoints. + +--- + +## 2. Características principales + +| Característica | Descripción | +|---|---| +| **Protocolo de voz en tiempo real** | Soporte nativo del WebSocket OpenAI `v1/realtime`: `session.update`, `response.create`, llamadas a funciones, deltas de audio, interrupciones. Compatible directamente con el SDK Realtime oficial y con los "playgrounds" web. | +| **4 modos de ejecución** | `local` (micrófono → altavoces), `socket` (IPC TCP), `websocket` (WS audio crudo), `realtime` (protocolo OpenAI — predeterminado). | +| **6 backends de STT intercambiables** | Whisper / Whisper-MLX / MLX-Audio-Whisper / Faster-Whisper / Parakeet TDT (predeterminado) / Paraformer. | +| **4 backends de LLM intercambiables** | `transformers` (local) · `mlx-lm` (Apple Silicon) · `responses-api` (endpoint OpenAI con tool-calling) · `chat-completions` (cualquier servidor `/v1/chat/completions` compatible con OpenAI). | +| **5 backends de TTS intercambiables** | ChatTTS · Facebook MMS · Pocket (muy pequeño, CPU) · Kokoro · Qwen3-TTS (predeterminado, voz personalizada de 1.7B). | +| **Multiplataforma** | NVIDIA CUDA (Linux), Apple Silicon MLX + MPS (macOS), fallback CPU en cada etapa. | +| **Pool de N pipelines aisladas** | `--num_pipelines N` ejecuta N sesiones independientes en paralelo (cada una con sus propios VAD / STT / LLM / TTS y estado de conversación). Ideal para despliegues pequeños con múltiples conexiones simultáneas. | +| **VAD e interrupción integrados** | Detección de actividad vocal con umbral configurable; finalización del STT por silencio; el LLM se puede interrumpir mientras habla y se cancela de forma limpia mediante un `CancelScope`. | +| **Detección automática de idioma** | A través de `lingua-language-detector` para el idioma de la respuesta del asistente. | +| **Transcripción parcial en vivo** | Parakeet-TDT emite transcripciones parciales cada 500 ms para que los clientes muestren el texto "el usuario está hablando…". | +| **Soporte completo de tool-calling** | Con el backend `responses-api`: entrega en streaming de argumentos de función, tool calls paralelos, voces personalizadas sobre TTS — todo conforme a la superficie de la Responses API de OpenAI. | +| **Inyección RAG opcional en el servidor** | Hook transparente de recuperación en cada turno del LLM. Embeddings con Sentence-Transformers, persistencia del índice NPZ, top-k / umbral coseno configurables, inyección como mensaje del sistema o del usuario. Multilingüe por defecto (español / italiano / inglés listos para usar). | +| **API REST para base de conocimientos dinámica** | 11 endpoints para listar, buscar, hacer upsert, actualizar, eliminar y recargar contenidos de la KB en tiempo real — con persistencia entre reinicios mediante `kb/_dynamic.jsonl`. | +| **Configuración estructurada CLI / JSON** | Cada parámetro es un argumento `@dataclass` de HfArgumentParser con valores predeterminados razonables, o un fichero JSON de configuración completo que puedes pasar como único argumento. | +| **Empaquetado y flujo de publicación PyPI** | Configurado `uv build` + `twine check` + GitHub Actions (la etiqueta `vX.Y.Z` activa la subida). | + +--- + +## 3. Arquitectura a vista de pájaro + +Un turno único de conversación en modo **realtime** se ve así: + +``` + ┌─────────────────────────────────────────────────────────┐ + │ uvicorn + FastAPI │ + │ ┌───────────────────────────────────────────────────┐ │ + Mic / Navegador ──► WS │ │ RealtimeService (sesión + rutas + eventos) │ │ + (audio entradas + │ │ └───────────────────────────────────────────────────┘ │ + eventos) │ │ │ + │ │ Pool de pipelines ─► PipelineUnit #1 ─► PipelineUnit #N│ + │ └─────────────────────────────────────────────────────────┘ + │ │ + ▼ ▼ + ┌──────────────────────────────────────────────┐ + │ Una unidad de pipeline (una por usuario) │ + │ │ + │ 1. VAD ──► inicio / fin voz │ + │ 2. STT (5 sabores) ──► último texto usuario│ + │ │ │ + │ ▼ │ + │ ┌────────────────────────┐ │ + │ │ HOOK BÚSQUEDA RAG │ ◄─── kb/_index.npz + │ │ (inyecta solo si ≥ N) │ + md/txt/jsonl + │ └────────────────────────┘ + _dynamic.jsonl + │ │ │ + │ ▼ │ + │ 3. LLM (4 sabores) ──► texto + tool calls│ + │ │ │ + │ ▼ │ + │ 4. TTS (5 sabores) ──► PCM audio streaming│ + └──────────────────────────────────────────────┘ + │ + ▼ + ◄── WS audio / delta events +``` + +Invariantes clave: + +- **Cada conexión = una `PipelineUnit`** (asignada de forma atómica; colas y estado aislados; si las N unidades están ocupadas, la conexión N+1 se rechaza). +- **La etapa LLM nunca ve audio crudo**. La pipeline solo envía buffers de texto al manejador del LLM, por lo que cambiar de proveedor o de modelo sigue siendo transparente. +- **La etapa RAG es un efecto lateral puro sobre el búfer de texto**: lee el texto más reciente del usuario, ejecuta un embedding + top-k por coseno + filtro por umbral, y antepone el conocimiento relevante como un mensaje adicional del sistema (o del usuario) — con una línea de registro clara para ver siempre exactamente qué se inyectó. +- **TTS en streaming** (todos los backends modernos): los deltas de audio se emiten a medida que se sintetizan, por lo que el primer byte de una respuesta llega al altavoz mucho antes de que el LLM termine de generar el texto. + +--- + +## 4. Modos de ejecución (`--mode`) + +Configurables mediante [ModuleArguments.mode](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/module_arguments.py#L11-L16) +(predeterminado: `realtime`). + +| Modo | Entrada | Salida | Ideal para | +|---|---|---|---| +| `local` | Micrófono local vía `sounddevice` / `miniaudio` | Altavoces locales | Demos de escritorio, prototipado rápido en portátil. | +| `socket` | Chunks PCM crudos en socket TCP | PCM crudos por socket TCP | Sistemas legacy / embebidos, transporte personalizado. | +| `websocket` | PCM crudo sobre endpoint WS | PCM crudo sobre WS | Frontend a medida mínimo que solo envía audio. | +| `realtime` | WS con protocolo OpenAI Realtime (`/v1/realtime`) | Mismo protocolo + API REST RAG en el mismo servidor HTTP | **Producción e integración con SDK.** Todos los clientes que soportan la Realtime API (OpenAI SDK, playgrounds web, wrappers Swift / Kotlin / JS) se conectan aquí. | + +--- + +## 5. Backends soportados + +### 5.1 STT — Speech to Text + +| Nombre `--stt` | Familia de modelos | Modelo predeterminado | Ventajas hardware | +|---|---|---|---| +| `whisper` | HuggingFace Transformers Whisper | `distil-whisper/distil-large-v3` | CUDA / CPU | +| `whisper-mlx` | MLX Whisper | `mlx-community/whisper-large-v3-turbo` | Apple Silicon (macOS) | +| `mlx-audio-whisper` | Apple `mlx-audio` | `mlx-community/whisper-large-v3-turbo` | Apple Silicon | +| `faster-whisper` | Whisper cuantizada CTranslate2 | `tiny.en` | CPU con latencia muy baja | +| `parakeet-tdt` | **(predeterminado)** HuggingFace Parakeet TDT | `parakeet-tdt-1.1b` | Streaming amigable + **transcripción parcial en vivo** sobre CUDA | +| `paraformer` | Alibaba Paraformer | `paraformer-zh` | Despliegues solo chino | + +### 5.2 LLM — Modelo de lenguaje + +| `--llm_backend` | Descripción | Modelo predeterminado | +|---|---|---| +| `transformers` | Generación local HF Transformers en proceso | `Qwen/Qwen3-4B-Instruct-2507` | +| `mlx-lm` | Inferencia local cuantizada 4-bit / 8-bit en Apple Silicon | `mlx-community/...` | +| `responses-api` | **(predeterminado)** Endpoint remoto compatible con OpenAI Responses API (superficie completa de tool-calling + argumentos de función en streaming) | `gpt-5.4-mini` | +| `chat-completions` | Cualquier remoto `/v1/chat/completions` compatible con OpenAI — enchufa vLLM, TGI, Ollama, servidor llama.cpp, SGLang, TabbyAPI, etc. | (auto-detecta vía `base_url + /v1/models`) | + +### 5.3 TTS — Text to Speech + +| Nombre `--tts` | Backend | Modelo predeterminado | Puntos fuertes | +|---|---|---|---| +| `chatTTS` | ChatTTS (grupo opcional `chattts`) | — | Muy conversacional, inglés + chino, prosodia muy expresiva | +| `facebookMMS` | Facebook MMS (grupo opcional `facebook-mms`) | `facebook/mms-tts-eng` | Ultraligero, cubre más de 1.100 idiomas | +| `pocket` | PocketTTS (pequeño CPU) | — | Sin dependencias en dispositivo, ideal para embebidos / poca RAM | +| `kokoro` | Kokoro TTS (grupo opcional `kokoro`) | — | Estado del arte de TTS neural en inglés y japonés | +| `qwen3` | **(predeterminado)** Qwen3-TTS vía GGML `faster-qwen3-tts` | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Multilingüe, soporta clonación por voz de referencia, tasa de token 12 Hz → latencia ultrabaja. | + +Todos los backends de TTS se acceden a través de una interfaz de streaming común, por lo que el resto de la pipeline es agnóstica al backend. + +--- + +## 6. RAG del lado servidor (complemento opcional) + +El subsistema RAG está completamente documentado en: + +- 🌍 Guía lógica IT/EN: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md) +- 🇪🇸 Traducción al español: [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md) + +En resumen: + +``` +$ ./start_pipeline_rag.sh # arranca pipeline + API REST RAG +RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/.../kb +RAG: Índice cargado desde disco: 9 chunk (shape=(9, 384)). +RAG: Activo. kb=... top_k=3 umbral=0.250 inject_as=system idioma=es +RAG HTTP API montata su prefix='/v1/rag' +``` + +Puntos clave: + +- **Cero cambios en el cliente** — la inyección ocurre en el servidor, dentro del búfer de chat del LLM, antes de cada turno. +- **Modelos de embedding intercambiables** con un modelo multilingüe predeterminado (`paraphrase-multilingual-MiniLM-L12-v2`, 384 dims) y alternativas recomendadas para español-only y para bases de conocimientos muy grandes. +- **Persistencia del índice NPZ** — se reconstruye solo si el contenido cambió. Soporte de `--rag_force_rebuild`. +- **Tres formatos de fuente**: Markdown / TXT con chunking recursivo automático (tamaño + solapamiento configurables), o JSONL para chunks artesanales. +- **Comportamiento de recuperación configurable**: `--rag_top_k`, `--rag_threshold`, `--rag_inject_as (system/user)`, `--rag_language (es/it/en)`. +- **API REST dinámica thread-safe** (11 endpoints): `/status`, `/sources`, `/chunks`, `/chunks/list`, `/chunks/update`, `/upsert/document`, `/search`, `/add/document`, `/add/chunks`, `/remove`, `/reload`. +- **Persistencia entre reinicios**: las llamadas a la API con `persist=true` se adjuntan atómicamente a `kb/_dynamic.jsonl` y se re-indexan automáticamente en el siguiente arranque. +- **Patrón CRUD idempotente**: `/upsert/document` gestiona crear / reemplazar / eliminar (`text=""`) de forma atómica para que los clientes solo necesiten una URL. +- **Falla con elegancia**: cualquier excepción de recuperación se registra pero nunca rompe el turno de conversación. + +--- + +## 7. Arranque rápido + +### 7.1 Instalación (instalación editable estilo PyPI) + +```bash +git clone https://github.com/huggingface/speech-to-speech.git +cd speech-to-speech + +# Paquete base (incluye STT parakeet-tdt + TTS qwen3 por defecto) +uv pip install -e . + +# Opcional: añade el subsistema RAG para recuperación en la KB +uv pip install -e ".[rag]" + +# Opcional: selecciona los extras de TTS / STT que quieras +uv pip install -e ".[rag,chattts,kokoro,faster-whisper]" +``` + +### 7.2 Arrancar en modo Realtime (predeterminado) con un LLM externo + +Esta es la configuración más común: el LLM corre en un servidor remoto compatible con OpenAI +(p.ej. `http://127.0.0.1:8001/v1` con vLLM o TGI), STT + TTS corren localmente sobre CUDA / MLX. + +Crea un pequeño *wrapper* de shell (véase [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) para la plantilla completa): + +```bash +#!/bin/bash +set -euo pipefail + +LLM_BASE_URL="${LLM_BASE_URL:-http://127.0.0.1:8001/v1}" +LLM_MODEL="${LLM_MODEL:-Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4}" +LLM_API_KEY="${LLM_API_KEY:-placeholder}" +TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-Base}" +STT_MODEL="${STT_MODEL:-distil-whisper/distil-large-v3}" + +cd /home/azurian/speech-to-speech + +uv run speech-to-speech \ + --mode realtime \ + --ws_host 0.0.0.0 \ + --ws_port 12345 \ + --num_pipelines 2 \ + \ + --llm_backend chat-completions \ + --chat_completions_handler_base_url "$LLM_BASE_URL" \ + --chat_completions_handler_model_name "$LLM_MODEL" \ + --chat_completions_handler_api_key "$LLM_API_KEY" \ + \ + --stt whisper \ + --whisper_stt_model_name "$STT_MODEL" \ + \ + --tts qwen3 \ + --qwen3_tts_model_name "$TTS_MODEL" +``` + +Ejecútalo y después conecta cualquier cliente Realtime de OpenAI a: + +``` +ws://:12345/v1/realtime +``` + +### 7.3 Misma pipeline + RAG activado + +Añade estos flags (véase §6 para la referencia completa y el script auxiliar [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh)): + +```bash + --rag_enabled \ + --rag_kb_path ./kb \ + --rag_top_k 3 \ + --rag_threshold 0.25 \ + --rag_language es \ + --rag_inject_as system +``` + +La API REST de RAG aparece inmediatamente en el mismo servidor HTTP: + +```bash +curl http://127.0.0.1:12345/v1/rag/status | jq +``` + +--- + +## 8. Estructura del proyecto (puntos destacados) + +``` +speech-to-speech/ +├── pyproject.toml ← metadatos del paquete + grupos de deps opcionales +├── LICENSE ← Apache-2.0 +├── start_pipeline.sh ← script de arranque de referencia +├── start_pipeline_rag.sh ← script de arranque con RAG activado +│ +├── src/speech_to_speech/ +│ ├── s2s_pipeline.py ← punto de entrada (main), constructor de pipeline, pool realtime +│ ├── baseHandler.py +│ ├── chat.py +│ ├── pipeline/ ← tipos de cola, CancelScope, tipos de handler +│ │ +│ ├── arguments_classes/ ← args HfArgumentParser @dataclass (uno por handler) +│ │ ├── module_arguments.py ← mode / stt / tts / llm_backend / num_pipelines +│ │ ├── rag_arguments.py ← 12 flags específicos de RAG +│ │ └── ... (Whisper, Qwen3, ChatTTS, VAD, …) +│ │ +│ ├── STT/ ← seis manejadores STT +│ ├── TTS/ ← cinco manejadores TTS +│ ├── LLM/ +│ │ ├── base_openai_compatible_language_model.py ← Hook de inyección RAG + tool-calling completo +│ │ ├── chat_completions_language_model.py +│ │ ├── responses_api_language_model.py +│ │ └── language_model.py ← manejadores locales transformers / mlx-lm +│ │ +│ ├── RAG/ +│ │ ├── retriever.py ← singleton, embeddings, NPZ, search, CRUD completo +│ │ └── router.py ← 11 endpoints FastAPI /v1/rag +│ │ +│ └── api/openai_realtime/ +│ ├── websocket_router.py ← FastAPI + Realtime + montaje condicional RAG +│ ├── service.py ← Bucle de eventos Realtime + rutas de sesión / handlers +│ └── ... +│ +├── kb/ +│ ├── 01_faq_producto.md ← ejemplo FAQ en español (incluido en scaffold) +│ ├── 02_politicas_internas.md ← ejemplo políticas en español +│ ├── README.md ← guía de formatos +│ └── _dynamic.jsonl ← chunks añadidos por API (auto-generado) +│ +└── docs/ + ├── RAG_SERVER_SIDE.md ← Guía RAG en profundidad IT/EN (más de 600 líneas) + └── RAG_SERVER_SIDE.es.md ← Traducción al español +``` + +--- + +## 9. Versionado y publicación + +El repositorio trae una pipeline completa de lanzamiento a PyPI en +`.github/workflows/publish.yml`: + +1. Sube la `version` en [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L7) + y `__version__` en [src/speech_to_speech/__init__.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/__init__.py). +2. Fusiona una PR de release que contenga solo esos dos cambios. +3. Etiqueta y sube: + ```bash + git checkout main && git pull origin main + git tag -a vX.Y.Z -m "Release vX.Y.Z" + git push origin vX.Y.Z + ``` +4. El workflow ejecuta automáticamente `uv build` + `twine check --strict` + subida a PyPI. + +Consulta [AGENTS.md](file:///home/azurian/speech-to-speech/AGENTS.md) para las reglas de release a nivel de repositorio. + +--- + +## 10. Siguientes pasos + +- Empieza con los scripts de arranque: [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) (base) y [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) (con RAG). +- Profundiza en el subsistema RAG: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md) / [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md). +- Ajusta los manejadores por etapa leyendo las clases de argumentos en [arguments_classes](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes) — cada flag tiene su ayuda en línea. +- Construye tu propio cliente Realtime apoyándote en [service.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/service.py) y [websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py). diff --git a/docs/PROJECT_OVERVIEW.md b/docs/PROJECT_OVERVIEW.md new file mode 100644 index 0000000..8591a93 --- /dev/null +++ b/docs/PROJECT_OVERVIEW.md @@ -0,0 +1,337 @@ +# speech-to-speech — Project Overview + +> **Package**: `speech-to-speech` +> **Version**: `0.2.11` +> **Author**: Hugging Face +> **License**: Apache-2.0 +> **Python**: 3.10 – 3.12 +> **Tagline**: Low-latency end-to-end Speech-to-Speech pipeline for building realtime voice agents. + +--- + +## 1. What is `speech-to-speech`? + +`speech-to-speech` is a fully modular, **low-latency audio pipeline** that turns a user's spoken input into a spoken reply by chaining four AI stages together: +**VAD → STT → LLM → TTS**. + +Out of the box it exposes an **OpenAI-compatible Realtime WebSocket endpoint** (`ws://host:port/v1/realtime`), +so any browser or SDK that knows how to speak the OpenAI Realtime Protocol can connect and start having natural voice conversations in seconds — without writing a single line of server code. + +The pipeline is designed to be **backend-agnostic at every stage**: you can plug different STT / LLM / TTS models +depending on the available hardware (NVIDIA CUDA, Apple Silicon MLX, or CPU-only), the language(s) of your users, +and the latency / quality trade-off you need. + +A built-in, optional **RAG (Retrieval Augmented Generation) server-side subsystem** lets the LLM answers be grounded +in your own private knowledge base (Markdown / TXT / JSONL documents), while staying 100 % transparent for the client. +Dynamic knowledge base updates are exposed over the same HTTP server via a 11-endpoint REST API. + +--- + +## 2. Key Features + +| Feature | Description | +|---|---| +| **Real-time voice protocol** | Native OpenAI `v1/realtime` WebSocket support: `session.update`, `response.create`, function calling, audio deltas, interruptions. Drop-in compatible with the official Realtime SDK and browser playgrounds. | +| **4 execution modes** | `local` (mic → speakers), `socket` (TCP IPC), `websocket` (raw audio WS), `realtime` (OpenAI protocol — default). | +| **6 pluggable STT backends** | Whisper / Whisper-MLX / MLX-Audio-Whisper / Faster-Whisper / Parakeet TDT (default) / Paraformer. | +| **4 pluggable LLM backends** | `transformers` (local) · `mlx-lm` (Apple Silicon) · `responses-api` (OpenAI tool-calling aware endpoint) · `chat-completions` (any OpenAI-compatible `/v1/chat/completions` server). | +| **5 pluggable TTS backends** | ChatTTS · Facebook MMS · Pocket (tiny CPU) · Kokoro · Qwen3-TTS (default, custom voice 1.7B). | +| **Multi-hardware ready** | NVIDIA CUDA (Linux), Apple Silicon MLX + MPS (macOS), CPU fallback for every stage. | +| **Pool of N isolated pipelines** | `--num_pipelines N` runs N independent sessions in parallel (each one with its own VAD / STT / LLM / TTS handlers and conversation state). Ideal for small multi-concurrent deployments. | +| **Built-in VAD & interruption** | Voice-activity detection with configurable threshold and streaming silence → STT finalization; the LLM can be interrupted mid-speech by the user and cancelled cleanly through a `CancelScope`. | +| **Automatic language detection** | Via `lingua-language-detector` for the assistant reply language. | +| **Live partial transcription** | Parakeet-TDT emits live partial transcriptions every 500 ms so clients can display "user is speaking …" text. | +| **Full tool-calling support** | With `responses-api` backend: streamed function argument delivery, parallel tool calls, custom voices over TTS — everything compliant with the OpenAI Responses API surface. | +| **Optional RAG server-side injection** | Transparent retrieval hook on every LLM turn. Sentence-Transformers embeddings, NPZ-index persistence, configurable top-k / cosine threshold, system or user-message injection. Multilingual by default (Spanish / Italian / English out of the box). | +| **Dynamic Knowledge Base REST API** | 11 endpoints to list, search, upsert, update, remove, and reload KB contents at runtime — including cross-restart persistence through `kb/_dynamic.jsonl`. | +| **Structured CLI / JSON configuration** | Every parameter is a `@dataclass` HfArgumentParser argument with sane defaults, or a full JSON config file you can pass in as single argument. | +| **PyPI-ready packaging & release flow** | Configured `uv build` + `twine check` + GitHub Actions publish workflow (tag `vX.Y.Z` triggers upload). | + +--- + +## 3. Architecture at a glance + +A single conversation turn in **realtime** mode looks like this: + +``` + ┌─────────────────────────────────────────────────────────┐ + │ uvicorn + FastAPI │ + │ ┌───────────────────────────────────────────────────┐ │ + Mic / Browser ──► WS │ │ RealtimeService (session + routing + events) │ │ + (audio in + │ │ └───────────────────────────────────────────────────┘ │ + events) │ │ │ + │ │ Pipeline pool ──► PipelineUnit #1 ──► PipelineUnit #N│ + │ └─────────────────────────────────────────────────────────┘ + │ │ + ▼ ▼ + ┌──────────────────────────────────────────────┐ + │ Single pipeline unit (one per user) │ + │ │ + │ 1. VAD ──► speech start / end │ + │ 2. STT (5 flavours) ──► last user text │ + │ │ │ + │ ▼ │ + │ ┌────────────────────────┐ │ + │ │ RAG SEARCH HOOK │ ◄─── kb/_index.npz + │ │ (inject only if ≥ N) │ + md/txt/jsonl + │ └────────────────────────┘ + _dynamic.jsonl + │ │ │ + │ ▼ │ + │ 3. LLM (4 flavours) ──► text reply + tools│ + │ │ │ + │ ▼ │ + │ 4. TTS (5 flavours) ──► streamed PCM audio│ + └──────────────────────────────────────────────┘ + │ + ▼ + ◄── WS audio / delta events +``` + +Key invariants: + +- **Each connection = one `PipelineUnit`** (allocated atomically; queues and state are isolated; if N units are busy, the `N+1`-th connection is rejected). +- **The LLM stage never sees raw audio**. The pipeline only forwards text buffers to the LLM handler, so model vendors and backend switches remain transparent. +- **The RAG stage is a pure side-effect on the text buffer**: it reads the most recent user text, runs an embedding + cosine top-k + threshold filter, and prepends matching knowledge as an extra system (or user) message — with a clear log line so you can always see exactly what was injected. +- **TTS streams** (all modern backends): audio deltas are emitted as they are synthesised, so the first byte of a reply reaches the speaker well before the LLM has finished generating text. + +--- + +## 4. Execution modes (`--mode`) + +Configurable via [ModuleArguments.mode](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/module_arguments.py#L11-L16) +(default: `realtime`). + +| Mode | Input | Output | Best for | +|---|---|---|---| +| `local` | Local microphone via `sounddevice` / `miniaudio` | Local speakers | Desktop demos, quick prototyping on laptop. | +| `socket` | Raw PCM chunks on TCP socket | Raw PCM on TCP socket | Legacy / embedded setups, custom transport. | +| `websocket` | Raw PCM over WS endpoint | Raw PCM over WS | Minimal custom frontend that just pushes audio. | +| `realtime` | OpenAI Realtime Protocol WS (`/v1/realtime`) | Same protocol + RAG REST API on the same HTTP server | **Production & SDK integration.** All clients that support the Realtime API (OpenAI SDK, web playgrounds, Swift / Kotlin / JS wrappers) connect here. | + +--- + +## 5. Supported backends + +### 5.1 STT — Speech to Text + +| `--stt` name | Model family | Default model | Device highlights | +|---|---|---|---| +| `whisper` | HuggingFace Transformers Whisper | `distil-whisper/distil-large-v3` | CUDA / CPU | +| `whisper-mlx` | MLX Whisper | `mlx-community/whisper-large-v3-turbo` | Apple Silicon (macOS) | +| `mlx-audio-whisper` | Apple `mlx-audio` | `mlx-community/whisper-large-v3-turbo` | Apple Silicon | +| `faster-whisper` | CTranslate2 Quantized Whisper | `tiny.en` | Very low latency CPU | +| `parakeet-tdt` | **(default)** HuggingFace Parakeet TDT | `parakeet-tdt-1.1b` | Streaming-friendly + **live partial transcription** on CUDA | +| `paraformer` | Alibaba Paraformer | `paraformer-zh` | Chinese-only deployments | + +### 5.2 LLM — Language Model + +| `--llm_backend` | Description | Default model | +|---|---|---| +| `transformers` | Local, process-bound HF Transformers generation | `Qwen/Qwen3-4B-Instruct-2507` | +| `mlx-lm` | Apple Silicon local 4-bit / 8-bit quantised inference | `mlx-community/...` | +| `responses-api` | **(default)** OpenAI Responses-API compatible remote endpoint (full tool-calling surface + streamed function args) | `gpt-5.4-mini` | +| `chat-completions` | Any OpenAI-compatible `/v1/chat/completions` remote — plug in vLLM, TGI, Ollama, llama.cpp server, SGLang, TabbyAPI, etc. | (auto-detects via `base_url + /v1/models`) | + +### 5.3 TTS — Text to Speech + +| `--tts` name | Backend | Default model | Strengths | +|---|---|---|---| +| `chatTTS` | ChatTTS (optional group `chattts`) | — | Highly conversational, English + Chinese, very expressive prosody | +| `facebookMMS` | Facebook MMS (optional group `facebook-mms`) | `facebook/mms-tts-eng` | Ultra-lightweight, covers 1,100+ languages | +| `pocket` | PocketTTS (small CPU) | — | Zero-dependency on-device, great for embedded / low-RAM | +| `kokoro` | Kokoro TTS (optional group `kokoro`) | — | State of the art English & Japanese neural TTS | +| `qwen3` | **(default)** Qwen3-TTS via `faster-qwen3-tts` GGML | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Multilingual, supports reference voice cloning, 12 Hz token rate → ultra-low latency. | + +All TTS backends are reachable through a common streaming interface so the rest of the pipeline is backend-agnostic. + +--- + +## 6. RAG Server-Side (optional add-on) + +The RAG subsystem is fully documented in: + +- 🌍 English / Spanish logic: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md) +- 🇪🇸 Spanish translation: [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md) + +At a glance: + +``` +$ ./start_pipeline_rag.sh # starts pipeline + RAG REST API +RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/.../kb +RAG: Índice cargado desde disco: 9 chunk (shape=(9, 384)). +RAG: Activo. kb=... top_k=3 umbral=0.250 inject_as=system idioma=es +RAG HTTP API montata su prefix='/v1/rag' +``` + +Highlights: + +- **Zero client changes** — injection happens server-side in the LLM chat buffer before every turn. +- **Pluggable embedding models** with a default multilingual model (`paraphrase-multilingual-MiniLM-L12-v2`, 384 dims) and recommended alternatives for Spanish-only and very-large-KB scenarios. +- **NPZ index persistence** — rebuilds only if content changed. Supports `--rag_force_rebuild`. +- **Three source formats**: Markdown / TXT with automatic recursive chunking (size + overlap configurable), or JSONL for hand-crafted chunks. +- **Configurable retrieval behaviour**: `--rag_top_k`, `--rag_threshold`, `--rag_inject_as (system/user)`, `--rag_language (es/it/en)`. +- **Thread-safe dynamic REST API** (11 endpoints): `/status`, `/sources`, `/chunks`, `/chunks/list`, `/chunks/update`, `/upsert/document`, `/search`, `/add/document`, `/add/chunks`, `/remove`, `/reload`. +- **Cross-restart persistence**: API calls with `persist=true` are appended atomically to `kb/_dynamic.jsonl` and automatically re-indexed on the next startup. +- **Idempotent CRUD pattern**: `/upsert/document` handles create / replace / delete (`text=""`) atomically so clients only need one URL. +- **Fails gracefully**: any retrieval exception is logged but never breaks the conversation turn. + +--- + +## 7. Quick start + +### 7.1 Install (PyPI-style editable install) + +```bash +git clone https://github.com/huggingface/speech-to-speech.git +cd speech-to-speech + +# Base package (includes default parakeet-tdt STT + qwen3 TTS) +uv pip install -e . + +# Optional: add the RAG subsystem for server-side KB retrieval +uv pip install -e ".[rag]" + +# Optional: select which TTS / STT extras you want to bring along +uv pip install -e ".[rag,chattts,kokoro,faster-whisper]" +``` + +### 7.2 Launch in Realtime mode (default) using an external LLM + +This is the most common configuration: LLM runs on a remote OpenAI-compatible server +(e.g. `http://127.0.0.1:8001/v1` with vLLM or TGI), STT + TTS run locally on CUDA / MLX. + +Create a small shell wrapper (see [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) for the full template): + +```bash +#!/bin/bash +set -euo pipefail + +LLM_BASE_URL="${LLM_BASE_URL:-http://127.0.0.1:8001/v1}" +LLM_MODEL="${LLM_MODEL:-Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4}" +LLM_API_KEY="${LLM_API_KEY:-placeholder}" +TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-Base}" +STT_MODEL="${STT_MODEL:-distil-whisper/distil-large-v3}" + +cd /home/azurian/speech-to-speech + +uv run speech-to-speech \ + --mode realtime \ + --ws_host 0.0.0.0 \ + --ws_port 12345 \ + --num_pipelines 2 \ + \ + --llm_backend chat-completions \ + --chat_completions_handler_base_url "$LLM_BASE_URL" \ + --chat_completions_handler_model_name "$LLM_MODEL" \ + --chat_completions_handler_api_key "$LLM_API_KEY" \ + \ + --stt whisper \ + --whisper_stt_model_name "$STT_MODEL" \ + \ + --tts qwen3 \ + --qwen3_tts_model_name "$TTS_MODEL" +``` + +Run it, then connect any OpenAI Realtime client to: + +``` +ws://:12345/v1/realtime +``` + +### 7.3 Same pipeline + RAG enabled + +Append these flags (see §6 for the full reference and the helper script [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh)): + +```bash + --rag_enabled \ + --rag_kb_path ./kb \ + --rag_top_k 3 \ + --rag_threshold 0.25 \ + --rag_language es \ + --rag_inject_as system +``` + +The RAG REST API appears immediately on the same HTTP server: + +```bash +curl http://127.0.0.1:12345/v1/rag/status | jq +``` + +--- + +## 8. Project layout (highlights) + +``` +speech-to-speech/ +├── pyproject.toml ← package metadata + optional deps groups +├── LICENSE ← Apache-2.0 +├── start_pipeline.sh ← reference launch script +├── start_pipeline_rag.sh ← launch script + RAG enabled +│ +├── src/speech_to_speech/ +│ ├── s2s_pipeline.py ← entry point (main), pipeline builder, realtime pool +│ ├── baseHandler.py +│ ├── chat.py +│ ├── pipeline/ ← queue types, CancelScope, handler types +│ │ +│ ├── arguments_classes/ ← @dataclass HfArgumentParser args (one per handler) +│ │ ├── module_arguments.py ← mode / stt / tts / llm_backend / num_pipelines +│ │ ├── rag_arguments.py ← 12 RAG-specific flags +│ │ └── ... (Whisper, Qwen3, ChatTTS, VAD, …) +│ │ +│ ├── STT/ ← six STT handlers +│ ├── TTS/ ← five TTS handlers +│ ├── LLM/ +│ │ ├── base_openai_compatible_language_model.py ← RAG injection hook + full tool-calling +│ │ ├── chat_completions_language_model.py +│ │ ├── responses_api_language_model.py +│ │ └── language_model.py ← local transformers / mlx-lm handlers +│ │ +│ ├── RAG/ +│ │ ├── retriever.py ← singleton, embeddings, NPZ, search, full CRUD +│ │ └── router.py ← 11 FastAPI /v1/rag endpoints +│ │ +│ └── api/openai_realtime/ +│ ├── websocket_router.py ← FastAPI + Realtime + conditional RAG mount +│ ├── service.py ← Realtime event loop + session/handler routing +│ └── ... +│ +├── kb/ +│ ├── 01_faq_producto.md ← example Spanish FAQ (included in scaffold) +│ ├── 02_politicas_internas.md ← example Spanish policies +│ ├── README.md ← format guide +│ └── _dynamic.jsonl ← API-added chunks (auto-generated) +│ +└── docs/ + ├── RAG_SERVER_SIDE.md ← Italian/English in-depth RAG guide (600+ lines) + └── RAG_SERVER_SIDE.es.md ← Spanish translation +``` + +--- + +## 9. Versioning and publishing + +The repository ships with a full PyPI release pipeline in +`.github/workflows/publish.yml`: + +1. Bump `version` in [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L7) + and `__version__` in [src/speech_to_speech/__init__.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/__init__.py). +2. Merge a release PR containing only those two changes. +3. Tag and push: + ```bash + git checkout main && git pull origin main + git tag -a vX.Y.Z -m "Release vX.Y.Z" + git push origin vX.Y.Z + ``` +4. The workflow runs `uv build` + `twine check --strict` + PyPI upload automatically. + +See [AGENTS.md](file:///home/azurian/speech-to-speech/AGENTS.md) for the repository-level release rules. + +--- + +## 10. Where to go next + +- Start with the launch scripts: [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) (base) and [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) (RAG enabled). +- Dive into the RAG subsystem: [RAG_SERVER_SIDE.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.md) / [RAG_SERVER_SIDE.es.md](file:///home/azurian/speech-to-speech/docs/RAG_SERVER_SIDE.es.md). +- Tune the per-stage handlers by reading the argument classes in [arguments_classes](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes) — every flag has inline help. +- Build your own Realtime client against [service.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/service.py) and [websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py). diff --git a/docs/RAG_SERVER_SIDE.es.md b/docs/RAG_SERVER_SIDE.es.md new file mode 100644 index 0000000..4435ea1 --- /dev/null +++ b/docs/RAG_SERVER_SIDE.es.md @@ -0,0 +1,678 @@ +# RAG Server-Side — Guía Oficial + +> **Funcionalidad**: Retrieval Augmented Generation integrado en la pipeline speech-to-speech. +> **Enfoque**: 2 — Inyección transparente en el lado servidor (sin cambios en el cliente). +> **Versión mínima speech-to-speech**: `0.2.11` + +--- + +## 1. Visión general + +El RAG server-side enriquece **automáticamente** cada respuesta del LLM con fragmentos relevantes extraídos de una base de conocimientos local. Todo el ciclo ocurre de forma invisible para el cliente Realtime: + +``` +Usuario habla → STT → 🟡 RAG RETRIEVAL (hook interno) → LLM → TTS → Audio al usuario + ↓ + kb/*.md, kb/*.txt, kb/*.jsonl + ↓ + top-k chunk inyectados en el prompt +``` + +### Ventajas +- ✅ **Cero cambios en el cliente** — funciona con cualquier SDK Realtime +- ✅ **Singleton global compartido** (1 sola copia de embeddings para N pipelines paralelas) +- ✅ **Persistencia del índice NPZ** — reconstrucción solo si los documentos cambian +- ✅ **100% compatible con los logs existentes** (`LLM REQUEST PROMPT` incluye los chunk inyectados) +- ✅ **Soporte multilingüe** (ES/IT/EN configurable) + +--- + +## 2. Instalación de dependencias + +Las dependencias RAG son opcionales (grupo `rag` en [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94)): + +```bash +cd /home/azurian/speech-to-speech +uv pip install -e ".[rag]" +``` + +**Contenido del grupo**: +- `sentence-transformers>=3.0.0` (arrastra automáticamente `torch`, `numpy`, `transformers` que ya están presentes) + +> ✅ **En DGX Spark GB10**: el modelo de embedding se carga nativamente sobre CUDA (detectado por `--rag_device auto`, valor por defecto). + +--- + +## 3. Estructura de la base de conocimientos + +La KB reside en la carpeta configurada mediante el parámetro `--rag_kb_path` (por defecto: `./kb`). + +``` +kb/ +├── README.md ← instrucciones del scaffold (auto-generado) +├── 01_faq_producto.md ← ejemplo español incluido +├── 02_politicas_internas.md ← ejemplo español incluido +│ +├── manual/ ← subcarpetas soportadas +│ ├── 01_instalacion.md +│ └── 02_facturacion.txt +│ +├── datos/ +│ └── clientes.jsonl ← formato pre-chunkizado +│ +├── _index.npz ← ⚙️ índice generado (NO modificar) +├── _chunks.jsonl ← ⚙️ catálogo chunk (NO modificar) +└── _dynamic.jsonl ← ⚙️ chunk dinámicos persistidos vía API +``` + +### Formatos soportados + +#### A. Archivos Markdown / TXT (recomendado, esfuerzo cero) + +Cualquier `*.md` o `*.txt` en la carpeta o subcarpetas se procesa así: +1. Lectura en UTF-8 (con fallback latin-1 y sustitución de errores) +2. División automática en chunk: + - **tamaño de chunk** por defecto: `512` caracteres (parámetro `--rag_chunk_size`) + - **overlap** por defecto: `64` caracteres (parámetro `--rag_chunk_overlap`) + - Algoritmo: splitter recursivo con separadores `\n\n → \n → . ? ! ; , → espacio → carácter` +3. Cada chunk recibe `source = path_relativo#chunk_index` + +#### B. JSONL pre-chunkizado (control total) + +Si prefieres gestionar chunk y metadatos manualmente (ej. extracción PDF estructurada), crea `*.jsonl` con **una línea por chunk**: + +```jsonl +{"text": "Horario soporte lun-vie 09 a 18h", "source": "faq_horarios", "chunk_index": 0, "metadata": {"categoria": "soporte", "pagina": 12}} +{"text": "Devolución 14 días naturales", "source": "faq_compras", "chunk_index": 0, "metadata": {"categoria": "ventas"}} +``` + +Campos soportados: + +| Campo | Obligatorio | Notas | +|---|---|---| +| `text` | ✅ | Cuerpo del chunk (string) | +| `source` | ❌ | Por defecto: `nombre_archivo.jsonl#lineaN` | +| `chunk_index` | ❌ | Por defecto: número de línea 0-based | +| cualquier otro | ❌ | Guardado en `chunk.metadata` y mostrado en los logs | + +--- + +## 4. Persistencia del índice NPZ + +Para evitar recalcular cientos/miles de embeddings en cada arranque: + +### Primer arranque +``` +archivos md/txt/jsonl → chunking → embedding → guarda: + kb/_index.npz (matriz numpy float32 N × embedding_dim) + kb/_chunks.jsonl (texto, source, metadata por cada línea) +``` +Tiempo estimado: ~500 chunk/s en GB10 con MiniLM-L12-v2. + +### Arranques sucesivos +``` +Si existen _index.npz Y _chunks.jsonl Y las dimensiones coinciden: + → carga directamente desde disco (<1 segundo) +Si no: + → reconstruye desde cero +``` + +### Forzar reconstrucción +Usa **un solo** método cuando añadas/modifiques documentos: +1. **Flag CLI**: añade `--rag_force_rebuild` al arranque (recomendado) +2. **Manual**: borra `kb/_index.npz` y `kb/_chunks.jsonl` + +--- + +## 5. Configuración (CLI / JSON) + +Todos los parámetros están definidos en [rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) y son accesibles: +- Vía **flag CLI** en el comando `speech-to-speech ... --rag_enabled` +- Vía **archivo JSON** pasado como único argumento +- Vía `HfArgumentParser` (patrón estándar del proyecto) + +### Tabla de parámetros + +| Flag CLI | Valor por defecto | Descripción | +|---|---|---| +| `--rag_enabled` | `False` | **Interruptor maestro**. Sin este flag, todo lo demás se ignora. | +| `--rag_kb_path` | `./kb` | Ruta absoluta/relativa de la carpeta KB. | +| `--rag_embedding_model` | `paraphrase-multilingual-MiniLM-L12-v2` | Modelo HuggingFace `sentence-transformers`. *Multilingüe*, buen compromiso calidad/velocidad. | +| `--rag_device` | `auto` | `auto` / `cuda` / `cpu` / `mps`. | +| `--rag_top_k` | `3` | Número máximo de chunk inyectados por turno. **No superar 5-6 si la salida del TTS es larga** (crecen los tokens de contexto). | +| `--rag_threshold` | `0.25` | Umbral mínimo de similitud coseno. Rango `[0,1]`.
• `0.15`–`0.25` → recall alto, recupera casi todo
• `0.35`–`0.45` → precisión alta, solo coincidencias seguras
• `0.5+` → muy restrictivo | +| `--rag_language` | `es` | Idioma del encabezado de inyección: `es`, `it`, `en`. Cambia la cabecera del bloque RAG que pasa al LLM. | +| `--rag_inject_as` | `system` | Dónde inyectar los resultados:
• `system` → concatena a las instrucciones system (**recomendado**)
• `user` → añade como último mensaje del usuario | +| `--rag_chunk_size` | `512` | Caracteres máximos por chunk (solo md/txt). | +| `--rag_chunk_overlap` | `64` | Overlap en caracteres entre chunk adyacentes. | +| `--rag_embedding_batch_size` | `32` | Batch size de embeddings durante la construcción del índice. | +| `--rag_force_rebuild` | `False` | Si `True`, ignora el índice NPZ y regenera todo. | + +### Modelos de embedding recomendados + +| Modelo | Idiomas | Dim embedding | Velocidad GB10 | Mejor para | +|---|---|---|---|---| +| `paraphrase-multilingual-MiniLM-L12-v2` (por defecto) | 50+ | 384 | ⚡⚡⚡ | ES/IT/EN mezclado, KB genéricas | +| `hiiamsid/sentence_similarity_spanish_es` | Solo ES | 768 | ⚡⚡ | KB en español exclusivamente | +| `BAAI/bge-m3` | Multilingüe | 1024 | ⚡ | KB grandes, consultas complejas | +| `intfloat/multilingual-e5-large-instruct` | Multilingüe | 1024 | ⚡ | Consultas complejas con instrucciones | + +--- + +## 6. Integración en start_pipeline.sh + +Añade a tu [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) las líneas siguientes (al final del comando `speech-to-speech`): + +```bash +#!/bin/bash +# ... (flags existentes: llm_backend, model_name, base_url, etc.) + +# ========== RAG SERVER-SIDE ========== + --rag_enabled \ + --rag_kb_path ./kb \ + --rag_top_k 3 \ + --rag_threshold 0.30 \ + --rag_language es \ + --rag_inject_as system \ + --rag_device auto +# --rag_force_rebuild # ⚠️ COMENTA después de reconstruir el índice! +``` + +**Ejemplo de arranque completo** (con los parámetros que ya usas): + +```bash +uv run speech-to-speech \ + --llm_backend chat-completions \ + --chat_completions_handler_base_url "$LLM_BASE_URL" \ + --chat_completions_handler_model_name "$LLM_MODEL" \ + --chat_completions_handler_api_key "$LLM_API_KEY" \ + --mode realtime \ + --realtime_host 0.0.0.0 --realtime_port 8000 \ + --tts qwen3 --qwen3_tts_model_name "$TTS_MODEL" \ + --stt whisper --whisper_stt_model_name "$STT_MODEL" \ + \ + --rag_enabled \ + --rag_kb_path ./kb \ + --rag_top_k 3 \ + --rag_threshold 0.30 \ + --rag_language es \ + --rag_inject_as system +``` + +--- + +## 7. Arquitectura interna — Secuencia de operaciones + +### 7.1 Setup (una sola vez al arranque) +Ocurre en [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py#L1029-L1059) dentro de `main()`: + +``` +main() + ├─► si rag_enabled == True: + │ ├─► RAGRetriever(...) → carga modelo sentence-transformers + │ ├─► .build_index() → carga NPZ o rebuild desde cero + │ │ ├─► _load_index() si existe → carga inmediata + │ │ └─► _load_text_files() + _load_jsonl() + encode() + _save_index() + │ └─► set_global_rag(rag) → registra el singleton global + └─► build_pipeline(...) + └─► cada LLM handler, en su setup(): self.rag = get_global_rag() +``` + +### 7.2 Por cada turno de usuario (runtime) +Hook en [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py#L609-L613), línea **610**: + +``` +process(request: LLMIn) + ├─► construye active_chat + ├─► _apply_config(instructions, wants_audio) → system message con instrucciones + ├─► _inject_rag_context(active_chat, turn_id, turn_revision) ← 🔴 NUESTRO HOOK + │ ├─► (1) query = extrae último texto de usuario desde .buffer + │ ├─► (2) results = rag.search(query, top_k, threshold) + │ │ ├─► embedding query (1 vector) + │ │ ├─► @ (matriz embeddings @ query.T) → similitud coseno + │ │ ├─► argpartition top-k + argsort score + │ │ └─► filtro por threshold + │ ├─► (3) si results vacío → log debug, NINGUNA inyección + │ ├─► (4) si no, formatea en bloque de texto: + │ │ "Fragmentos relevantes recuperados..." + │ │ "[1] (source=xxx.md, score=0.672)\n" + │ ├─► (5a) inject_as=system → añade como system message + │ └─► (5b) inject_as=user → añade como último user message + ├─► resolve_auto_language(lang prompt) + ├─► _generate() → + │ ├─► LOG: LLM REQUEST PROMPT ← incluye los chunk RAG! + │ ├─► stream LLM + │ └─► LOG: LLM RESPONSE ← respuesta final generada con el contexto + └─► yield chunks → TTS +``` + +--- + +## 8. Mensajes de log + +Todos los logs RAG usan el mismo formato de la pipeline (prefijo `pipeline X`, nivel INFO o DEBUG). + +### Setup OK +``` +RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/kb +RAG: Índice cargado desde disco: 27 chunk (shape=(27, 384)). +RAG: Activo. kb=/home/kb top_k=3 umbral=0.300 inject_as=system idioma=es chunk=512 +RAG hook activo en BaseOpenAICompatibleHandler (inject_as=system, idioma=es) +``` + +### Setup: ningún documento +``` +RAG: Ningún documento encontrado en /home/kb. Índice vacío (retrieval no inyectará nada). +``` + +### Retrieval exitoso +``` +RAG: inyectados 2 chunk para turn=turn_91a75 rev=1 — 01_faq_producto.md#c1(0.67); 02_politicas_internas.md#c0(0.52) +``` + +### Ninguna coincidencia +``` +RAG: ningún chunk relevante para turn=turn_91a75 (query='Hola, ¿cómo te llamas?' umbral=0.300) +``` + +### Fallo de search (¡no bloquea la pipeline!) +``` +RAG search fallida para turn=turn_91a75 rev=1: +``` + +--- + +## 9. Depuración y troubleshooting + +### Síntoma 1: "RAG no inyecta nada, aunque sé que el documento está ahí" +1. Revisa los logs — ¿qué mensaje aparece? `inyectados X` o `ningún chunk`? +2. Si es `ningún chunk` → baja el umbral: `--rag_threshold 0.20` o incluso `0.15`. +3. Sube top_k: `--rag_top_k 5`. +4. Reconstruye el índice: añade `--rag_force_rebuild` (quizás modificaste archivos después del primer arranque). +5. Prueba directamente: compara el texto de la pregunta vs el texto del chunk — los modelos MiniLM pueden tener resultados escasos con frases muy coloquiales. **Solución**: aumenta el overlap o redacta chunk con preguntas+respuestas explícitas. + +### Síntoma 2: "Primer arranque lentísimo / crash CUDA OOM" +1. Reduce el batch size de embedding: `--rag_embedding_batch_size 4` (por defecto 32). +2. Si tienes miles de chunk, valora `paraphrase-multilingual-MiniLM-L12-v2` (384 dim, la mitad de memoria que bge-m3). + +### Síntoma 3: "El LLM ignora los chunk e inventa datos" +1. Refuerza las `instructions` de sistema (vía session.update) con un prompt como: + > *"Responde SOLO con la información contenida en los Fragmentos relevantes recuperados. Si no hay información, di 'No tengo información sobre eso'."* +2. Prueba `--rag_inject_as user` (algunos modelos respetan más los bloques inyectados dentro de los mensajes de usuario). +3. Sube el **umbral** (reduce falsos positivos): `--rag_threshold 0.40`. +4. Verifica que los chunk tengan sentido (elimina cabeceras Markdown, metadatos no pertinentes). + +### Síntoma 4: "El log LLM REQUEST PROMPT no muestra los chunk" +Verifica: +- Que `--rag_enabled` se haya pasado realmente (¿aparece el log `RAG: Activo` en el arranque?). +- Que el handler LLM sea una subclase de `BaseOpenAICompatibleHandler` (chat-completions y responses-api lo son; mlx-lm local y transformers no — usan [language_model_handler.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/language_model_handler.py) que NO comparte la base; para estos últimos el RAG no está enganchado todavía). + +--- + +## 10. Ejemplos de sesiones reales + +### Caso: FAQ de producto en español +- **Usuario (audio)**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"* +- **STT**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"* +- **RAG retrieval**: + ``` + Score 0.71 → 01_faq_producto.md#c3: "¿Ofrecéis reembolsos? Sí. Todos los planes tienen un periodo de devolución de 14 días naturales..." + Score 0.23 → 02_politicas_internas.md#c1: "Retención de registros: datos usuario dado de baja 6 meses..." ← descartado por threshold=0.30 + ``` +- **Texto inyectado en el system**: + ``` + Fragmentos relevantes recuperados de la base de conocimientos: + [1] (source=01_faq_producto.md, score=0.712) + ¿Ofrecéis reembolsos? + + Sí. Todos los planes tienen un periodo de devolución de **14 días naturales** + desde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos + se evalúan caso por caso por el equipo de soporte. + ``` +- **Respuesta LLM**: *"Tienes 14 días naturales desde la compra para solicitar un reembolso sin necesidad de dar justificaciones. Pasado ese plazo, el equipo de soporte lo evalúa caso por caso."* +- **TTS**: el audio se genera y se envía al cliente. + +--- + +## 11. Archivos del proyecto involucrados + +| Archivo | Rol | +|---|---| +| [RAG/\_\_init\_\_.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/__init__.py) | Exports públicos (singleton + dataclass) | +| [RAG/retriever.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py) | Core: chunking, embedding, persistencia NPZ, search | +| [arguments_classes/rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) | Dataclass HfArgumentParser 12 parámetros | +| [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py) (`main()`, líneas 1029-1059) | Setup singleton global + build index | +| [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py) | Import singleton (línea 41); setup hook (líneas 163-166); injection (línea 610); helper `_extract_last_user_text` + `_inject_rag_context` (líneas 274-347) | +| [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94) | Grupo optional-deps `rag` | +| [kb/README.md](file:///home/azurian/speech-to-speech/kb/README.md) | Scaffold inicial de la KB | +| [kb/01_faq_producto.md](file:///home/azurian/speech-to-speech/kb/01_faq_producto.md) | Ejemplo FAQ español | +| [kb/02_politicas_internas.md](file:///home/azurian/speech-to-speech/kb/02_politicas_internas.md) | Ejemplo políticas español | +| [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) | Añadir flags --rag_* al comando (véase §6) | +| [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) | Script preconfigurado (variables de entorno + todos los flags RAG) | +| [RAG/router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/router.py) | **NUEVO** — Router FastAPI `/v1/rag/*` (11 endpoints HTTP) para administración dinámica | +| [api/openai_realtime/websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py#L511-L526) | Montaje condicional del router RAG en el mismo FastAPI del websocket | + +--- + +## 13. 🆕 API HTTP dinámicas (puebla la KB en caliente) + +La pipeline expone **11 endpoints JSON** en el mismo servidor que sirve el websocket +(`ws://host:12345` → HTTP sobre `http://host:12345`). +Base path común: `/v1/rag`. + +> ⚠️ Si RAG está desactivado (sin `--rag_enabled`) todos los endpoints responden +> **503 Service Unavailable** con `{"code": "RAG_NOT_ENABLED", ...}`. + +### 13.1 Endpoints disponibles + +| Método | Path | Descripción | +|---|---|---| +| `GET` | `/v1/rag/status` | Estado: número chunk, dimensión embedding, fuentes, dispositivo, umbrales | +| `GET` | `/v1/rag/sources` | **LIST** — fuentes únicas + contador de chunk por cada una | +| `GET` | `/v1/rag/chunks` | **LIST** con query params: filtros, paginación, ordenación por relevancia | +| `POST` | `/v1/rag/chunks/list` | Mismo listado pero con body JSON (recomendado para consultas complejas) | +| `POST` | `/v1/rag/chunks/update` | **UPDATE** de un solo chunk (texto / metadatos / fuente) | +| `POST` | `/v1/rag/upsert/document` | **UPSERT atómico** de un documento entero (borra lo viejo + inserta lo nuevo). ✅ Uso más común para CRUD | +| `POST` | `/v1/rag/search` | Busca chunk con el mismo algoritmo que se usa para la inyección (depuración/pruebas) | +| `POST` | `/v1/rag/add/document` | INSERT puro de texto libre (chunking automático) | +| `POST` | `/v1/rag/add/chunks` | INSERT puro de chunk preformateados | +| `POST` | `/v1/rag/remove` | **REMOVE**: por prefijo o coincidencia exacta | +| `POST` | `/v1/rag/reload` | Reconstrucción total del índice desde los archivos en `kb/` | + +> 💡 **Patrón CRUD recomendado**: +> - CREATE → `/upsert/document` (idempotente: si la source no existe la crea) +> - READ → `/sources` (lista fuentes) + `/chunks?source_exact=X` (detalle) +> - UPDATE → `/upsert/document` (misma source, texto nuevo → sustitución atómica) +> - DELETE → `/upsert/document` con `text: ""` o `/remove` con `exact: true` + +--- + +### 13.2 Ejemplos curl + +> Base URL: `http://127.0.0.1:12345/v1/rag` (cambia puerto y host si modificaste el script). + +#### 📤 Añadir un documento (texto libre, chunking automático) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/document \ + -H 'Content-Type: application/json' \ + -d '{ + "source": "crm/cliente_456_nota_20260826", + "text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual, renovación 15/02/2027. Contacto: +34 600 111 222, javier@empresa.es. Notas: prefiere que le llamen por la mañana antes de las 11h. Ha reportado un incidente con la factura número 2026-08-114 el 20/08/2026 que ya fue resuelto con un descuento del 15% aplicado en la siguiente factura.", + "metadata": {"cliente_id": 456, "pais": "es", "categoria": "crm"}, + "persist": true, + "dynamic": true + }' | jq +``` + +**Respuesta:** +```json +{ + "added": 1, + "sources": ["crm/cliente_456_nota_20260826"], + "total_after": 10 +} +``` + +> 💡 **`persist: true`** → el chunk también se escribe en `kb/_dynamic.jsonl`, +> por lo tanto **en el próximo arranque** de la pipeline seguirá presente. +> Ponlo en `false` para chunk temporales (solo la sesión actual). + +#### 📤 Añadir N chunk preformateados (ej. desde DB) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/chunks \ + -H 'Content-Type: application/json' \ + -d '{ + "items": [ + {"text": "Pedido 8942 — 25/08/2026. Productos: Teclado RGB Pro x1, Ratón Ergonómico x1. Estado: enviado, tracking 1Z999AA10123456784. Entrega estimada: 27/08/2026.", + "source": "pedidos/cliente_456/8942", + "metadata": {"pedido_id": 8942, "estado": "enviado"}}, + {"text": "Dirección de envío habitual del cliente 456: Av. Diagonal 444, puerta 3, 08013 Barcelona, España. Contacto entrega: +34 600 111 222.", + "source": "direcciones/cliente_456/principal", + "metadata": {"tipo": "envio", "predeterminada": true}} + ], + "persist": true, + "dynamic": true + }' | jq +``` + +#### 🔎 Probar el retrieval vía HTTP (depuración, sin audio) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/search \ + -H 'Content-Type: application/json' \ + -d '{ + "query": "¿Qué pedido tiene Javier García y cuándo llega?", + "top_k": 3, + "threshold": 0.25 + }' | jq +``` + +Respuesta con los chunk recuperados y el score: +```json +{ + "query": "¿Qué pedido tiene Javier García y cuándo llega?", + "count": 2, + "results": [ + {"text": "Pedido 8942 — 25/08/2026...", + "source": "pedidos/cliente_456/8942", "score": 0.813}, + {"text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual...", + "source": "crm/cliente_456_nota_20260826", "score": 0.742} + ] +} +``` + +#### 📊 Ver estado general + +```bash +curl -sS http://127.0.0.1:12345/v1/rag/status | jq +``` + +#### 🗑️ Eliminar todos los chunk de un cliente + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \ + -H 'Content-Type: application/json' \ + -d '{"source_prefix": "crm/cliente_456"}' | jq +``` + +Sugerencia: usa prefijos inteligentes para organizar la KB: +``` +source_prefix = "crm/" → borra toda la agenda +source_prefix = "pedidos/" → borra todos los datos de pedidos +source_prefix = "dynamic#" → borra todos los chunk temporales auto-numerados +``` + +#### 🛡️ Remove con coincidencia exacta (solo source EXACTA) + +```bash +# Borra SÓLO el documento 'crm/cliente_456_nota_20260826' +# (no borra 'crm/cliente_456_nota_20260827') +curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \ + -H 'Content-Type: application/json' \ + -d '{"source_prefix": "crm/cliente_456_nota_20260826", "exact": true}' | jq +``` + +#### 📋 Listado de fuentes (inventario del índice) + +```bash +# Todas las fuentes del índice, con conteo de chunk +curl -sS http://127.0.0.1:12345/v1/rag/sources | jq +``` + +Ejemplo salida: +```json +{ + "count": 5, + "items": [ + {"source": "01_faq_producto.md", "chunk_count": 5, "has_dynamic": false}, + {"source": "crm/cliente_456_nota_20260826", "chunk_count": 1, "has_dynamic": true}, + {"source": "pedidos/cliente_456/8942", "chunk_count": 1, "has_dynamic": true} + ] +} +``` + +#### 🔍 Listar chunk con filtros y paginación + +```bash +# GET con query params: solo chunk de cliente_456, paginado +curl -sS 'http://127.0.0.1:12345/v1/rag/chunks?source_prefix=crm/cliente_456&limit=10&offset=0' | jq + +# POST con body JSON: ORDENA por relevancia sobre una consulta + filtro +curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/list \ + -H 'Content-Type: application/json' \ + -d '{ + "source_prefix": "crm/", + "query": "descuento aplicado en factura", + "min_score": 0.20, + "limit": 10, + "include_text": true + }' | jq +``` + +Respuesta: +```json +{ + "total": 2, + "offset": 0, + "limit": 10, + "query": "descuento aplicado en factura", + "items": [ + { + "index": 9, + "source": "crm/cliente_456_nota_20260826", + "chunk_index": 1000009, + "score": 0.731, + "metadata": {"cliente_id": 456, "categoria": "crm"}, + "is_dynamic": true, + "text": "Cliente: Javier García... 15% descuento..." + } + ] +} +``` + +Campos especiales: +- `index`: la posición numérica en el vector de chunk — pásala directamente a `/chunks/update` +- `score`: `null` sin query, float 0-1 con ordenación por relevancia + +#### 🆙 Actualizar un solo chunk + +Usa `/chunks/update` cuando debas modificar **únicamente un trozo específico** (ej. corregir una errata, cambiar metadatos). Para actualizar **un documento entero** usa en su lugar `/upsert/document`. + +```bash +# Opción A: actualiza por `index` (de list_chunks → campo "index": 9) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \ + -H 'Content-Type: application/json' \ + -d '{ + "index": 9, + "new_text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2028 (2 años). Descuento 20% acordado en la llamada del 10/08.", + "new_metadata": {"cliente_id": 456, "categoria": "crm", "ultima_revision": "2026-08-26"} + }' | jq + +# Opción B: actualiza por (source, chunk_index) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \ + -H 'Content-Type: application/json' \ + -d '{ + "source": "crm/cliente_456_nota_20260826", + "chunk_index": 1000009, + "new_source": "crm/cliente_456/nota_principal" + }' | jq +``` + +Respuesta éxito `200`: +```json +{ + "updated": 1, + "chunk": { + "index": 9, + "source": "crm/cliente_456/nota_principal", + "chunk_index": 1000009, + "text": "Cliente: Javier García...", + "metadata": {"cliente_id": 456, "...": "..."}, + "is_dynamic": true + } +} +``` + +Errores HTTP posibles: +- `404 CHUNK_NOT_FOUND` +- `409 AMBIGUOUS_SOURCE` → pasaste `source` sola pero existen N chunk con la misma source; añade `chunk_index` o usa `index` + +#### 🔄 Upsert documento (CRUD recomendado) + +**Éste es el endpoint que usarás en el 90% de los casos.** +Idempotente sobre el campo `source`: +- Si la `source` no existe → CREATE +- Si la `source` existe → DELETE de todos los chunk antiguos + split del nuevo texto + INSERT +- Si mandas `text: ""` → DELETE atómico (sin re-insert) + +```bash +# CREATE/UPDATE de un documento entero (ej. nota cliente) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \ + -H 'Content-Type: application/json' \ + -d '{ + "source": "crm/cliente_456/perfil", + "text": "Javier García — ID 456. Plan Premium Anual, renovación automática 15/02/2028. Precio: 24€/mes. Email javier@empresa.es. Tel +34 600 111 222. Dirección envío habitual: Av. Diagonal 444, puerta 3, 08013 Barcelona. Notas: siempre prefiere atención por la mañana, antes de las 11h. Caso abierto 2026-08-0124: seguimiento calidad postventa, cerrado con valoración 5/5.", + "metadata": {"cliente_id": 456, "pais": "es", "actualizado": "2026-08-26"}, + "dynamic": true, + "persist": true + }' | jq +``` + +Respuesta: +```json +{ + "removed": 2, + "added": 1, + "sources": ["crm/cliente_456/perfil"], + "total_after": 10, + "mode": "upsert" +} +``` + +DELETE vía upsert: +```bash +# Elimina del todo todos los chunk asociados a esta source (atómico) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \ + -H 'Content-Type: application/json' \ + -d '{"source": "crm/cliente_456/perfil", "text": ""}' | jq +``` + +#### 🔄 Recargar toda la KB desde archivos (después de copiar nuevos md/txt) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/reload \ + -H 'Content-Type: application/json' \ + -d '{"force_rebuild": true}' | jq +``` + +--- + +### 13.3 Persistencia entre reinicios: `kb/_dynamic.jsonl` + +Todos los chunk añadidos vía API con `persist: true` se añaden a `kb/_dynamic.jsonl` (un JSON por línea). En el próximo arranque de la pipeline: +1. Se reconstruye el índice estático desde los archivos md/txt/jsonl +2. **Inmediatamente después** se recargan y re-embeddan también todos los chunk de `_dynamic.jsonl` + +Por lo tanto el conocimiento añadido por HTTP es **duradero** y no se pierde al reiniciar. + +--- + +### 13.4 Seguridad en hilos (thread safety) + +Todas las API usan un `threading.RLock()` dentro de [RAGRetriever](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py): +- ✅ Varias llamadas `/search` pueden ejecutarse en paralelo durante la conversación +- ✅ Una `add_document`/`remove` bloquea brevemente el índice solo para las operaciones numpy de vstack/take — el retrieval no se pierde, espera +- ✅ La persistencia en disco (NPZ + JSONL) ocurre fuera del lock, por lo que no ralentiza la llamada de la conversación + +--- + +## 14. Hoja de ruta / posibles mejoras + +- [ ] **Soporte PDF**: añadir `pypdf` o `pdfplumber` en un grupo opcional +- [ ] **MMR reranking** en lugar del solo top-k por coseno (mejor diversidad entre chunk) +- [ ] **HyDE** (el LLM genera una consulta hipotética + embedding de la misma, para consultas coloquiales) +- [ ] **Cross-encoder reranker** de 2 etapas para KB grandes (>10k chunk) +- [ ] **Hook sobre LanguageModelHandler local** (mlx-lm / transformers) — actualmente el RAG solo está enganchado para los backends `chat-completions` y `responses-api` +- [ ] **Watchdog KB**: auto-rebuild del índice cuando cambian los archivos (inotify / watchdog) diff --git a/docs/RAG_SERVER_SIDE.md b/docs/RAG_SERVER_SIDE.md new file mode 100644 index 0000000..6d14a46 --- /dev/null +++ b/docs/RAG_SERVER_SIDE.md @@ -0,0 +1,676 @@ +# RAG Server-Side — Guida Ufficiale + +> **Feature**: Retrieval Augmented Generation integrato nella pipeline speech-to-speech. +> **Approccio**: 2 — Injection trasparente lato server (nessuna modifica lato client). +> **Versione minima speech-to-speech**: `0.2.11` + +--- + +## 1. Panoramica + +L'RAG server-side arricchisce **automaticamente** ogni risposta dell'LLM con brani rilevanti estratti da una knowledge base locale. L'intero ciclo avviene in modo invisibile al client Realtime: + +``` +Utente parla → STT → 🟡 RAG RETRIEVAL (hook interno) → LLM → TTS → Audio all'utente + ↓ + kb/*.md, kb/*.txt, kb/*.jsonl + ↓ + top-k chunk iniettati nel prompt +``` + +### Vantaggi +- ✅ Zero modifiche lato client — funziona con qualunque SDK Realtime +- ✅ Riuso del singleton globale (1 sola copia degli embeddings per N pipeline parallele) +- ✅ Persistenza dell'indice NPZ — ricostruzione solo se i documenti cambiano +- ✅ 100% compatibile con i log già esistenti (`LLM REQUEST PROMPT` include i chunk iniettati) +- ✅ Supporto multilingue (IT/ES/EN configurabile) + +--- + +## 2. Installazione dipendenze + +Le dipendenze RAG sono opzionali (gruppo `rag` in [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94)): + +```bash +cd /home/azurian/speech-to-speech +uv pip install -e ".[rag]" +``` + +**Contenuto del gruppo**: +- `sentence-transformers>=3.0.0` (porta automaticamente `torch`, `numpy`, `transformers` già presenti) + +> ✅ **Su DGX Spark GB10**: il modello embedding viene caricato nativamente su CUDA (detected da `--rag_device auto`, default). + +--- + +## 3. Struttura Knowledge Base + +La KB risiede nella cartella configurata dal parametro `--rag_kb_path` (default: `./kb`). + +``` +kb/ +├── README.md ← istruzioni scaffold (auto-generato) +├── 01_faq_producto.md ← esempio spagnolo incluso +├── 02_politicas_internas.md ← esempio spagnolo incluso +│ +├── manual/ ← sotto-cartelle supportate +│ ├── 01_instalacion.md +│ └── 02_facturacion.txt +│ +├── datos/ +│ └── clientes.jsonl ← formato pre-chunkizzato +│ +├── _index.npz ← ⚙️ indice generato (NON modificare) +└── _chunks.jsonl ← ⚙️ catalogo chunk (NON modificare) +``` + +### Formati supportati + +#### A. File Markdown / TXT (raccomandato, zero sforzo) + +Qualsiasi `*.md` o `*.txt` nella cartella o sottocartelle viene: +1. Letto in UTF-8 (fallback latin-1 con sostituzione errori) +2. Splittato automaticamente in chunk: + - **chunk size** default: `512` caratteri (parametro `--rag_chunk_size`) + - **overlap** default: `64` caratteri (parametro `--rag_chunk_overlap`) + - Algoritmo: splitter ricorsivo per separatori `\n\n → \n → . ? ! ; , → spazio → carattere` +3. Ogni chunk riceve `source = path_relativo#chunk_index` + +#### B. JSONL pre-chunkizzato (controllo fine) + +Se preferisci gestire chunk e metadata manualmente (es. da estrazione PDF strutturata), crea `*.jsonl` con **una riga per chunk**: + +```jsonl +{"text": "Horario soporte lun-vier 09 a 18h", "source": "faq_horarios", "chunk_index": 0, "metadata": {"categoria": "soporte", "pagina": 12}} +{"text": "Devolución 14 días naturales", "source": "faq_compras", "chunk_index": 0, "metadata": {"categoria": "ventas"}} +``` + +Campi supportati: +| Campo | Obbligatorio | Note | +|---|---|---| +| `text` | ✅ | Corpo del chunk (stringa) | +| `source` | ❌ | Default: `nome_file.jsonl#rigaN` | +| `chunk_index` | ❌ | Default: numero riga 0-based | +| qualsiasi altro | ❌ | Salvato in `chunk.metadata` e riportato nei log | + +--- + +## 4. Persistenza indice NPZ + +Per evitare di ricalcolare centinaia/migliaia di embeddings ad ogni avvio: + +### Primo avvio +``` +file md/txt/jsonl → chunking → embedding → salva: + kb/_index.npz (matrice numpy float32 N × embedding_dim) + kb/_chunks.jsonl (testo, source, metadata per ogni riga) +``` +Tempo stimato: ~500 chunk/s su GB10 con MiniLM-L12-v2. + +### Avvii successivi +``` +Se esiste _index.npz E _chunks.jsonl E le dimensioni coincidono: + → carica direttamente da disco (<1 secondo) +Altrimenti: + → ricostruisce da zero +``` + +### Forzare ricostruzione +Usa **uno** di questi metodi quando aggiungi/modifichi documenti: +1. **Flag CLI**: aggiungi `--rag_force_rebuild` all'avvio (consigliato) +2. **Manuale**: cancella `kb/_index.npz` e `kb/_chunks.jsonl` + +--- + +## 5. Configurazione (CLI / JSON) + +Tutti i parametri sono definiti in [rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) e sono accessibili: +- Via **flag CLI** nel comando `speech-to-speech ... --rag_enabled` +- Via **file JSON** passato come unico argomento +- Via `HfArgumentParser` (pattern standard del progetto) + +### Tabella parametri + +| Flag CLI | Default | Descrizione | +|---|---|---| +| `--rag_enabled` | `False` | **Master switch**. Senza questo flag, tutto il resto viene ignorato. | +| `--rag_kb_path` | `./kb` | Path assoluto/relativo della cartella KB. | +| `--rag_embedding_model` | `paraphrase-multilingual-MiniLM-L12-v2` | Modello HuggingFace `sentence-transformers`. *Multilingue*, ottimo compromesso qualità/velocità. | +| `--rag_device` | `auto` | `auto` / `cuda` / `cpu` / `mps`. | +| `--rag_top_k` | `3` | Numero massimo di chunk iniettati per turno. **Non superare 5-6 se TTS output lungo** (crescono i token di contesto). | +| `--rag_threshold` | `0.25` | Soglia minima similarità coseno. Range `[0,1]`.
• `0.15`–`0.25` → recall alto, recuperi quasi tutto
• `0.35`–`0.45` → precisione alta, solo match certi
• `0.5+` → molto restrittivo | +| `--rag_language` | `es` | Lingua header injection: `es`, `it`, `en`. Cambia l'intestazione del blocco RAG passato all'LLM. | +| `--rag_inject_as` | `system` | Dove iniettare i risultati:
• `system` → concatenato alle istruzioni system (**consigliato**)
• `user` → accodato all'ultimo messaggio utente | +| `--rag_chunk_size` | `512` | Caratteri massimi per chunk (solo md/txt). | +| `--rag_chunk_overlap` | `64` | Overlap caratteri tra chunk adiacenti. | +| `--rag_embedding_batch_size` | `32` | Batch size embedding durante build indice. | +| `--rag_force_rebuild` | `False` | Se `True`, ignora l'indice NPZ e rigenera tutto. | + +### Modelli embedding consigliati + +| Modello | Lingue | Dim embedding | Velocità GB10 | Migliore per | +|---|---|---|---|---| +| `paraphrase-multilingual-MiniLM-L12-v2` (default) | 50+ | 384 | ⚡⚡⚡ | IT/ES/EN mix, KB generiche | +| `hiiamsid/sentence_similarity_spanish_es` | Solo ES | 768 | ⚡⚡ | KB solo spagnola | +| `BAAI/bge-m3` | Multilingue | 1024 | ⚡ | KB grandi, query complesse | +| `intfloat/multilingual-e5-large-instruct` | Multilingue | 1024 | ⚡ | Query complesse, istruzioni | + +--- + +## 6. Integrazione in start_pipeline.sh + +Aggiungi al tuo [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) le righe seguenti (alla fine del comando `speech-to-speech`): + +```bash +#!/bin/bash +# ... (flags esistenti: llm_backend, model_name, base_url, ecc.) + +# ========== RAG SERVER-SIDE ========== + --rag_enabled \ + --rag_kb_path ./kb \ + --rag_top_k 3 \ + --rag_threshold 0.30 \ + --rag_language es \ + --rag_inject_as system \ + --rag_device auto +# --rag_force_rebuild # ⚠️ COMMENTA dopo aver ricostruito l'indice! +``` + +**Esempio start completo** (con i parametri che già usi): +```bash +uv run speech-to-speech \ + --llm_backend chat-completions \ + --chat_completions_handler_base_url "$LLM_BASE_URL" \ + --chat_completions_handler_model_name "$LLM_MODEL" \ + --chat_completions_handler_api_key "$LLM_API_KEY" \ + --mode realtime \ + --realtime_host 0.0.0.0 --realtime_port 8000 \ + --tts qwen3 --qwen3_tts_model_name "$TTS_MODEL" \ + --stt whisper --whisper_stt_model_name "$STT_MODEL" \ + \ + --rag_enabled \ + --rag_kb_path ./kb \ + --rag_top_k 3 \ + --rag_threshold 0.30 \ + --rag_language es \ + --rag_inject_as system +``` + +--- + +## 7. Architettura interna — Sequenza operazioni + +### 7.1 Setup (una tantum ad avvio) +Avviene in [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py#L1029-L1059) dentro `main()`: + +``` +main() + ├─► if rag_enabled == True: + │ ├─► RAGRetriever(...) → carica modello sentence-transformers + │ ├─► .build_index() → NPZ o rebuild da zero + │ │ ├─► _load_index() se esiste → carica immediato + │ │ └─► _load_text_files() + _load_jsonl() + encode() + _save_index() + │ └─► set_global_rag(rag) → registra singleton globale + └─► build_pipeline(...) + └─► ogni LLM handler, nel suo setup(): self.rag = get_global_rag() +``` + +### 7.2 Per ogni turno utente (runtime) +Hook in [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py#L609-L613), riga **610**: + +``` +process(request: LLMIn) + ├─► build active_chat + ├─► _apply_config(instructions, wants_audio) → system message con le istruzioni + ├─► _inject_rag_context(active_chat, turn_id, turn_revision) ← 🔴 NOSTRO HOOK + │ ├─► (1) query = estrai ultimo testo utente da .buffer + │ ├─► (2) results = rag.search(query, top_k, threshold) + │ │ ├─► embedding query (1 vettore) + │ │ ├─► @ (matrice embeddings @ query.T) → coseno similarity + │ │ ├─► argpartition top-k + argsort score + │ │ └─► filtro per threshold + │ ├─► (3) se results vuoti → log debug, NIENTE injection + │ ├─► (4) altrimenti formatta in blocco testuale: + │ │ "Fragmentos relevantes recuperados..." + │ │ "[1] (source=xxx.md, score=0.672)\n" + │ ├─► (5a) inject_as=system → aggiungi come system message + │ └─► (5b) inject_as=user → aggiungi come user message finale + ├─► resolve_auto_language(lang prompt) + ├─► _generate() → + │ ├─► LOG: LLM REQUEST PROMPT ← include i chunk RAG! + │ ├─► stream LLM + │ └─► LOG: LLM RESPONSE ← risposta finale generata col contesto + └─► yield chunks → TTS +``` + +--- + +## 8. Messaggi di log + +Tutti i log RAG usano lo stesso format della pipeline (prefix `pipeline X`, livello INFO o DEBUG). + +### Setup OK +``` +RAG: Inizializzazione modello embedding=paraphrase-multilingual-MiniLM-L12-v2 device=cuda su kb_path=/home/kb +RAG: Indice caricato da disco: 27 chunk (shape=(27, 384)). +RAG: Attivo. kb=/home/kb top_k=3 soglia=0.300 inject_as=system lingua=es chunk=512 +RAG hook attivo in BaseOpenAICompatibleHandler (inject_as=system, lingua=es) +``` + +### Setup: nessun documento +``` +RAG: Nessun documento trovato in /home/kb. Indice vuoto (retrieval non inietterà niente). +``` + +### Retrieval riuscito +``` +RAG: iniettati 2 chunk per turn=turn_91a75 rev=1 — 01_faq_producto.md#c1(0.67); 02_politicas_internas.md#c0(0.52) +``` + +### Nessun match +``` +RAG: nessun chunk rilevante per turn=turn_91a75 (query='Hola, ¿cómo te llamas?' soglia=0.300) +``` + +### Fallimento search (non blocca la pipeline!) +``` +RAG search fallita per turn=turn_91a75 rev=1: +``` + +--- + +## 9. Debug & troubleshooting + +### Sintomo 1: "RAG non inietta niente, anche se so che il documento c'è" +1. Controlla i log — che messaggio compare? `iniettati X` o `nessun chunk`? +2. Se `nessun chunk` → abbassa la soglia: `--rag_threshold 0.20` o anche `0.15`. +3. Aumenta top_k: `--rag_top_k 5`. +4. Ricostruisci indice: aggiungi `--rag_force_rebuild` (magari hai modificato i file dopo il primo avvio). +5. Testa direttamente: confronta il testo della domanda vs il testo del chunk — i modelli MiniLM possono avere scarsi risultati con frasi troppo colloquiali. **Soluzione**: aumenta overlap o scrivi chunk con domande+risposte esplicite. + +### Sintomo 2: "Primo avvio lentissimo / crash CUDA OOM" +1. Riduci batch size embedding: `--rag_embedding_batch_size 4` (default 32). +2. Se hai migliaia di chunk, prendi in considerazione `paraphrase-multilingual-MiniLM-L12-v2` (384 dim, metà memoria di bge-m3). + +### Sintomo 3: "L'LLM ignora i chunk e inventa fatti" +1. Rinforza le `instructions` di sistema (via session.update) con un prompt tipo: + > *"Responde SOLO con información contenuta nei Fragmentos relevantes recuperados. Si no hay información, di 'No tengo información sobre eso'."* +2. Passa a `--rag_inject_as user` (alcuni modelli rispettano di più i blocchi injection dentro i messaggi utente). +3. Aumenta la **soglia** (riduci falsi positivi): `--rag_threshold 0.40`. +4. Verifica che i chunk abbiano senso (elimina intestazioni Markdown, metadati non pertinenti). + +### Sintomo 4: "Il log LLM REQUEST PROMPT non mostra i chunk" +Verifica: +- `--rag_enabled` è effettivamente passato (log `RAG: Attivo` in avvio?) +- L'handler LLM è una sottoclasse di `BaseOpenAICompatibleHandler` (chat-completions e responses-api lo sono; mlx-lm locale e transformers no — usano [language_model_handler.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/language_model_handler.py) che NON condivide la base; per questi ultimi il RAG al momento non è agganciato). + +--- + +## 10. Esempi di sessioni reali + +### Caso: FAQ prodotto in spagnolo +- **Utente audio**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"* +- **STT**: *"¿Cuánto tiempo tengo para solicitar un reembolso?"* +- **RAG retrieval**: + ``` + Score 0.71 → 01_faq_producto.md#c3: "¿Ofrecéis reembolsos? Sí. Todos los planes tienen un periodo de devolución de 14 días naturales..." + Score 0.23 → 02_politicas_internas.md#c1: "Retención de registros: datos usuario dado de baja 6 meses..." ← scartato da threshold=0.30 + ``` +- **Testo iniettato nel system**: + ``` + Fragmentos relevantes recuperados de la base de conocimientos: + [1] (source=01_faq_producto.md, score=0.712) + ¿Ofrecéis reembolsos? + + Sí. Todos los planes tienen un periodo de devolución de **14 días naturales** + desde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos + se evalúan caso por caso por el equipo de soporte. + ``` +- **Risposta LLM**: *"Tienes 14 días naturales desde la compra para solicitar un reembolso sin necesidad de dar justificaciones. Pasado ese plazo, el equipo de soporte lo evalúa caso por caso."* +- **TTS**: l'audio viene generato e spedito al client. + +--- + +## 11. File del progetto coinvolti + +| File | Ruolo | +|---|---| +| [RAG/\_\_init\_\_.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/__init__.py) | Export pubblici (singleton + dataclass) | +| [RAG/retriever.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py) | Core: chunking, embedding, persistenza NPZ, search | +| [arguments_classes/rag_arguments.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/arguments_classes/rag_arguments.py) | Dataclass HfArgumentParser 12 parametri | +| [s2s_pipeline.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/s2s_pipeline.py) (`main()`, righe 1029-1059) | Setup singleton globale + build index | +| [base_openai_compatible_language_model.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/LLM/base_openai_compatible_language_model.py) | Import singleton (riga 41); setup hook (riga 163-166); injection (riga 610); helper `_extract_last_user_text` + `_inject_rag_context` (righe 274-347) | +| [pyproject.toml](file:///home/azurian/speech-to-speech/pyproject.toml#L92-L94) | Gruppo optional-deps `rag` | +| [kb/README.md](file:///home/azurian/speech-to-speech/kb/README.md) | Scaffold iniziale KB | +| [kb/01_faq_producto.md](file:///home/azurian/speech-to-speech/kb/01_faq_producto.md) | Esempio FAQ spagnolo | +| [kb/02_politicas_internas.md](file:///home/azurian/speech-to-speech/kb/02_politicas_internas.md) | Esempio politiche spagnolo | +| [start_pipeline.sh](file:///home/azurian/speech-to-speech/start_pipeline.sh) | Aggiungere flag --rag_* al comando (vedi §6) | +| [start_pipeline_rag.sh](file:///home/azurian/speech-to-speech/start_pipeline_rag.sh) | Script preconfigurato (variabili d'ambiente + tutti i flag RAG) | +| [RAG/router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/router.py) | **NUOVO** — Router FastAPI `/v1/rag/*` (7 endpoint HTTP) per amministrazione dinamica | +| [api/openai_realtime/websocket_router.py](file:///home/azurian/speech-to-speech/src/speech_to_speech/api/openai_realtime/websocket_router.py#L511-L526) | Montaggio condizionale del router RAG nello stesso FastAPI del websocket | + +--- + +## 13. 🆕 API HTTP dinamiche (popola la KB a caldo) + +La pipeline espone **7 endpoint JSON** sullo stesso server che serve il websocket +(`ws://host:12345` → HTTP su `http://host:12345`). +Base path comune: `/v1/rag`. + +> ⚠️ Se RAG è disattivato (no `--rag_enabled`) tutti gli endpoint rispondono +> **503 Service Unavailable** con `{"code": "RAG_NOT_ENABLED", ...}`. + +### 13.1 Endpoint disponibili + +| Metodo | Path | Descrizione | +|---|---|---| +| `GET` | `/v1/rag/status` | Stato: numero chunk, embedding dim, sorgenti, device, soglie | +| `GET` | `/v1/rag/sources` | **LIST** — sorgenti uniche + chunk count per ognuna | +| `GET` | `/v1/rag/chunks` | **LIST** con query params: filtri, paginazione, ordinamento per rilevanza | +| `POST` | `/v1/rag/chunks/list` | Stesso listing, ma body JSON (consigliato per query complesse) | +| `POST` | `/v1/rag/chunks/update` | **UPDATE** singolo chunk (testo / metadata / sorgente) | +| `POST` | `/v1/rag/upsert/document` | **UPSERT atomico** di un documento intero (delete old + insert new). ✅ Uso più comune per CRUD | +| `POST` | `/v1/rag/search` | Cerca chunk con lo stesso algoritmo usato per l'iniezione (debug/test) | +| `POST` | `/v1/rag/add/document` | INSERT puro di testo libero (chunking auto) | +| `POST` | `/v1/rag/add/chunks` | INSERT puro di chunk preformattati | +| `POST` | `/v1/rag/remove` | **REMOVE**: per prefisso o exact match | +| `POST` | `/v1/rag/reload` | Ricostruzione totale indice da file in `kb/` | + +> 💡 **Pattern CRUD consigliato**: +> - CREATE → `/upsert/document` (idempotente: se la source non esiste la crea) +> - READ → `/sources` (lista sorgenti) + `/chunks?source_exact=X` (dettaglio) +> - UPDATE → `/upsert/document` (stessa source, testo nuovo → sostituzione atomica) +> - DELETE → `/upsert/document` con `text: ""` oppure `/remove` con `exact: true` + +--- + +### 13.2 Esempi curl + +> Base URL: `http://127.0.0.1:12345/v1/rag` (cambia porta e host se hai modificato lo script). + +#### 📤 Aggiungere un documento (testo libero, chunking automatico) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/document \ + -H 'Content-Type: application/json' \ + -d '{ + "source": "crm/cliente_456_nota_20260826", + "text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual, renovación 15/02/2027. Contacto: +34 600 111 222, javier@empresa.es. Notas: prefiere que le llamen por la mañana antes de las 11h. Ha reportado un incidente con la factura número 2026-08-114 el 20/08/2026 que ya fue resuelto con un descuento del 15% aplicado en la siguiente factura.", + "metadata": {"cliente_id": 456, "pais": "es", "categoria": "crm"}, + "persist": true, + "dynamic": true + }' | jq +``` + +**Risposta:** +```json +{ + "added": 1, + "sources": ["crm/cliente_456_nota_20260826"], + "total_after": 10 +} +``` + +> 💡 **`persist: true`** → il chunk viene scritto anche in `kb/_dynamic.jsonl`, +> quindi **al prossimo riavvio** della pipeline sarà ancora presente. +> Imposta a `false` per chunk temporanei (solo sessione corrente). + +#### 📤 Aggiungere N chunk pre-formattati (es. da DB) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/add/chunks \ + -H 'Content-Type: application/json' \ + -d '{ + "items": [ + {"text": "Pedido 8942 — 25/08/2026. Productos: Teclado RGB Pro x1, Ratón Ergonómico x1. Estado: enviado, tracking 1Z999AA10123456784. Entrega estimada: 27/08/2026.", + "source": "pedidos/cliente_456/8942", + "metadata": {"pedido_id": 8942, "estado": "enviado"}}, + {"text": "Dirección de envío habitual del cliente 456: Av. Diagonal 444, puerta 3, 08013 Barcelona, España. Contacto entrega: +34 600 111 222.", + "source": "direcciones/cliente_456/principal", + "metadata": {"tipo": "envio", "predeterminada": true}} + ], + "persist": true, + "dynamic": true + }' | jq +``` + +#### 🔎 Provare il retrieval da HTTP (debug, no audio) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/search \ + -H 'Content-Type: application/json' \ + -d '{ + "query": "¿Qué pedido tiene Javier García y cuándo llega?", + "top_k": 3, + "threshold": 0.25 + }' | jq +``` + +Risposta con i chunk recuperati e lo score: +```json +{ + "query": "¿Qué pedido tiene Javier García y cuándo llega?", + "count": 2, + "results": [ + {"text": "Pedido 8942 — 25/08/2026...", + "source": "pedidos/cliente_456/8942", "score": 0.813}, + {"text": "Cliente: Javier García, id=456. Plan contratado: Premium Anual...", + "source": "crm/cliente_456_nota_20260826", "score": 0.742} + ] +} +``` + +#### 📊 Ver stato generale + +```bash +curl -sS http://127.0.0.1:12345/v1/rag/status | jq +``` + +#### 🗑️ Eliminare tutti i chunk di un cliente + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \ + -H 'Content-Type: application/json' \ + -d '{"source_prefix": "crm/cliente_456"}' | jq +``` + +Suggerimento: usa prefissi intelligenti per organizzare la KB: +``` +source_prefix = "crm/" → cancella tutta la rubrica +source_prefix = "pedidos/" → cancella tutti i dati ordini +source_prefix = "dynamic#" → cancella tutti i chunk temporanei auto-numerati +``` + +#### 🛡️ Remove con exact match (solo source ESATTA) + +```bash +# Cancella SOLO il documento 'crm/cliente_456_nota_20260826' +# (non cancella 'crm/cliente_456_nota_20260827') +curl -sS -X POST http://127.0.0.1:12345/v1/rag/remove \ + -H 'Content-Type: application/json' \ + -d '{"source_prefix": "crm/cliente_456_nota_20260826", "exact": true}' | jq +``` + +#### 📋 Lista sorgenti (index inventory) + +```bash +# Tutte le sorgenti nell'indice, con conteggio chunk +curl -sS http://127.0.0.1:12345/v1/rag/sources | jq +``` + +Esempio output: +```json +{ + "count": 5, + "items": [ + {"source": "01_faq_producto.md", "chunk_count": 5, "has_dynamic": false}, + {"source": "crm/cliente_456_nota_20260826", "chunk_count": 1, "has_dynamic": true}, + {"source": "pedidos/cliente_456/8942", "chunk_count": 1, "has_dynamic": true} + ] +} +``` + +#### 🔍 Elencare chunk con filtri e paginazione + +```bash +# GET con query params: solo chunk di cliente_456, paged +curl -sS 'http://127.0.0.1:12345/v1/rag/chunks?source_prefix=crm/cliente_456&limit=10&offset=0' | jq + +# POST con body JSON: ORDINA per rilevanza su una query + filtro +curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/list \ + -H 'Content-Type: application/json' \ + -d '{ + "source_prefix": "crm/", + "query": "descuento aplicado en factura", + "min_score": 0.20, + "limit": 10, + "include_text": true + }' | jq +``` + +Risposta: +```json +{ + "total": 2, + "offset": 0, + "limit": 10, + "query": "descuento aplicado en factura", + "items": [ + { + "index": 9, + "source": "crm/cliente_456_nota_20260826", + "chunk_index": 1000009, + "score": 0.731, + "metadata": {"cliente_id": 456, "categoria": "crm"}, + "is_dynamic": true, + "text": "Cliente: Javier García... 15% descuento..." + } + ] +} +``` + +Campi speciali: +- `index`: la posizione numerica nel vettore di chunk — passala direttamente a `/chunks/update?index=...` +- `score`: `null` senza query, float 0-1 con ordinamento per rilevanza + +#### 🆙 Aggiornare un singolo chunk + +Usa `/chunks/update` quando devi modificare **solo un pezzetto specifico** (es. correggere un errore di battitura, cambiare metadata). Per aggiornare **un intero documento** usa invece `/upsert/document`. + +```bash +# Opzione A: aggiorna tramite `index` (da list_chunks → campo "index": 9) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \ + -H 'Content-Type: application/json' \ + -d '{ + "index": 9, + "new_text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2028 (2 años). Descuento 20% acordado en la llamada del 10/08.", + "new_metadata": {"cliente_id": 456, "categoria": "crm", "ultima_revision": "2026-08-26"} + }' | jq + +# Opzione B: aggiorna tramite (source, chunk_index) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/chunks/update \ + -H 'Content-Type: application/json' \ + -d '{ + "source": "crm/cliente_456_nota_20260826", + "chunk_index": 1000009, + "new_source": "crm/cliente_456/nota_principal" + }' | jq +``` + +Risposta successo `200`: +```json +{ + "updated": 1, + "chunk": { + "index": 9, + "source": "crm/cliente_456/nota_principal", + "chunk_index": 1000009, + "text": "Cliente: Javier García...", + "metadata": {"cliente_id": 456, "...": "..."}, + "is_dynamic": true + } +} +``` + +Errori HTTP possibili: +- `404 CHUNK_NOT_FOUND` +- `409 AMBIGUOUS_SOURCE` → hai passato `source` da sola ma esistono N chunk con stessa source; aggiungi `chunk_index` oppure usa `index` + +#### 🔄 Upsert documento (CRUD consigliato) + +**Questo è l'endpoint che userai nel 90% dei casi.** +Idempotente sul campo `source`: +- Se la `source` non esiste → CREATE +- Se la `source` esiste → DELETE tutti i chunk vecchi + split del nuovo testo + INSERT +- Se mandi `text: ""` → DELETE atomico (nessun re-insert) + +```bash +# CREATE/UPDATE di un documento intero (es. nota cliente) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \ + -H 'Content-Type: application/json' \ + -d '{ + "source": "crm/cliente_456/perfil", + "text": "Javier García — ID 456. Plan Premium Anual, renovación automática 15/02/2028. Precio: 24€/mes. Email javier@empresa.es. Tel +34 600 111 222. Dirección envío habitual: Av. Diagonal 444, puerta 3, 08013 Barcelona. Notas: siempre prefiere atención por la mañana, antes de las 11h. Caso abierto 2026-08-0124: seguimiento calidad postventa, cerrado con valoración 5/5.", + "metadata": {"cliente_id": 456, "pais": "es", "actualizado": "2026-08-26"}, + "dynamic": true, + "persist": true + }' | jq +``` + +Risposta: +```json +{ + "removed": 2, + "added": 1, + "sources": ["crm/cliente_456/perfil"], + "total_after": 10, + "mode": "upsert" +} +``` + +DELETE tramite upsert: +```bash +# Elimina del tutto tutti i chunk associati a questa source (atomico) +curl -sS -X POST http://127.0.0.1:12345/v1/rag/upsert/document \ + -H 'Content-Type: application/json' \ + -d '{"source": "crm/cliente_456/perfil", "text": ""}' | jq +``` + + +#### 🔄 Ricaricare tutta la KB da file (dopo aver copiato nuovi md/txt) + +```bash +curl -sS -X POST http://127.0.0.1:12345/v1/rag/reload \ + -H 'Content-Type: application/json' \ + -d '{"force_rebuild": true}' | jq +``` + +--- + +### 13.3 Persistenza cross-riavvio: `kb/_dynamic.jsonl` + +Tutti i chunk aggiunti via API con `persist: true` vengono appesi a `kb/_dynamic.jsonl` (un JSON per riga). Al prossimo avvio della pipeline: +1. Viene ricostruito l'indice statico da file md/txt/jsonl +2. **Subito dopo** vengono ricaricati e ri-embeddati anche tutti i chunk da `_dynamic.jsonl` + +Quindi la conoscenza aggiunta via HTTP è **durevole** e non si perde riavviando. + +--- + +### 13.4 Thread safety + +Tutte le API usano un `threading.RLock()` all'interno di [RAGRetriever](file:///home/azurian/speech-to-speech/src/speech_to_speech/RAG/retriever.py): +- ✅ Più chiamate `/search` possono andare in parallelo durante la conversazione +- ✅ Una `add_document`/`remove` blocca brevemente l'indice solo per le operazioni di vstack/take numpy — retrieval non si perde, viene atteso +- ✅ Persistenza su disco (NPZ + JSONL) avviene fuori dal lock, quindi non rallenta la chiamata conversazionale + +--- + +## 14. Roadmap / miglioramenti possibili + +- [ ] **Supporto PDF**: aggiungere `pypdf` o `pdfplumber` in gruppo opzionale +- [ ] **MMR reranking** invece del solo top-k coseno (migliore diversità chunk) +- [ ] **HyDE** (LLM genera query ipotetica + embedding di quella, per query colloquiali) +- [ ] **Cross-encoder reranker** 2-stage per KB grandi (>10k chunk) +- [ ] **Hook su LanguageModelHandler locale** (mlx-lm / transformers) — attualmente l'RAG è attivo solo per backends `chat-completions` e `responses-api` +- [ ] **Watchdog KB**: auto-rebuild indice quando i file cambiano (inotify / watchdog) diff --git a/docs/assets/endpoint-swap-dark.gif b/docs/assets/endpoint-swap-dark.gif new file mode 100644 index 0000000..43d57dd Binary files /dev/null and b/docs/assets/endpoint-swap-dark.gif differ diff --git a/docs/assets/endpoint-swap-light.gif b/docs/assets/endpoint-swap-light.gif new file mode 100644 index 0000000..4fd2578 Binary files /dev/null and b/docs/assets/endpoint-swap-light.gif differ diff --git a/female_short.wav b/female_short.wav new file mode 100644 index 0000000..370b19c Binary files /dev/null and b/female_short.wav differ diff --git a/kb/01_faq_producto.md b/kb/01_faq_producto.md new file mode 100644 index 0000000..5a99672 --- /dev/null +++ b/kb/01_faq_producto.md @@ -0,0 +1,33 @@ +# Preguntas Frecuentes - Producto (Ejemplo) + +> Este es un archivo de ejemplo. Sustitúyelo con tus documentos reales. + +## ¿Cómo puedo restablecer mi contraseña? + +Para restablecer tu contraseña ve a la página de inicio de sesión y haz clic en +*"Olvidé mi contraseña"*. Recibirás un correo electrónico con un enlace válido +durante 15 minutos. Si no recibes el correo, revisa la carpeta de spam o ponte en +contacto con soporte en soporte@empresa.es. + +## ¿Cuál es el horario de atención al cliente? + +El horario de atención telefónica y por chat es de **lunes a viernes de 09:00 a 18:00** +(hora peninsular española). Los fines de semana solo hay soporte por email con +respuesta máxima en 24 horas. + +## ¿Qué métodos de pago aceptáis? + +Aceptamos Visa, Mastercard, American Express, PayPal, transferencia bancaria y +Bizum. Para clientes empresariales disponemos de pago a 30 días tras aprobación +del crédito. + +## ¿Ofrecéis reembolsos? + +Sí. Todos los planes tienen un periodo de devolución de **14 días naturales** +desde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos +se evalúan caso por caso por el equipo de soporte. + +## ¿En qué idiomas está disponible la plataforma? + +La interfaz web está disponible en español, inglés, francés, italiano y portugués. +El soporte por chat se ofrece únicamente en español e inglés de momento. diff --git a/kb/02_politicas_internas.md b/kb/02_politicas_internas.md new file mode 100644 index 0000000..e1bec57 --- /dev/null +++ b/kb/02_politicas_internas.md @@ -0,0 +1,31 @@ +# Políticas Internas - Ejemplo + +> Este es un archivo de ejemplo. Elíminalo o sustitúyelo por tus políticas reales. + +## Política de privacidad de datos de clientes + +Todos los datos de los clientes se almacenan cifrados en repositorios UE (Irlanda +y Frankfurt) según el Reglamento General de Protección de Datos (RGPD). Los datos +de pago nunca se almacenan en nuestros servidores; son procesados directamente por +nuestro proveedor PCI-DSS certificado (Stripe). + +## Retención de registros + +- Datos de usuario activo: mientras dure la relación contractual + 1 mes. +- Datos de usuario dado de baja: 6 meses (obligación fiscal). +- Registros de auditoría y acceso: 2 años. +- Grabaciones de llamadas de soporte: 90 días, salvo controversia judicial. + +## Niveles de servicio (SLA) + +El servicio tiene un **SLA de disponibilidad del 99,9% mensual** para clientes del +plan Pro y Enterprise. Si no se cumple, se aplica un crédito proporcional del 10% +sobre la cuota mensual. Para incidentes de severidad 1 (servicio caído) el tiempo +medio de respuesta es inferior a 1 hora en horario comercial. + +## Seguridad: autenticación y sesiones + +- Todos los usuarios deben usar autenticación en dos pasos (2FA) obligatoria. +- Las sesiones web expiran a los 60 minutos de inactividad. +- Contraseñas: mínimo 12 caracteres, 1 mayúscula, 1 número, 1 símbolo. Rotación + recomendada cada 90 días. diff --git a/kb/README.md b/kb/README.md new file mode 100644 index 0000000..b54eed8 --- /dev/null +++ b/kb/README.md @@ -0,0 +1,19 @@ +# Base de Conocimiento - Knowledge Base (Scaffold de ejemplo) + +Coloca aquí tus documentos en formato **Markdown (.md)** o **Texto plano (.txt)**. +Se prefieres pre-chunkizar, puedes usar archivos **JSONL** (una línea por chunk): + +```jsonl +{"text": "Texto del chunk 1...", "source": "manual_usuario.pdf", "chunk_index": 0, "metadata": {"pagina": 12}} +{"text": "Texto del chunk 2...", "source": "manual_usuario.pdf", "chunk_index": 1, "metadata": {"pagina": 13}} +``` + +## Primera ejecución + +1. Añade tus documentos a esta carpeta (subcarpetas permitidas). +2. Arranca la pipeline con la flag `--rag_enabled`. +3. Se generarán automáticamente: + - `_index.npz` → matriz embeddings (numpy, persistente) + - `_chunks.jsonl` → lista chunks con metadata + +Para **reindexar** después de añadir documentos nuevos: usa `--rag_force_rebuild` o borra manualmente `_index.npz` y `_chunks.jsonl`. diff --git a/kb/_chunks.jsonl b/kb/_chunks.jsonl new file mode 100644 index 0000000..fb33b2a --- /dev/null +++ b/kb/_chunks.jsonl @@ -0,0 +1,11 @@ +{"text": "# Preguntas Frecuentes - Producto (Ejemplo)\n\n> Este es un archivo de ejemplo. Sustitúyelo con tus documentos reales.\n\n## ¿Cómo puedo restablecer mi contraseña?\n\nPara restablecer tu contraseña ve a la página de inicio de sesión y haz clic en\n*\"Olvidé mi contraseña\"*. Recibirás un correo electrónico con un enlace válido\ndurante 15 minutos. Si no recibes el correo, revisa la carpeta de spam o ponte en\ncontacto con soporte en soporte@empresa.es.\n\n## ¿Cuál es el horario de atención al cliente?", "source": "01_faq_producto.md", "chunk_index": 0, "metadata": {"splitted": true}} +{"text": "orte@empresa.es.\n\n## ¿Cuál es el horario de atención al cliente?El horario de atención telefónica y por chat es de **lunes a viernes de 09:00 a 18:00**\n(hora peninsular española). Los fines de semana solo hay soporte por email con\nrespuesta máxima en 24 horas.\n\n## ¿Qué métodos de pago aceptáis?\n\nAceptamos Visa, Mastercard, American Express, PayPal, transferencia bancaria y\nBizum. Para clientes empresariales disponemos de pago a 30 días tras aprobación\ndel crédito.\n\n## ¿Ofrecéis reembolsos?", "source": "01_faq_producto.md", "chunk_index": 1, "metadata": {"splitted": true}} +{"text": "a 30 días tras aprobación\ndel crédito.\n\n## ¿Ofrecéis reembolsos?Sí. Todos los planes tienen un periodo de devolución de **14 días naturales**\ndesde la compra, sin necesidad de justificación. Pasado ese plazo, los reembolsos\nse evalúan caso por caso por el equipo de soporte.\n\n## ¿En qué idiomas está disponible la plataforma?\n\nLa interfaz web está disponible en español, inglés, francés, italiano y portugués.\nEl soporte por chat se ofrece únicamente en español e inglés de momento.", "source": "01_faq_producto.md", "chunk_index": 2, "metadata": {"splitted": true}} +{"text": "# Políticas Internas - Ejemplo\n\n> Este es un archivo de ejemplo. Elíminalo o sustitúyelo por tus políticas reales.\n\n## Política de privacidad de datos de clientes\n\nTodos los datos de los clientes se almacenan cifrados en repositorios UE (Irlanda\ny Frankfurt) según el Reglamento General de Protección de Datos (RGPD). Los datos\nde pago nunca se almacenan en nuestros servidores; son procesados directamente por\nnuestro proveedor PCI-DSS certificado (Stripe).\n\n## Retención de registros", "source": "02_politicas_internas.md", "chunk_index": 0, "metadata": {"splitted": true}} +{"text": "oveedor PCI-DSS certificado (Stripe).\n\n## Retención de registros- Datos de usuario activo: mientras dure la relación contractual + 1 mes.\n- Datos de usuario dado de baja: 6 meses (obligación fiscal).\n- Registros de auditoría y acceso: 2 años.\n- Grabaciones de llamadas de soporte: 90 días, salvo controversia judicial.\n\n## Niveles de servicio (SLA)", "source": "02_politicas_internas.md", "chunk_index": 1, "metadata": {"splitted": true}} +{"text": "días, salvo controversia judicial.\n\n## Niveles de servicio (SLA)El servicio tiene un **SLA de disponibilidad del 99,9% mensual** para clientes del\nplan Pro y Enterprise. Si no se cumple, se aplica un crédito proporcional del 10%\nsobre la cuota mensual. Para incidentes de severidad 1 (servicio caído) el tiempo\nmedio de respuesta es inferior a 1 hora en horario comercial.\n\n## Seguridad: autenticación y sesiones", "source": "02_politicas_internas.md", "chunk_index": 2, "metadata": {"splitted": true}} +{"text": "ra en horario comercial.\n\n## Seguridad: autenticación y sesiones- Todos los usuarios deben usar autenticación en dos pasos (2FA) obligatoria.\n- Las sesiones web expiran a los 60 minutos de inactividad.\n- Contraseñas: mínimo 12 caracteres, 1 mayúscula, 1 número, 1 símbolo. Rotación\n recomendada cada 90 días.", "source": "02_politicas_internas.md", "chunk_index": 3, "metadata": {"splitted": true}} +{"text": "# Base de Conocimiento - Knowledge Base (Scaffold de ejemplo)\n\nColoca aquí tus documentos en formato **Markdown (.md)** o **Texto plano (.txt)**.\nSe prefieres pre-chunkizar, puedes usar archivos **JSONL** (una línea por chunk):\n\n```jsonl\n{\"text\": \"Texto del chunk 1...\", \"source\": \"manual_usuario.pdf\", \"chunk_index\": 0, \"metadata\": {\"pagina\": 12}}\n{\"text\": \"Texto del chunk 2...\", \"source\": \"manual_usuario.pdf\", \"chunk_index\": 1, \"metadata\": {\"pagina\": 13}}\n```\n\n## Primera ejecución", "source": "README.md", "chunk_index": 0, "metadata": {"splitted": true}} +{"text": "index\": 1, \"metadata\": {\"pagina\": 13}}\n```\n\n## Primera ejecución1. Añade tus documentos a esta carpeta (subcarpetas permitidas).\n2. Arranca la pipeline con la flag `--rag_enabled`.\n3. Se generarán automáticamente:\n - `_index.npz` → matriz embeddings (numpy, persistente)\n - `_chunks.jsonl` → lista chunks con metadata\n\nPara **reindexar** después de añadir documentos nuevos: usa `--rag_force_rebuild` o borra manualmente `_index.npz` y `_chunks.jsonl`.", "source": "README.md", "chunk_index": 1, "metadata": {"splitted": true}} +{"text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2027. Tel +34 600 111 222. Incidente factura 2026-08-114 resuelto con 15% descuento.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000009, "metadata": {"cliente_id": 456, "categoria": "crm", "_dynamic": true}} +{"text": "Cliente: Pippo Pappo, id=123. Falta pago rata mensual.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000010, "metadata": {"cliente_id": 123, "categoria": "crm", "_dynamic": true}} diff --git a/kb/_dynamic.jsonl b/kb/_dynamic.jsonl new file mode 100644 index 0000000..984cfff --- /dev/null +++ b/kb/_dynamic.jsonl @@ -0,0 +1,2 @@ +{"text": "Cliente: Javier García, id=456. Plan Premium Anual, renovación 15/02/2027. Tel +34 600 111 222. Incidente factura 2026-08-114 resuelto con 15% descuento.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000009, "metadata": {"cliente_id": 456, "categoria": "crm", "_dynamic": true}} +{"text": "Cliente: Pippo Pappo, id=123. Falta pago rata mensual.", "source": "crm/cliente_456_nota_20260826", "chunk_index": 1000010, "metadata": {"cliente_id": 123, "categoria": "crm", "_dynamic": true}} diff --git a/kb/_index.npz b/kb/_index.npz new file mode 100644 index 0000000..883794f Binary files /dev/null and b/kb/_index.npz differ diff --git a/logo.png b/logo.png new file mode 100644 index 0000000..bc79ea9 Binary files /dev/null and b/logo.png differ diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..976ba02 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..070f361 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,146 @@ +[build-system] +requires = ["setuptools>=77.0.3", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "speech-to-speech" +version = "0.2.11" +description = "Low-latency speech-to-speech pipeline" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +license-files = ["LICENSE"] +authors = [ + { name = "Hugging Face" }, +] +keywords = ["speech-to-speech", "voice-agents", "speech-recognition", "text-to-speech", "openai"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Multimedia :: Sound/Audio :: Speech", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [ + "fastapi>=0.115.0", + "httpx>=0.28.0", + "nltk==3.9.4", + "numpy>=1.26.0,<2.4.4; platform_system == 'Darwin'", + "numpy>=1.26.0; platform_system != 'Darwin'", + "openai==2.28.0", + "pillow>=10.0.0", + "pydantic>=2.0", + "rich>=13.0", + "scipy>=1.10.0", + "sounddevice==0.5.3; platform_system == 'Darwin'", + "sounddevice>=0.5.0; platform_system != 'Darwin'", + "soundfile>=0.13.0; platform_system == 'Darwin'", + "torch==2.11.0; platform_system == 'Darwin'", + "torch>=2.4.0; platform_system != 'Darwin'", + "torchaudio==2.11.0; platform_system == 'Darwin'", + "torchaudio>=2.4.0; platform_system != 'Darwin'", + "transformers==5.6.2; platform_system == 'Darwin'", + "transformers>=4.57.0; platform_system != 'Darwin'", + "uvicorn>=0.30.0", + "websockets>=12.0", + "nano-parakeet>=0.2.0; platform_system != 'Darwin'", + "faster-qwen3-tts[ggml]>=0.3.2; platform_system != 'Darwin' and platform_system != 'Windows'", + "faster-qwen3-tts>=0.3.2; platform_system == 'Windows'", + "lingua-language-detector>=2.0.2", + "miniaudio==1.61; platform_system == 'Darwin'", + "mlx==0.31.1; platform_system == 'Darwin'", + "mlx-audio==0.4.2; platform_system == 'Darwin'", + "mlx-lm==0.31.1; platform_system == 'Darwin'", + "mlx-metal==0.31.1; platform_system == 'Darwin'", + "misaki>=0.9.4; platform_system == 'Darwin'", + "espeakng-loader>=0.2.4; platform_system == 'Darwin'", + "spacy>=3.8.4; platform_system == 'Darwin'", + "phonemizer-fork>=3.3.2; platform_system == 'Darwin'", +] + +[project.optional-dependencies] +chattts = [ + "ChatTTS>=0.1.1", +] +facebook-mms = [ + "transformers>=4.57.0", +] +faster-whisper = [ + "faster-whisper>=1.0.3", +] +kokoro = [ + "kokoro>=0.9.2; platform_system != 'Darwin'", +] +language-detection = [ + "lingua-language-detector>=2.0.2", +] +mlx-lm = [ + "mlx-lm==0.31.1; platform_system == 'Darwin'", + "mlx-vlm==0.4.1; platform_system == 'Darwin'", +] +paraformer = [ + "funasr>=1.1.6", + "modelscope>=1.17.1", + "onnxruntime<1.24; python_version < '3.11'", +] +pocket = [ + "pocket-tts>=0.1.0", +] +rag = [ + "sentence-transformers>=3.0.0", +] +websocket = [ + "websockets>=12.0", +] +whisper-mlx = [ + "lightning-whisper-mlx>=0.0.10; platform_system == 'Darwin'", +] + +[project.urls] +Homepage = "https://github.com/huggingface/speech-to-speech" +Repository = "https://github.com/huggingface/speech-to-speech" +Issues = "https://github.com/huggingface/speech-to-speech/issues" + +[project.scripts] +speech-to-speech = "speech_to_speech.s2s_pipeline:main" + +[dependency-groups] +dev = [ + "ruff", + "mypy", + "pytest", + "pytest-asyncio", + "websockets>=12.0", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" + +[tool.uv] +package = true +prerelease = "allow" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["speech_to_speech*"] +exclude = ["tests*"] + +[tool.setuptools.package-data] +speech_to_speech = ["TTS/*.wav"] + +[tool.ruff] +src = ["src"] +line-length = 120 + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +ignore_missing_imports = true +warn_unused_configs = true +check_untyped_defs = false diff --git a/scripts/benchmark_stt.py b/scripts/benchmark_stt.py new file mode 100644 index 0000000..74ecbd8 --- /dev/null +++ b/scripts/benchmark_stt.py @@ -0,0 +1,385 @@ +""" +STT Benchmarking Script + +This script benchmarks different Speech-to-Text (STT) handlers to compare their performance. +Measures: inference time, warmup time, memory usage, and transcription quality. + +Usage: + python benchmark_stt.py --audio_file path/to/audio.wav --iterations 10 + python benchmark_stt.py --audio_file path/to/audio.wav --handlers whisper mlx-audio-whisper +""" + +import argparse +import json +import logging +import time +from pathlib import Path +from queue import Queue +from threading import Event +from typing import Any, Dict, List, Optional + +import numpy as np +import soundfile as sf + +from speech_to_speech.pipeline.messages import VADAudio + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +class BenchmarkResult: + """Stores benchmark results for a single STT handler.""" + + def __init__(self, handler_name: str): + self.handler_name = handler_name + self.warmup_time = 0.0 + self.inference_times: list[float] = [] + self.time_to_first_token: list[float] = [] + self.transcriptions: list[str] = [] + self.errors: list[str] = [] + + def add_inference(self, time_taken: float, transcription: Any, ttft: Optional[float] = None): + self.inference_times.append(time_taken) + self.transcriptions.append(transcription) + if ttft is not None: + self.time_to_first_token.append(ttft) + + def add_error(self, error: str): + self.errors.append(error) + + def get_stats(self) -> Dict[str, Any]: + """Calculate statistics from benchmark results.""" + if not self.inference_times: + return { + "handler": self.handler_name, + "status": "failed", + "errors": self.errors, + } + + stats = { + "handler": self.handler_name, + "warmup_time": self.warmup_time, + "avg_inference_time": np.mean(self.inference_times), + "min_inference_time": np.min(self.inference_times), + "max_inference_time": np.max(self.inference_times), + "std_inference_time": np.std(self.inference_times), + "total_iterations": len(self.inference_times), + "errors": self.errors, + "sample_transcription": self.transcriptions[0] if self.transcriptions else None, + } + + # Add time to first token stats if available + if self.time_to_first_token: + stats["avg_time_to_first_token"] = np.mean(self.time_to_first_token) + stats["min_time_to_first_token"] = np.min(self.time_to_first_token) + stats["max_time_to_first_token"] = np.max(self.time_to_first_token) + stats["std_time_to_first_token"] = np.std(self.time_to_first_token) + + return stats + + +def load_audio(audio_path: str) -> np.ndarray: + """Load audio file and return as numpy array.""" + logger.info(f"Loading audio from: {audio_path}") + audio, sample_rate = sf.read(audio_path) + + # Convert to mono if stereo + if len(audio.shape) > 1: + audio = audio.mean(axis=1) + + # Resample to 16kHz if needed (most whisper models expect 16kHz) + if sample_rate != 16000: + logger.warning(f"Audio sample rate is {sample_rate}Hz, resampling to 16000Hz") + try: + import librosa + audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000) + except ImportError: + logger.error("librosa not installed. Please install it with: pip install librosa") + logger.error("Attempting scipy resampling as fallback...") + from scipy import signal + # Calculate resampling ratio + num_samples = int(len(audio) * 16000 / sample_rate) + audio = signal.resample(audio, num_samples) + + return audio.astype(np.float32) + + +def benchmark_handler( + handler_name: str, + audio: np.ndarray, + iterations: int, + handler_kwargs: Optional[Dict[str, Any]] = None +) -> BenchmarkResult: + """Benchmark a single STT handler.""" + logger.info(f"Benchmarking {handler_name}...") + result = BenchmarkResult(handler_name) + + try: + # Create queues and events for handler + stop_event = Event() + queue_in: Queue[Any] = Queue() + queue_out: Queue[Any] = Queue() + + handler: Any = None + if handler_name == "whisper": + from speech_to_speech.STT.whisper_stt_handler import WhisperSTTHandler + setup_kwargs = handler_kwargs or { + "model_name": "distil-whisper/distil-large-v3", + "device": "cuda", + "torch_dtype": "float16", + } + handler = WhisperSTTHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_kwargs=setup_kwargs + ) + + elif handler_name == "whisper-mlx": + from speech_to_speech.STT.lightning_whisper_mlx_handler import LightningWhisperSTTHandler + setup_kwargs = handler_kwargs or { + "model_name": "large-v3", + "device": "mps", + } + handler = LightningWhisperSTTHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_kwargs=setup_kwargs + ) + + elif handler_name == "mlx-audio-whisper": + from speech_to_speech.STT.mlx_audio_whisper_handler import MLXAudioWhisperSTTHandler + setup_kwargs = handler_kwargs or { + "model_name": "mlx-community/whisper-large-v3-turbo", + } + handler = MLXAudioWhisperSTTHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_kwargs=setup_kwargs + ) + + elif handler_name == "faster-whisper": + from speech_to_speech.STT.faster_whisper_handler import FasterWhisperSTTHandler + setup_kwargs = handler_kwargs or { + "model_name": "large-v3", + "device": "auto", + "compute_type": "float16", + } + handler = FasterWhisperSTTHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_kwargs=setup_kwargs + ) + + elif handler_name == "moonshine": + from archive.STT.moonshine_handler import MoonshineSTTHandler + handler = MoonshineSTTHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + ) + + elif handler_name == "parakeet-tdt": + from speech_to_speech.STT.parakeet_tdt_handler import ParakeetTDTSTTHandler + setup_kwargs = handler_kwargs or { + "device": "mps", + "enable_live_transcription": False, + } + handler = ParakeetTDTSTTHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_kwargs=setup_kwargs + ) + + elif handler_name == "parakeet-tdt-progressive": + from speech_to_speech.STT.parakeet_tdt_handler import ParakeetTDTSTTHandler + setup_kwargs = handler_kwargs or { + "device": "mps", + "enable_live_transcription": True, + "live_transcription_update_interval": 0.25, + } + handler = ParakeetTDTSTTHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_kwargs=setup_kwargs + ) + else: + raise ValueError(f"Unknown handler: {handler_name}") + + # Warmup is done in handler setup + logger.info(f"Handler {handler_name} initialized and warmed up") + + # Additional warmup on the actual audio (excluded from timings) + for _ in handler.process(VADAudio(audio=audio)): + pass + + # Run benchmark iterations + for i in range(iterations): + logger.info(f"Iteration {i+1}/{iterations} for {handler_name}") + + start_time = time.perf_counter() + + # Process audio + transcription = None + time_to_first_token = None + first_output = True + + for output in handler.process(VADAudio(audio=audio)): + # Measure time to first token + if first_output: + time_to_first_token = time.perf_counter() - start_time + first_output = False + + if isinstance(output, tuple): + transcription = output[0] # (text, language) + else: + transcription = output + + end_time = time.perf_counter() + + time_taken = end_time - start_time + result.add_inference(time_taken, transcription, time_to_first_token) + + ttft_str = f", TTFT: {time_to_first_token:.4f}s" if time_to_first_token else "" + text_preview = str(transcription)[:50] if transcription is not None else "(none)" + logger.info(f" Time: {time_taken:.4f}s{ttft_str}, Text: {text_preview}...") + + # Cleanup + handler.cleanup() + stop_event.set() + + except Exception as e: + logger.error(f"Error benchmarking {handler_name}: {e}", exc_info=True) + result.add_error(str(e)) + + return result + + +def print_results(results: List[BenchmarkResult]): + """Print benchmark results in a formatted table.""" + print("\n" + "="*80) + print("BENCHMARK RESULTS") + print("="*80) + + for result in results: + stats = result.get_stats() + print(f"\nHandler: {stats['handler']}") + print("-" * 80) + + if stats.get("status") == "failed": + print(" Status: FAILED") + print(f" Errors: {stats['errors']}") + continue + + print(f" Warmup Time: {stats['warmup_time']:.4f}s") + print(f" Avg Inference Time: {stats['avg_inference_time']:.4f}s") + print(f" Min Inference Time: {stats['min_inference_time']:.4f}s") + print(f" Max Inference Time: {stats['max_inference_time']:.4f}s") + print(f" Std Deviation: {stats['std_inference_time']:.4f}s") + + # Print time to first token stats if available + if 'avg_time_to_first_token' in stats: + print("\n Time to First Token:") + print(f" Avg TTFT: {stats['avg_time_to_first_token']:.4f}s") + print(f" Min TTFT: {stats['min_time_to_first_token']:.4f}s") + print(f" Max TTFT: {stats['max_time_to_first_token']:.4f}s") + print(f" Std TTFT: {stats['std_time_to_first_token']:.4f}s") + + print(f"\n Total Iterations: {stats['total_iterations']}") + print(f" Sample Transcription: {stats['sample_transcription']}") + + if stats['errors']: + print(f" Errors: {stats['errors']}") + + # Comparison table + print("\n" + "="*80) + print("COMPARISON (Average Inference Time)") + print("="*80) + + successful_results = [r for r in results if r.inference_times] + if successful_results: + sorted_results = sorted(successful_results, key=lambda x: np.mean(x.inference_times)) + + fastest = sorted_results[0] + fastest_time = np.mean(fastest.inference_times) + + for result in sorted_results: + avg_time = np.mean(result.inference_times) + speedup = avg_time / fastest_time + print(f" {result.handler_name:25s}: {avg_time:.4f}s ({speedup:.2f}x slower than fastest)") + + +def save_results(results: List[BenchmarkResult], output_file: str): + """Save benchmark results to JSON file.""" + data = { + "results": [r.get_stats() for r in results], + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + } + + with open(output_file, 'w') as f: + json.dump(data, f, indent=2) + + logger.info(f"Results saved to: {output_file}") + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark STT handlers") + parser.add_argument( + "--audio_file", + type=str, + required=True, + help="Path to audio file for benchmarking" + ) + parser.add_argument( + "--handlers", + nargs="+", + default=["whisper", "whisper-mlx", "mlx-audio-whisper", "faster-whisper", "parakeet-tdt", "parakeet-tdt-progressive"], + help="List of handlers to benchmark (default: all)" + ) + parser.add_argument( + "--iterations", + type=int, + default=5, + help="Number of iterations per handler (default: 5)" + ) + parser.add_argument( + "--output", + type=str, + default="stt_benchmark_results.json", + help="Output JSON file for results (default: stt_benchmark_results.json)" + ) + + args = parser.parse_args() + + # Validate audio file exists + if not Path(args.audio_file).exists(): + logger.error(f"Audio file not found: {args.audio_file}") + return + + # Load audio + audio = load_audio(args.audio_file) + logger.info(f"Audio loaded: {len(audio)} samples, {len(audio)/16000:.2f}s duration") + + # Run benchmarks + results = [] + for handler_name in args.handlers: + result = benchmark_handler(handler_name, audio, args.iterations) + results.append(result) + + # Print and save results + print_results(results) + save_results(results, args.output) + + logger.info("Benchmarking complete!") + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_tts.py b/scripts/benchmark_tts.py new file mode 100644 index 0000000..dc8490f --- /dev/null +++ b/scripts/benchmark_tts.py @@ -0,0 +1,395 @@ +""" +TTS Benchmarking Script + +Benchmarks Text-to-Speech (TTS) handlers to compare performance. +Measures: warmup time, inference time, time-to-first-chunk, audio duration, and RTF. + +Usage: + python benchmark_tts.py --text "Hello world" --iterations 3 + python benchmark_tts.py --handlers kokoro qwen3 pocket_tts +""" + +import argparse +import json +import logging +import time +from queue import Queue +from threading import Event +from typing import Any, Dict, List, Optional + +import numpy as np + +from speech_to_speech.pipeline.messages import TTSInput + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + +DEFAULT_SAMPLE_RATE = 16000 +VALID_QWEN3_MLX_QUANTIZATIONS = ("bf16", "4bit", "6bit", "8bit") + + +class BenchmarkResult: + """Stores benchmark results for a single TTS handler.""" + + def __init__(self, handler_name: str): + self.handler_name = handler_name + self.warmup_time = 0.0 + self.inference_times: list[float] = [] + self.time_to_first_chunk: list[float] = [] + self.audio_durations: list[float] = [] + self.errors: list[str] = [] + + def add_inference(self, time_taken: float, audio_duration: float, ttfc: Optional[float] = None): + self.inference_times.append(time_taken) + self.audio_durations.append(audio_duration) + if ttfc is not None: + self.time_to_first_chunk.append(ttfc) + + def add_error(self, error: str): + self.errors.append(error) + + def get_stats(self) -> Dict[str, Any]: + if not self.inference_times: + return { + "handler": self.handler_name, + "status": "failed", + "errors": self.errors, + } + + avg_time = float(np.mean(self.inference_times)) + avg_audio = float(np.mean(self.audio_durations)) + avg_rtf = avg_audio / avg_time if avg_time > 0 else 0.0 + + stats = { + "handler": self.handler_name, + "warmup_time": self.warmup_time, + "avg_inference_time": avg_time, + "min_inference_time": float(np.min(self.inference_times)), + "max_inference_time": float(np.max(self.inference_times)), + "std_inference_time": float(np.std(self.inference_times)), + "avg_audio_duration": avg_audio, + "min_audio_duration": float(np.min(self.audio_durations)), + "max_audio_duration": float(np.max(self.audio_durations)), + "std_audio_duration": float(np.std(self.audio_durations)), + "avg_rtf": avg_rtf, + "total_iterations": len(self.inference_times), + "errors": self.errors, + } + + if self.time_to_first_chunk: + stats["avg_time_to_first_chunk"] = float(np.mean(self.time_to_first_chunk)) + stats["min_time_to_first_chunk"] = float(np.min(self.time_to_first_chunk)) + stats["max_time_to_first_chunk"] = float(np.max(self.time_to_first_chunk)) + stats["std_time_to_first_chunk"] = float(np.std(self.time_to_first_chunk)) + + return stats + + +def benchmark_handler( + handler_name: str, + text: str, + iterations: int, + handler_kwargs: Optional[Dict[str, Any]] = None, + language_code: Optional[str] = "en", +) -> BenchmarkResult: + logger.info(f"Benchmarking {handler_name}...") + result = BenchmarkResult(handler_name) + + try: + stop_event = Event() + should_listen = Event() + queue_in: Queue[Any] = Queue() + queue_out: Queue[Any] = Queue() + + handler: Any = None + setup_kwargs = handler_kwargs or {} + + start_setup = time.perf_counter() + + if handler_name == "kokoro": + from speech_to_speech.TTS.kokoro_handler import KokoroTTSHandler + setup_kwargs = {"device": "auto", **setup_kwargs} + handler = KokoroTTSHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_args=(should_listen,), + setup_kwargs=setup_kwargs, + ) + elif handler_name == "pocket_tts": + from speech_to_speech.TTS.pocket_tts_handler import PocketTTSHandler + setup_kwargs = {"device": "cpu", **setup_kwargs} + handler = PocketTTSHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_args=(should_listen,), + setup_kwargs=setup_kwargs, + ) + elif handler_name == "qwen3": + from speech_to_speech.TTS.qwen3_tts_handler import Qwen3TTSHandler + setup_kwargs = { + "device": "cuda", + "model_name": "Qwen/Qwen3-TTS-12Hz-0.6B-Base", + "ref_audio": "TTS/ref_audio.wav", + **setup_kwargs, + } + handler = Qwen3TTSHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_args=(should_listen,), + setup_kwargs=setup_kwargs, + ) + elif handler_name == "chatTTS": + from speech_to_speech.TTS.chatTTS_handler import ChatTTSHandler + setup_kwargs = {"device": "cuda", **setup_kwargs} + handler = ChatTTSHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_args=(should_listen,), + setup_kwargs=setup_kwargs, + ) + elif handler_name == "facebookMMS": + from speech_to_speech.TTS.facebookmms_handler import FacebookMMSTTSHandler + setup_kwargs = {"device": "cuda", "language": "en", **setup_kwargs} + handler = FacebookMMSTTSHandler( + stop_event, + queue_in=queue_in, + queue_out=queue_out, + setup_args=(should_listen,), + setup_kwargs=setup_kwargs, + ) + else: + raise ValueError(f"Unknown handler: {handler_name}") + + result.warmup_time = time.perf_counter() - start_setup + logger.info(f"Handler {handler_name} initialized and warmed up in {result.warmup_time:.3f}s") + + for i in range(iterations): + logger.info(f"Iteration {i+1}/{iterations} for {handler_name}") + start_time = time.perf_counter() + time_to_first_chunk = None + first_output = True + total_samples = 0 + + tts_input = TTSInput(text=text, language_code=language_code) + for chunk in handler.process(tts_input): + if first_output: + time_to_first_chunk = time.perf_counter() - start_time + first_output = False + if chunk is None: + continue + try: + total_samples += len(chunk) + except Exception: + pass + + end_time = time.perf_counter() + time_taken = end_time - start_time + audio_duration = total_samples / DEFAULT_SAMPLE_RATE if total_samples > 0 else 0.0 + + result.add_inference(time_taken, audio_duration, time_to_first_chunk) + ttfc_str = f", TTFC: {time_to_first_chunk:.4f}s" if time_to_first_chunk else "" + logger.info( + f" Time: {time_taken:.4f}s{ttfc_str}, Audio: {audio_duration:.2f}s, RTF: {audio_duration / time_taken if time_taken > 0 else 0:.2f}" + ) + + handler.cleanup() + stop_event.set() + + except Exception as e: + logger.error(f"Error benchmarking {handler_name}: {e}", exc_info=True) + result.add_error(str(e)) + + return result + + +def normalize_qwen3_mlx_quantizations(values: List[str] | None) -> List[str]: + if not values: + return [] + + normalized = [] + seen = set() + for value in values: + quantization = str(value).strip().lower() + if quantization in ("default", "none", ""): + quantization = "bf16" + if quantization not in VALID_QWEN3_MLX_QUANTIZATIONS: + raise ValueError( + "Unsupported qwen3 MLX quantization " + f"{value!r}. Supported values: {', '.join(VALID_QWEN3_MLX_QUANTIZATIONS)}" + ) + if quantization in seen: + continue + seen.add(quantization) + normalized.append(quantization) + return normalized + + +def build_benchmark_targets(args) -> List[tuple[str, str, Dict[str, Any]]]: + targets = [] + qwen3_quantizations = normalize_qwen3_mlx_quantizations(args.qwen3_mlx_quantizations) + + for handler_name in args.handlers: + if handler_name == "qwen3" and qwen3_quantizations: + for quantization in qwen3_quantizations: + targets.append( + ( + f"qwen3[{quantization}]", + "qwen3", + {"mlx_quantization": quantization}, + ) + ) + continue + + targets.append((handler_name, handler_name, {})) + + return targets + + +def print_results(results: List[BenchmarkResult]): + print("\n" + "=" * 80) + print("TTS BENCHMARK RESULTS") + print("=" * 80) + + for result in results: + stats = result.get_stats() + print(f"\nHandler: {stats['handler']}") + print("-" * 80) + + if stats.get("status") == "failed": + print(" Status: FAILED") + print(f" Errors: {stats['errors']}") + continue + + print(f" Warmup Time: {stats['warmup_time']:.4f}s") + print(f" Avg Inference Time: {stats['avg_inference_time']:.4f}s") + print(f" Min Inference Time: {stats['min_inference_time']:.4f}s") + print(f" Max Inference Time: {stats['max_inference_time']:.4f}s") + print(f" Std Deviation: {stats['std_inference_time']:.4f}s") + + print(f" Avg Audio Duration: {stats['avg_audio_duration']:.2f}s") + print(f" Min Audio Duration: {stats['min_audio_duration']:.2f}s") + print(f" Max Audio Duration: {stats['max_audio_duration']:.2f}s") + print(f" Std Audio Duration: {stats['std_audio_duration']:.4f}s") + print(f" Avg RTF: {stats['avg_rtf']:.2f}") + + if "avg_time_to_first_chunk" in stats: + print("\n Time to First Chunk:") + print(f" Avg TTFC: {stats['avg_time_to_first_chunk']:.4f}s") + print(f" Min TTFC: {stats['min_time_to_first_chunk']:.4f}s") + print(f" Max TTFC: {stats['max_time_to_first_chunk']:.4f}s") + print(f" Std TTFC: {stats['std_time_to_first_chunk']:.4f}s") + + print(f"\n Total Iterations: {stats['total_iterations']}") + + if stats["errors"]: + print(f" Errors: {stats['errors']}") + + print("\n" + "=" * 80) + print("COMPARISON (Average Inference Time)") + print("=" * 80) + + successful_results = [r for r in results if r.inference_times] + if successful_results: + sorted_results = sorted(successful_results, key=lambda x: np.mean(x.inference_times)) + fastest = sorted_results[0] + fastest_time = np.mean(fastest.inference_times) + + for result in sorted_results: + avg_time = np.mean(result.inference_times) + speedup = avg_time / fastest_time + print(f" {result.handler_name:25s}: {avg_time:.4f}s ({speedup:.2f}x slower than fastest)") + + +def save_results(results: List[BenchmarkResult], output_file: str): + data = { + "results": [r.get_stats() for r in results], + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + } + + with open(output_file, "w") as f: + json.dump(data, f, indent=2) + + logger.info(f"Results saved to: {output_file}") + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark TTS handlers") + parser.add_argument( + "--text", + type=str, + default="Hello from the speech to speech benchmark. This is a latency test.", + help="Text to synthesize", + ) + parser.add_argument( + "--handlers", + nargs="+", + default=["kokoro", "qwen3", "pocket_tts"], + help="List of handlers to benchmark (kokoro, qwen3, pocket_tts, chatTTS, facebookMMS)", + ) + parser.add_argument( + "--iterations", + type=int, + default=3, + help="Number of iterations per handler (default: 3)", + ) + parser.add_argument( + "--output", + type=str, + default="tts_benchmark_results.json", + help="Output JSON file for results (default: tts_benchmark_results.json)", + ) + parser.add_argument( + "--language_code", + type=str, + default="en", + help="Language code to pass to TTS handlers (default: en)", + ) + parser.add_argument( + "--qwen3_mlx_quantizations", + nargs="+", + default=None, + help=( + "Optional list of Apple Silicon MLX Qwen3-TTS quantizations to benchmark " + "as separate variants. Supported values: bf16, 4bit, 6bit, 8bit." + ), + ) + + args = parser.parse_args() + + if not args.handlers: + logger.error("No handlers provided") + return + + try: + targets = build_benchmark_targets(args) + except ValueError as e: + logger.error(str(e)) + return + + results = [] + for result_name, handler_name, handler_kwargs in targets: + result = benchmark_handler( + handler_name, + args.text, + args.iterations, + handler_kwargs=handler_kwargs, + language_code=args.language_code, + ) + result.handler_name = result_name + results.append(result) + + print_results(results) + save_results(results, args.output) + + logger.info("TTS benchmarking complete!") + + +if __name__ == "__main__": + main() diff --git a/scripts/listen_and_play.py b/scripts/listen_and_play.py new file mode 100644 index 0000000..68f212c --- /dev/null +++ b/scripts/listen_and_play.py @@ -0,0 +1,136 @@ +import socket +import threading +from dataclasses import dataclass, field +from queue import Queue + +import numpy as np +import sounddevice as sd +from transformers import HfArgumentParser + + +@dataclass +class ListenAndPlayArguments: + send_rate: int = field(default=16000, metadata={"help": "In Hz. Default is 16000."}) + recv_rate: int = field(default=16000, metadata={"help": "In Hz. Default is 16000."}) + list_play_chunk_size: int = field( + default=1024, + metadata={"help": "The size of data chunks (in bytes). Default is 1024."}, + ) + host: str = field( + default="localhost", + metadata={ + "help": "The hostname or IP address for listening and playing. Default is 'localhost'." + }, + ) + send_port: int = field( + default=12345, + metadata={"help": "The network port for sending data. Default is 12345."}, + ) + recv_port: int = field( + default=12346, + metadata={"help": "The network port for receiving data. Default is 12346."}, + ) + + +def listen_and_play( + send_rate=16000, + recv_rate=44100, + list_play_chunk_size=1024, + host="localhost", + send_port=12345, + recv_port=12346, +): + send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + send_socket.connect((host, send_port)) + + recv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + recv_socket.connect((host, recv_port)) + + print("Recording and streaming...") + + stop_event = threading.Event() + recv_queue = Queue() + send_queue = Queue() + + # Pre-generate a static dither buffer (±1 LSB, -96 dB) to keep the audio + # sink active without calling numpy inside the real-time audio callback. + dither_bytes = np.random.randint( + -1, 2, size=list_play_chunk_size, dtype=np.int16 + ).tobytes() + + def callback_recv(outdata, frames, time, status): + if not recv_queue.empty(): + data = recv_queue.get() + outdata[: len(data)] = data + outdata[len(data) :] = b"\x00" * (len(outdata) - len(data)) + else: + outdata[:] = dither_bytes + + def callback_send(indata, frames, time, status): + if recv_queue.empty(): + data = bytes(indata) + send_queue.put(data) + + def send(stop_event, send_queue): + while not stop_event.is_set(): + data = send_queue.get() + send_socket.sendall(data) + + def recv(stop_event, recv_queue): + def receive_full_chunk(conn, chunk_size): + data = b"" + while len(data) < chunk_size: + packet = conn.recv(chunk_size - len(data)) + if not packet: + return None # Connection has been closed + data += packet + return data + + while not stop_event.is_set(): + data = receive_full_chunk(recv_socket, list_play_chunk_size * 2) + if data: + recv_queue.put(data) + + try: + send_stream = sd.RawInputStream( + samplerate=send_rate, + channels=1, + dtype="int16", + blocksize=list_play_chunk_size, + callback=callback_send, + ) + recv_stream = sd.RawOutputStream( + samplerate=recv_rate, + channels=1, + dtype="int16", + blocksize=list_play_chunk_size, + callback=callback_recv, + ) + threading.Thread(target=send_stream.start).start() + threading.Thread(target=recv_stream.start).start() + + send_thread = threading.Thread(target=send, args=(stop_event, send_queue)) + send_thread.start() + recv_thread = threading.Thread(target=recv, args=(stop_event, recv_queue)) + recv_thread.start() + + input("Press Enter to stop...") + + except KeyboardInterrupt: + print("Finished streaming.") + + finally: + stop_event.set() + # Given that socket::recv is blocking in receive_data_chunk, shut it down to allow the thread to continue. + recv_socket.shutdown(socket.SHUT_RDWR) + recv_thread.join() + send_thread.join() + send_socket.close() + recv_socket.close() + print("Connection closed.") + + +if __name__ == "__main__": + parser = HfArgumentParser((ListenAndPlayArguments,)) # type: ignore[arg-type] + (listen_and_play_kwargs,) = parser.parse_args_into_dataclasses() + listen_and_play(**vars(listen_and_play_kwargs)) diff --git a/scripts/listen_and_play_realtime.py b/scripts/listen_and_play_realtime.py new file mode 100644 index 0000000..7ee43d1 --- /dev/null +++ b/scripts/listen_and_play_realtime.py @@ -0,0 +1,385 @@ +import argparse +import asyncio +import base64 +import time +from dataclasses import dataclass, field +from queue import Empty, Queue +from threading import Event, Lock +from typing import Any, Optional + +from openai import AsyncOpenAI + + +@dataclass +class ListenAndPlayRealtimeArguments: + host: str = field( + default="127.0.0.1", + metadata={"help": "Realtime server host. Default is 127.0.0.1."}, + ) + port: int = field( + default=8765, + metadata={"help": "Realtime server port. Default is 8765."}, + ) + model: str = field( + default="local", + metadata={"help": "Model name sent to the OpenAI-compatible realtime client."}, + ) + api_key: str = field( + default="test-key", + metadata={"help": "API key for the OpenAI SDK client. Local server ignores it."}, + ) + base_url: Optional[str] = field( + default=None, + metadata={"help": "Optional HTTP base URL, e.g. http://127.0.0.1:8765/v1"}, + ) + websocket_base_url: Optional[str] = field( + default=None, + metadata={"help": "Optional WS base URL, e.g. ws://127.0.0.1:8765/v1"}, + ) + send_rate: int = field( + default=16000, + metadata={"help": "Microphone sample rate in Hz. Default is 16000."}, + ) + recv_rate: int = field( + default=16000, + metadata={"help": "Speaker sample rate in Hz. Default is 16000."}, + ) + chunk_size: int = field( + default=1024, + metadata={"help": "Audio callback block size in samples. Default is 1024."}, + ) + input_device: Optional[int] = field( + default=None, + metadata={"help": "Optional sounddevice input device index."}, + ) + output_device: Optional[int] = field( + default=None, + metadata={"help": "Optional sounddevice output device index."}, + ) + instructions: Optional[str] = field( + default=None, + metadata={"help": "Optional session instructions to apply on connect."}, + ) + voice: Optional[str] = field( + default=None, + metadata={ + "help": ( + "TTS voice sent as session.audio.output.voice. " + "Local Kokoro: e.g. bm_fable, af_heart, am_adam. " + "OpenAI Realtime: e.g. marin, cedar, alloy." + ), + }, + ) + print_json: bool = field( + default=False, + metadata={"help": "Print raw event payloads in addition to friendly logs."}, + ) + block_mic_during_playback: bool = field( + default=False, + metadata={ + "help": "If set, pause microphone capture while speaker audio is playing. Disabled by default so barge-in works." + }, + ) + + +def _make_client(args: ListenAndPlayRealtimeArguments) -> AsyncOpenAI: + base_url = args.base_url or f"http://{args.host}:{args.port}/v1" + websocket_base_url = args.websocket_base_url or f"ws://{args.host}:{args.port}/v1" + return AsyncOpenAI( + api_key=args.api_key, + base_url=base_url, + websocket_base_url=websocket_base_url, + ) + + +def _build_session_update(args: ListenAndPlayRealtimeArguments) -> dict: + def maybe_pcm_format(rate: int) -> Optional[dict]: + # The OpenAI realtime Pydantic models only validate audio/pcm at 24 kHz. + # Our local pipeline defaults to 16 kHz internally when format is omitted, + # so omit the field for the common local case instead of sending an + # invalid 16 kHz declaration. + if rate == 16000: + return None + if rate == 24000: + return {"type": "audio/pcm", "rate": 24000} + raise ValueError( + f"Unsupported rate {rate}. Use 16000 for the local pipeline default " + f"or 24000 to match the OpenAI realtime audio format schema." + ) + + input_cfg = { + "turn_detection": {"type": "server_vad", "interrupt_response": True}, + } + output_cfg: dict[str, Any] = {} + + input_format = maybe_pcm_format(args.send_rate) + output_format = maybe_pcm_format(args.recv_rate) + if input_format is not None: + input_cfg["format"] = input_format + if output_format is not None: + output_cfg["format"] = output_format + if args.voice: + output_cfg["voice"] = args.voice + + session = { + "type": "realtime", + "audio": { + "input": input_cfg, + "output": output_cfg, + }, + } + if args.instructions: + session["instructions"] = args.instructions + return {"type": "session.update", "session": session} + + +async def listen_and_play_realtime(args: ListenAndPlayRealtimeArguments) -> None: + import sounddevice as sd + + client = _make_client(args) + + mic_queue: Queue[bytes] = Queue(maxsize=128) + stop_event = Event() + playback_buffer = bytearray() + playback_lock = Lock() + speaker_active_until = [0.0] + partial_user_text = "" + live_user_width = 0 + saw_user_speech = False + + def render_live_user_text(text: str, final: bool = False) -> None: + nonlocal live_user_width + line = f"USER: {text}" + padded = line + if live_user_width > len(line): + padded += " " * (live_user_width - len(line)) + + if final: + print(f"\r{padded}", flush=True) + live_user_width = 0 + return + + print(f"\r{padded}", end="", flush=True) + live_user_width = len(line) + + def clear_live_user_text() -> None: + nonlocal live_user_width + if live_user_width == 0: + return + print("\r" + (" " * live_user_width) + "\r", end="", flush=True) + live_user_width = 0 + + def clear_playback_buffer() -> None: + speaker_active_until[0] = 0.0 + with playback_lock: + playback_buffer.clear() + + def callback_recv(outdata, _frames, _time_info, status): + if status: + print(f"Speaker status: {status}", flush=True) + + needed = len(outdata) + with playback_lock: + available = min(needed, len(playback_buffer)) + if available: + outdata[:available] = playback_buffer[:available] + del playback_buffer[:available] + if available < needed: + outdata[available:] = b"\x00" * (needed - available) + + def callback_send(indata, _frames, _time_info, status): + if status: + print(f"Mic status: {status}", flush=True) + + if args.block_mic_during_playback: + with playback_lock: + speaker_active = bool(playback_buffer) + if speaker_active or time.monotonic() < speaker_active_until[0]: + return + + try: + mic_queue.put_nowait(bytes(indata)) + except Exception: + pass + + async def send_audio(conn): + while not stop_event.is_set(): + try: + chunk = await asyncio.to_thread(mic_queue.get, True, 0.1) + except Empty: + continue + + await conn.send( + { + "type": "input_audio_buffer.append", + "audio": base64.b64encode(chunk).decode("ascii"), + } + ) + + async def receive_events(conn): + nonlocal partial_user_text, saw_user_speech + + while not stop_event.is_set(): + event = await conn.recv() + + if args.print_json: + try: + print(f"EVENT: {event.model_dump_json()}", flush=True) + except Exception: + print(f"EVENT: {event}", flush=True) + + if event.type == "session.created": + print("Connected.", flush=True) + elif event.type == "input_audio_buffer.speech_started": + clear_playback_buffer() + partial_user_text = "" + if saw_user_speech: + print("", flush=True) + saw_user_speech = True + elif event.type == "input_audio_buffer.speech_stopped": + pass + elif event.type == "conversation.item.input_audio_transcription.delta": + # This server currently sends the latest partial hypothesis in + # each "delta" event rather than a token-level suffix, so render + # the newest snapshot instead of concatenating repeated text. + partial_user_text = event.delta.strip() + if partial_user_text: + render_live_user_text(partial_user_text) + elif event.type == "conversation.item.input_audio_transcription.completed": + partial_user_text = "" + render_live_user_text(event.transcript.strip(), final=True) + elif event.type == "response.created": + clear_live_user_text() + print("ASSISTANT: ", flush=True) + elif event.type == "response.output_audio.delta": + audio = base64.b64decode(event.delta) + with playback_lock: + playback_buffer.extend(audio) + speaker_active_until[0] = time.monotonic() + max(0.15, len(audio) / (2 * args.recv_rate)) + elif event.type == "response.output_audio.done": + print("ASSISTANT: