vocero-s2s/demo/worklets/mic-capture.js

// @ts-check
/**
 * AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz)
 * down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it
 * back to the main thread in fixed-size chunks.
 *
 * The Hugging Face speech-to-speech WebSocket route expects the
 * `input_audio_buffer.append` payload at 16 kHz PCM16 mono.
 *
 * Design notes:
 *   - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a
 *     cheap low-pass before decimating. Good enough for voice STT; we lose
 *     a tiny bit of >8 kHz content which the pipeline discards anyway.
 *   - Output frames are emitted at the cadence dictated by `chunkMs`
 *     (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime
 *     server batches incoming audio so the cadence is flexible; 20-100 ms
 *     is the sweet spot.
 *   - Float -> Int16 saturates to [-1, 1] before scaling.
 *   - Optional noise gate: per-chunk RMS decides open/closed against a
 *     threshold; the gain ramps (fast attack, hold, slow release) so word
 *     onsets aren't clipped and quiet tails don't click. The gate only
 *     affects the audio we SEND; the main-thread visualiser taps the raw
 *     mic separately. We post the chunk RMS up every frame so the Settings
 *     mic meter can show the live level against the threshold.
 */

const TARGET_RATE = 16000;
const DEFAULT_CHUNK_MS = 40;
// Gate envelope timing (fixed; only the threshold is user-tunable).
const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive
const GATE_HOLD_MS = 250; // stay open this long after the level drops back under
const GATE_RELEASE_MS = 80; // then fade closed over this long (no click)

class MicCaptureProcessor extends AudioWorkletProcessor {
  constructor(options) {
    super();
    const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS;
    this._inputRate = sampleRate;
    this._ratio = this._inputRate / TARGET_RATE;
    this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000);
    this._scratch = new Float32Array(0);
    this._decimated = new Float32Array(this._chunkSamples16k);
    this._enabled = true;

    // Noise gate state. Disabled by default (pure passthrough).
    this._gateEnabled = false;
    this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open
    this._gateGain = 1; // smoothed gain currently applied
    this._holdRemaining = 0; // samples left before the gate may start closing
    this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE));
    this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE));
    this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE);

    this.port.onmessage = (e) => {
      const data = e.data;
      if (data?.kind === "enable") this._enabled = !!data.value;
      else if (data?.kind === "gate") {
        this._gateEnabled = !!data.enabled;
        // dB -> linear amplitude. When off, threshold 0 keeps the gate open.
        this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0;
      }
    };
  }

  /**
   * Append `incoming` to the internal scratch buffer, then emit as many
   * full output chunks as we have material for.
   * @param {Float32Array} incoming
   */
  _ingest(incoming) {
    if (incoming.length === 0) return;
    const next = new Float32Array(this._scratch.length + incoming.length);
    next.set(this._scratch, 0);
    next.set(incoming, this._scratch.length);
    this._scratch = next;
    this._maybeEmit();
  }

  _maybeEmit() {
    const r = this._ratio;
    const n = this._chunkSamples16k;
    const needIn = Math.ceil(n * r);
    const dec = this._decimated;
    while (this._scratch.length >= needIn) {
      // 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter.
      let sumSq = 0;
      if (Math.abs(r - 3) < 1e-6) {
        // 48 kHz -> 16 kHz fast path with boxcar lowpass.
        for (let i = 0; i < n; i++) {
          const idx = i * 3;
          const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3;
          dec[i] = s;
          sumSq += s * s;
        }
      } else {
        // Generic path: linear interpolation. Slower but works at any rate
        // (e.g. some Windows boxes report sampleRate=44100).
        for (let i = 0; i < n; i++) {
          const srcPos = i * r;
          const idx = Math.floor(srcPos);
          const frac = srcPos - idx;
          const a = this._scratch[idx];
          const b = this._scratch[idx + 1] ?? a;
          const s = a + (b - a) * frac;
          dec[i] = s;
          sumSq += s * s;
        }
      }
      const rms = Math.sqrt(sumSq / n);

      // 2. Decide the gate target for this chunk, then ramp sample-by-sample.
      let target = 1;
      if (this._gateEnabled) {
        if (rms >= this._thresholdLin) {
          this._holdRemaining = this._holdSamples; // re-arm the hold
        } else if (this._holdRemaining > 0) {
          this._holdRemaining -= n; // coasting through the hold window
        } else {
          target = 0;
        }
      }

      // 3. Apply the (smoothed) gain and pack to Int16.
      const out = new Int16Array(n);
      let gain = this._gateGain;
      for (let i = 0; i < n; i++) {
        const coef = target > gain ? this._attackCoef : this._releaseCoef;
        gain = target + (gain - target) * coef;
        const s = dec[i] * gain;
        const clamped = s < -1 ? -1 : s > 1 ? 1 : s;
        out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
      }
      this._gateGain = gain;

      // Shift the scratch buffer to keep only the trailing unused samples.
      const consumed = Math.floor(n * r);
      this._scratch = this._scratch.slice(consumed);

      // Live input level for the Settings meter (raw RMS, pre-gate).
      this.port.postMessage({ kind: "level", rms });

      if (this._enabled) {
        this.port.postMessage(out.buffer, [out.buffer]);
      }
      // When disabled (mic muted) we silently consume input so the worklet
      // stays alive and the buffer never grows unbounded.
    }
  }

  process(inputs) {
    const input = inputs[0];
    if (!input || input.length === 0 || !input[0]) return true;
    const mono = input[0];
    if (mono.length > 0) this._ingest(mono);
    return true;
  }
}

registerProcessor("mic-capture", MicCaptureProcessor);