refactor: rename API and frontend projects to Vocero
Rename the project directories, Python distribution and import package, and npm package. Update launcher, Docker, tooling, tests, lockfiles, and documentation while preserving existing API contracts. Make AGENTS.md self-contained and remove external skill and workflow references.
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
type PlaybackMessage =
|
||||
{ kind: "config"; inputRate: number } | { kind: "audio"; samples: Float32Array } | { kind: "clear" };
|
||||
|
||||
class AudioPlaybackProcessor extends AudioWorkletProcessor {
|
||||
private rate = 16000;
|
||||
private queue: Float32Array[] = [];
|
||||
private offset = 0;
|
||||
private fraction = 0;
|
||||
private playing = false;
|
||||
private fadeIn = 0;
|
||||
private fadeOut = 0;
|
||||
private lastSample = 0;
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.port.onmessage = (event: MessageEvent<PlaybackMessage>) => {
|
||||
const data = event.data;
|
||||
switch (data.kind) {
|
||||
case "config":
|
||||
if (data.inputRate > 0) {
|
||||
this.rate = data.inputRate;
|
||||
}
|
||||
break;
|
||||
case "audio":
|
||||
if (data.samples.length) {
|
||||
this.queue.push(data.samples);
|
||||
}
|
||||
break;
|
||||
case "clear":
|
||||
this.queue = [];
|
||||
this.offset = 0;
|
||||
this.fraction = 0;
|
||||
this.fadeOut = 32;
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
private bufferedMs() {
|
||||
return (this.queue.reduce((sum, buffer) => sum + buffer.length, -this.offset) / this.rate) * 1000;
|
||||
}
|
||||
private read(): number | null {
|
||||
const head = this.queue[0];
|
||||
if (!head) {
|
||||
return null;
|
||||
}
|
||||
const a = head[this.offset] ?? 0;
|
||||
const b = head[this.offset + 1] ?? this.queue[1]?.[0] ?? a;
|
||||
return a + (b - a) * this.fraction;
|
||||
}
|
||||
private advance() {
|
||||
this.fraction += this.rate / sampleRate;
|
||||
while (this.fraction >= 1) {
|
||||
this.fraction--;
|
||||
this.offset++;
|
||||
}
|
||||
while (this.queue[0] && this.offset >= this.queue[0].length) {
|
||||
this.offset -= this.queue[0].length;
|
||||
this.queue.shift();
|
||||
}
|
||||
}
|
||||
process(_inputs: Float32Array[][], outputs: Float32Array[][]) {
|
||||
const channels = outputs[0];
|
||||
const output = channels?.[0];
|
||||
if (!output) {
|
||||
return true;
|
||||
}
|
||||
for (let index = 0; index < output.length; index++) {
|
||||
let value = 0;
|
||||
if (!this.playing && this.queue.length && this.bufferedMs() >= 90) {
|
||||
this.playing = true;
|
||||
this.fadeIn = 32;
|
||||
this.fadeOut = 0;
|
||||
}
|
||||
if (this.playing) {
|
||||
const sample = this.read();
|
||||
if (sample === null) {
|
||||
value = this.lastSample * (1 - 1 / 32);
|
||||
this.lastSample = value;
|
||||
if (Math.abs(value) < 1e-4) {
|
||||
this.playing = false;
|
||||
this.lastSample = 0;
|
||||
}
|
||||
} else {
|
||||
value = sample;
|
||||
this.lastSample = sample;
|
||||
this.advance();
|
||||
}
|
||||
if (this.fadeIn > 0) {
|
||||
value *= 1 - this.fadeIn-- / 32;
|
||||
}
|
||||
if (this.fadeOut > 0) {
|
||||
value *= this.fadeOut-- / 32;
|
||||
if (!this.fadeOut) {
|
||||
this.playing = false;
|
||||
this.lastSample = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
output[index] = value;
|
||||
const stereo = channels?.[1];
|
||||
if (stereo) {
|
||||
stereo[index] = value;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor("audio-playback", AudioPlaybackProcessor);
|
||||
@@ -0,0 +1,23 @@
|
||||
export function encodePcm(buffer: ArrayBuffer) {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = "";
|
||||
for (let index = 0; index < bytes.length; index += 0x8000) {
|
||||
binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
export function decodePcm(base64: string) {
|
||||
const binary = atob(base64);
|
||||
const buffer = new ArrayBuffer(binary.length);
|
||||
const bytes = new Uint8Array(buffer);
|
||||
for (let index = 0; index < binary.length; index++) {
|
||||
bytes[index] = binary.charCodeAt(index);
|
||||
}
|
||||
const view = new DataView(buffer);
|
||||
const samples = new Float32Array(Math.floor(binary.length / 2));
|
||||
for (let index = 0; index < samples.length; index++) {
|
||||
const value = view.getInt16(index * 2, true);
|
||||
samples[index] = value < 0 ? value / 0x8000 : value / 0x7fff;
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { NoiseGate } from "../lib/contracts";
|
||||
import playbackUrl from "./audio-playback.worklet.ts?worker&url";
|
||||
import { decodePcm } from "./codec";
|
||||
import captureUrl from "./mic-capture.worklet.ts?worker&url";
|
||||
|
||||
const BANDS = [2, 5, 9, 16, 28, 52];
|
||||
export const MIC_CONSTRAINTS: MediaStreamConstraints = {
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
video: false,
|
||||
};
|
||||
|
||||
/** Construct inside the click handler, before awaiting microphone permission. */
|
||||
export function createAudioContext() {
|
||||
const context = new AudioContext({ latencyHint: "interactive" });
|
||||
void context.resume().catch(() => undefined);
|
||||
return context;
|
||||
}
|
||||
|
||||
export class AudioEngine {
|
||||
private readonly context: AudioContext;
|
||||
private stream: MediaStream | null = null;
|
||||
private capture: AudioWorkletNode | null = null;
|
||||
private playback: AudioWorkletNode | null = null;
|
||||
private nodes: AudioNode[] = [];
|
||||
private frame = 0;
|
||||
private closed = false;
|
||||
private gate: NoiseGate;
|
||||
private muted = false;
|
||||
|
||||
constructor(context: AudioContext, gate: NoiseGate) {
|
||||
this.context = context;
|
||||
this.gate = gate;
|
||||
}
|
||||
|
||||
async start(onChunk: (chunk: ArrayBuffer) => void, onLevel: (rms: number) => void, speaking: () => boolean) {
|
||||
const stream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
|
||||
if (this.closed) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
track.stop();
|
||||
});
|
||||
return;
|
||||
}
|
||||
this.stream = stream;
|
||||
await Promise.all([
|
||||
this.context.audioWorklet.addModule(captureUrl),
|
||||
this.context.audioWorklet.addModule(playbackUrl),
|
||||
]);
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
const capture = new AudioWorkletNode(this.context, "mic-capture", { numberOfInputs: 1, numberOfOutputs: 0 });
|
||||
const playback = new AudioWorkletNode(this.context, "audio-playback", {
|
||||
numberOfInputs: 0,
|
||||
numberOfOutputs: 1,
|
||||
outputChannelCount: [1],
|
||||
});
|
||||
// The checked-in speech pipeline emits native 16 kHz PCM, despite older 24 kHz comments.
|
||||
playback.port.postMessage({ kind: "config", inputRate: 16000 });
|
||||
this.capture = capture;
|
||||
this.playback = playback;
|
||||
this.setGate(this.gate);
|
||||
this.setMuted(this.muted);
|
||||
capture.port.onmessage = (event: MessageEvent<ArrayBuffer | { kind: "level"; rms: number }>) => {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
if (event.data instanceof ArrayBuffer) {
|
||||
onChunk(event.data);
|
||||
} else {
|
||||
document.documentElement.style.setProperty("--audio-level", String(Math.min(1, event.data.rms * 5)));
|
||||
onLevel(event.data.rms);
|
||||
}
|
||||
};
|
||||
const input = this.context.createMediaStreamSource(stream);
|
||||
const micAnalyser = this.context.createAnalyser();
|
||||
const outAnalyser = this.context.createAnalyser();
|
||||
micAnalyser.fftSize = outAnalyser.fftSize = 256;
|
||||
micAnalyser.smoothingTimeConstant = 0;
|
||||
outAnalyser.smoothingTimeConstant = 0.3;
|
||||
input.connect(capture);
|
||||
input.connect(micAnalyser);
|
||||
playback.connect(outAnalyser);
|
||||
outAnalyser.connect(this.context.destination);
|
||||
this.nodes = [input, capture, playback, micAnalyser, outAnalyser];
|
||||
const buffer = new Uint8Array(micAnalyser.frequencyBinCount);
|
||||
const levels = new Float32Array(5);
|
||||
const tick = () => {
|
||||
const source = speaking() ? outAnalyser : micAnalyser;
|
||||
source.getByteFrequencyData(buffer);
|
||||
for (let band = 0; band < 5; band++) {
|
||||
const lo = BANDS[band] ?? 0;
|
||||
const hi = BANDS[band + 1] ?? 52;
|
||||
let sum = 0;
|
||||
for (let index = lo; index < hi; index++) {
|
||||
sum += buffer[index] ?? 0;
|
||||
}
|
||||
const target = sum / ((hi - lo) * 255);
|
||||
const previous = levels[band] ?? 0;
|
||||
levels[band] = previous + (target - previous) * (target > previous ? 0.6 : 0.18);
|
||||
document.documentElement.style.setProperty(`--bar${band}`, String(levels[band]));
|
||||
}
|
||||
outAnalyser.getByteFrequencyData(buffer);
|
||||
document.documentElement.style.setProperty("--ai-audio-level", String(Math.max(...buffer.subarray(0, 52)) / 255));
|
||||
this.frame = requestAnimationFrame(tick);
|
||||
};
|
||||
this.frame = requestAnimationFrame(tick);
|
||||
}
|
||||
play(base64: string) {
|
||||
if (!base64 || this.closed) {
|
||||
return;
|
||||
}
|
||||
const samples = decodePcm(base64);
|
||||
this.playback?.port.postMessage({ kind: "audio", samples }, [samples.buffer]);
|
||||
}
|
||||
clear() {
|
||||
this.playback?.port.postMessage({ kind: "clear" });
|
||||
}
|
||||
setGate(gate: NoiseGate) {
|
||||
this.gate = gate;
|
||||
this.capture?.port.postMessage({ kind: "gate", ...gate });
|
||||
}
|
||||
setMuted(muted: boolean) {
|
||||
this.muted = muted;
|
||||
this.stream?.getAudioTracks().forEach((track) => {
|
||||
track.enabled = !muted;
|
||||
});
|
||||
this.capture?.port.postMessage({ kind: "enable", value: !muted });
|
||||
}
|
||||
async close() {
|
||||
this.closed = true;
|
||||
cancelAnimationFrame(this.frame);
|
||||
this.stream?.getTracks().forEach((track) => {
|
||||
track.stop();
|
||||
});
|
||||
this.capture?.port.close();
|
||||
this.playback?.port.close();
|
||||
this.nodes.forEach((node) => {
|
||||
node.disconnect();
|
||||
});
|
||||
if (this.context.state !== "closed") {
|
||||
await this.context.close();
|
||||
}
|
||||
for (let index = 0; index < 5; index++) {
|
||||
document.documentElement.style.removeProperty(`--bar${index}`);
|
||||
}
|
||||
document.documentElement.style.removeProperty("--ai-audio-level");
|
||||
document.documentElement.style.removeProperty("--audio-level");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
const TARGET_RATE = 16000;
|
||||
type Control = { kind: "enable"; value: boolean } | { kind: "gate"; enabled: boolean; thresholdDb: number };
|
||||
|
||||
/** Preserve the existing 40 ms PCM16 frames and smoothed gate envelope. */
|
||||
class MicCaptureProcessor extends AudioWorkletProcessor {
|
||||
private readonly ratio = sampleRate / TARGET_RATE;
|
||||
private readonly chunkSize = 640;
|
||||
private scratch = new Float32Array(0);
|
||||
private readonly decimated = new Float32Array(this.chunkSize);
|
||||
private enabled = true;
|
||||
private gateEnabled = false;
|
||||
private threshold = 0;
|
||||
private gain = 1;
|
||||
private hold = 0;
|
||||
private readonly attack = Math.exp(-1 / (0.005 * TARGET_RATE));
|
||||
private readonly release = Math.exp(-1 / (0.08 * TARGET_RATE));
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.port.onmessage = (event: MessageEvent<Control>) => {
|
||||
if (event.data.kind === "enable") {
|
||||
this.enabled = event.data.value;
|
||||
} else {
|
||||
this.gateEnabled = event.data.enabled;
|
||||
this.threshold = event.data.enabled ? 10 ** (event.data.thresholdDb / 20) : 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
process(inputs: Float32Array[][]) {
|
||||
const incoming = inputs[0]?.[0];
|
||||
if (!incoming?.length) {
|
||||
return true;
|
||||
}
|
||||
const next = new Float32Array(this.scratch.length + incoming.length);
|
||||
next.set(this.scratch);
|
||||
next.set(incoming, this.scratch.length);
|
||||
this.scratch = next;
|
||||
const needed = Math.ceil(this.chunkSize * this.ratio);
|
||||
while (this.scratch.length >= needed) {
|
||||
let sum = 0;
|
||||
for (let index = 0; index < this.chunkSize; index++) {
|
||||
const position = index * this.ratio;
|
||||
const start = Math.floor(position);
|
||||
const a = this.scratch[start] ?? 0;
|
||||
const value =
|
||||
Math.abs(this.ratio - 3) < 1e-6
|
||||
? (a + (this.scratch[start + 1] ?? 0) + (this.scratch[start + 2] ?? 0)) / 3
|
||||
: a + ((this.scratch[start + 1] ?? a) - a) * (position - start);
|
||||
this.decimated[index] = value;
|
||||
sum += value * value;
|
||||
}
|
||||
const rms = Math.sqrt(sum / this.chunkSize);
|
||||
let target = 1;
|
||||
if (this.gateEnabled) {
|
||||
if (rms >= this.threshold) {
|
||||
this.hold = TARGET_RATE * 0.25;
|
||||
} else if (this.hold > 0) {
|
||||
this.hold -= this.chunkSize;
|
||||
} else {
|
||||
target = 0;
|
||||
}
|
||||
}
|
||||
const output = new Int16Array(this.chunkSize);
|
||||
for (let index = 0; index < this.chunkSize; index++) {
|
||||
const coefficient = target > this.gain ? this.attack : this.release;
|
||||
this.gain = target + (this.gain - target) * coefficient;
|
||||
const value = Math.max(-1, Math.min(1, (this.decimated[index] ?? 0) * this.gain));
|
||||
output[index] = value < 0 ? value * 0x8000 : value * 0x7fff;
|
||||
}
|
||||
this.scratch = this.scratch.slice(Math.floor(this.chunkSize * this.ratio));
|
||||
this.port.postMessage({ kind: "level", rms });
|
||||
if (this.enabled) {
|
||||
this.port.postMessage(output.buffer, [output.buffer]);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor("mic-capture", MicCaptureProcessor);
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
export {};
|
||||
|
||||
declare global {
|
||||
const sampleRate: number;
|
||||
abstract class AudioWorkletProcessor {
|
||||
readonly port: MessagePort;
|
||||
abstract process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean;
|
||||
}
|
||||
function registerProcessor(
|
||||
name: string,
|
||||
processor: new (options?: AudioWorkletNodeOptions) => AudioWorkletProcessor,
|
||||
): void;
|
||||
}
|
||||
Reference in New Issue
Block a user