diff --git a/routes/stt.py b/routes/stt.py
index 9b7a227..9dde7db 100644
--- a/routes/stt.py
+++ b/routes/stt.py
@@ -2,7 +2,10 @@
from __future__ import annotations
import asyncio
+import json
+import re
import struct
+import time
import uuid
from pathlib import Path
@@ -274,6 +277,142 @@ def _transcribe_audio(src: Path, settings: dict, backend: str = "configured") ->
return _transcription_text_from_response(resp), backend
+def _word_tokens(text: str) -> list[str]:
+ return re.findall(r"[\w']+", (text or "").lower(), flags=re.UNICODE)
+
+
+def _word_accuracy(reference: str, hypothesis: str) -> float | None:
+ ref = _word_tokens(reference)
+ hyp = _word_tokens(hypothesis)
+ if not ref:
+ return None
+ prev = list(range(len(hyp) + 1))
+ for i, rw in enumerate(ref, 1):
+ cur = [i]
+ for j, hw in enumerate(hyp, 1):
+ cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (0 if rw == hw else 1)))
+ prev = cur
+ return max(0.0, (1.0 - prev[-1] / max(1, len(ref))) * 100.0)
+
+
+def _stt_device_hint(engine: dict, settings: dict) -> str:
+ text = " ".join(str(engine.get(k, "")) for k in ("id", "backend", "label", "url", "model", "source")).lower()
+ if any(token in text for token in ("nvidia", "parakeet", "nemotron", "whisperx", "faster", "cuda", "gpu", "blackwell")):
+ return "CUDA/GPU"
+ if "cpu" in text or "whisper_cpp" in text or "whisper.cpp" in text:
+ return "CPU"
+ backend = _clean_stt_backend(str(engine.get("backend") or engine.get("id") or ""))
+ metrics = _STT_BACKEND_METRICS.get(backend, {})
+ speed = str(metrics.get("speed", "")).lower()
+ if "gpu" in speed or "cuda" in speed:
+ return "CUDA/GPU"
+ if "cpu" in speed:
+ return "CPU"
+ return "Unknown"
+
+
+def _transcribe_url_for_benchmark(src: Path, url: str, model: str, api_key: str = "") -> str:
+ base = _validate_http_url(url, allow_private=True).rstrip("/")
+ headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+ last_resp: requests.Response | None = None
+ last_error = ""
+ for path in ("/v1/audio/transcriptions", "/transcribe", "/audio/transcriptions"):
+ try:
+ with src.open("rb") as f:
+ resp = requests.post(
+ f"{base}{path}",
+ files={"file": ("benchmark.wav", f, "audio/wav")},
+ data={"model": model or "whisper-1", "response_format": "text"},
+ headers=headers,
+ timeout=_STT_REQUEST_TIMEOUT,
+ )
+ last_resp = resp
+ if resp.status_code in {404, 405}:
+ continue
+ if not resp.ok and (model or "whisper-1") != "whisper-1":
+ with src.open("rb") as f:
+ retry = requests.post(
+ f"{base}{path}",
+ files={"file": ("benchmark.wav", f, "audio/wav")},
+ data={"model": "whisper-1", "response_format": "text"},
+ headers=headers,
+ timeout=_STT_REQUEST_TIMEOUT,
+ )
+ if retry.ok:
+ return _transcription_text_from_response(retry)
+ last_resp = retry
+ if resp.ok:
+ return _transcription_text_from_response(resp)
+ try:
+ body = resp.json()
+ last_error = body.get("detail") or body.get("error") or body.get("message") or str(body)
+ except Exception:
+ last_error = resp.text[:400].strip()
+ except Exception as e:
+ last_error = str(e)
+ status = f"HTTP {last_resp.status_code}" if last_resp is not None else "STT request failed"
+ raise RuntimeError(f"{status}: {last_error or 'No compatible transcription endpoint'}")
+
+
+def _benchmark_stt_engine(src: Path, engine: dict, settings: dict, reference: str) -> dict:
+ label = str(engine.get("label") or engine.get("id") or engine.get("backend") or "STT")
+ backend = str(engine.get("backend") or engine.get("id") or "").strip()
+ url = str(engine.get("url") or "").strip()
+ model = str(engine.get("model") or "").strip() or (backend and _stt_backend_model(backend)) or "whisper-1"
+ device = _stt_device_hint(engine, settings)
+ t0 = time.perf_counter()
+ try:
+ if url:
+ api_key = _stt_backend_api_key(settings, _clean_stt_backend(backend)) if backend else settings.get("whisper_api_key", "")
+ text = _transcribe_url_for_benchmark(src, url, model, api_key)
+ used_url = _validate_http_url(url, allow_private=True).rstrip("/")
+ else:
+ text, used_backend = _transcribe_audio(src, settings, backend or "configured")
+ used_url = _stt_backend_url(settings, used_backend)
+ model = model or _stt_backend_model(used_backend)
+ elapsed = time.perf_counter() - t0
+ accuracy = _word_accuracy(reference, text)
+ return {
+ "ok": True,
+ "engine": label,
+ "backend": backend,
+ "url": used_url,
+ "model": model,
+ "device": device,
+ "time_sec": elapsed,
+ "accuracy": accuracy,
+ "output": text,
+ }
+ except Exception as e:
+ return {
+ "ok": False,
+ "engine": label,
+ "backend": backend,
+ "url": url,
+ "model": model,
+ "device": device,
+ "time_sec": time.perf_counter() - t0,
+ "accuracy": None,
+ "output": "",
+ "error": str(e),
+ }
+
+
+async def _default_stt_benchmark_engines() -> list[dict]:
+ payload = await stt_backends()
+ engines = []
+ for item in payload.get("backends", []):
+ if item.get("available"):
+ engines.append({
+ "id": item.get("id"),
+ "backend": item.get("id"),
+ "label": item.get("label"),
+ "url": item.get("url"),
+ "model": item.get("model") or (item.get("models") or ["whisper-1"])[0],
+ })
+ return engines
+
+
# ── STT routes ────────────────────────────────────────────────────────────────
@router.get("/api/stt-backends")
@@ -380,3 +519,63 @@ async def transcribe_bytes(
p.unlink(missing_ok=True)
except Exception:
pass
+
+@router.post("/api/stt-benchmark")
+async def stt_benchmark(
+ audio: UploadFile = File(...),
+ reference_text: str = Form(""),
+ reference_file: UploadFile | None = File(None),
+ engines_json: str = Form(""),
+):
+ suffix = Path(audio.filename or "audio.wav").suffix.lower() or ".wav"
+ if suffix not in (_AUDIO_EXTS | _UPLOAD_EXTS):
+ raise HTTPException(400, "Unsupported audio type")
+ tmp = TEMP_DIR / f"{uuid.uuid4().hex}_stt_bench{suffix}"
+ wav_tmp = tmp
+ try:
+ with tmp.open("wb") as f:
+ _copy_limited(audio.file, f, _MAX_UPLOAD_BYTES)
+ if suffix != ".wav":
+ wav_tmp = _to_wav_16k(tmp)
+ reference = (reference_text or "").strip()
+ if reference_file is not None:
+ raw = await reference_file.read()
+ if raw:
+ reference = raw[:2_000_000].decode("utf-8", errors="replace").strip()
+ if not reference:
+ raise HTTPException(400, "Reference text is required")
+ try:
+ parsed = json.loads(engines_json) if engines_json else []
+ engines = parsed if isinstance(parsed, list) else []
+ except Exception:
+ engines = []
+ if not engines:
+ engines = await _default_stt_benchmark_engines()
+ if not engines:
+ raise HTTPException(400, "No STT engines selected")
+ settings = _load_settings()
+ results = []
+ for engine in engines:
+ if not isinstance(engine, dict):
+ continue
+ results.append(await asyncio.to_thread(_benchmark_stt_engine, wav_tmp, engine, settings, reference))
+ ok_rows = [r for r in results if r.get("ok")]
+ best_time = min((r.get("time_sec") for r in ok_rows if isinstance(r.get("time_sec"), (int, float))), default=None)
+ best_accuracy = max((r.get("accuracy") for r in ok_rows if isinstance(r.get("accuracy"), (int, float))), default=None)
+ return {
+ "ok": True,
+ "reference_words": len(_word_tokens(reference)),
+ "results": results,
+ "best_time_sec": best_time,
+ "best_accuracy": best_accuracy,
+ }
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(500, f"STT benchmark failed: {e}")
+ finally:
+ for p in {tmp, wav_tmp}:
+ try:
+ p.unlink(missing_ok=True)
+ except Exception:
+ pass
diff --git a/static/js/benchmark.js b/static/js/benchmark.js
index 7b1559e..b27f25e 100644
--- a/static/js/benchmark.js
+++ b/static/js/benchmark.js
@@ -216,3 +216,421 @@
});
})();
+// ── Benchmark section ─────────────────────────────────────────────────────
+
+(function initBenchmarkSection() {
+ let initialized = false;
+ let sttEngines = [];
+ let turnHistoryCount = 0;
+
+ const q = (id) => document.getElementById(id);
+ const fmtMs = (ms) => ms == null ? '—' : ms >= 1000 ? (ms / 1000).toFixed(2) + 's' : Math.round(ms) + 'ms';
+ const fmtSec = (sec) => sec == null ? '—' : Number(sec).toFixed(Number(sec) < 10 ? 2 : 1) + 's';
+ const esc = (s) => typeof escHtml === 'function' ? escHtml(s) : String(s ?? '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
+ const say = (msg, type = '') => typeof toast === 'function' ? toast(msg, type) : console.log(msg);
+
+ function setBenchTab(name) {
+ document.querySelectorAll('.bench-tab').forEach(btn => btn.classList.toggle('active', btn.dataset.benchTab === name));
+ document.querySelectorAll('.bench-pane').forEach(pane => pane.classList.toggle('active', pane.dataset.benchPane === name));
+ }
+
+ async function fetchJson(url, opts) {
+ const r = await fetch(url, opts);
+ if (!r.ok) {
+ let detail = r.statusText || `HTTP ${r.status}`;
+ try { const d = await r.json(); detail = typeof d.detail === 'string' ? d.detail : JSON.stringify(d.detail || d); }
+ catch (_) { try { detail = await r.text(); } catch (_) {} }
+ throw new Error(detail || `HTTP ${r.status}`);
+ }
+ return r.json();
+ }
+
+ function dedupeEngines(items) {
+ const seen = new Set();
+ const out = [];
+ for (const item of items) {
+ const key = `${item.url || ''}|${item.model || ''}|${item.id || item.label || ''}`;
+ if (seen.has(key) || !item.url) continue;
+ seen.add(key);
+ out.push(item);
+ }
+ return out;
+ }
+
+ async function loadBenchmarkSttEngines() {
+ const list = q('bench-stt-engines');
+ const count = q('bench-stt-engine-count');
+ if (!list) return;
+ list.innerHTML = 'Checking STT engines...
';
+ let settings = {};
+ const items = [];
+ try { settings = await fetchJson('/api/settings'); } catch (_) {}
+
+ try {
+ const d = await fetchJson('/api/stt-backends');
+ (d.backends || []).forEach(b => items.push({
+ id: b.id,
+ backend: b.id,
+ label: b.label || b.id,
+ url: b.url,
+ model: (b.models && b.models[0]) || b.model || 'whisper-1',
+ available: !!b.available,
+ source: 'configured',
+ }));
+ } catch (e) {
+ console.warn('[benchmark stt backends]', e);
+ }
+
+ try {
+ const d = await fetchJson('/api/local-containers');
+ (d.containers || []).filter(c => c.role === 'stt' || c.role === 'stt+tts').forEach(c => {
+ const url = c.port ? `http://host.docker.internal:${c.port}` : '';
+ const model = settings?.engine_local_models?.['dc-' + c.name] || c.model || 'whisper-1';
+ items.push({
+ id: c.name,
+ label: c.label || c.name,
+ url,
+ model,
+ available: c.running === true || c.status === 'running',
+ source: c.stack || 'docker',
+ });
+ });
+ } catch (e) {
+ console.warn('[benchmark local containers]', e);
+ }
+
+ sttEngines = dedupeEngines(items);
+ if (count) count.textContent = sttEngines.length ? `${sttEngines.length} engine${sttEngines.length === 1 ? '' : 's'} loaded.` : 'No engines found.';
+ if (!sttEngines.length) {
+ list.innerHTML = 'No STT engines found. Check Engines -> Speech to Text.
';
+ return;
+ }
+ list.innerHTML = sttEngines.map((e, i) => `
+ `).join('');
+ }
+
+ function selectedSttEngines() {
+ return [...document.querySelectorAll('.bench-stt-engine-check:checked')].map(cb => {
+ const i = Number(cb.dataset.index);
+ const item = { ...sttEngines[i] };
+ const modelInput = document.querySelector(`.bench-engine-model[data-index="${i}"]`);
+ item.model = modelInput?.value.trim() || item.model || 'whisper-1';
+ return item;
+ }).filter(Boolean);
+ }
+
+ async function runSttBenchmark() {
+ const audio = q('bench-stt-audio')?.files?.[0];
+ const refFile = q('bench-stt-ref-file')?.files?.[0];
+ const refText = q('bench-stt-ref-text')?.value || '';
+ const engines = selectedSttEngines();
+ if (!audio) { say('Choose an audio file first', 'error'); return; }
+ if (!refFile && !refText.trim()) { say('Add a reference .txt or paste reference text', 'error'); return; }
+ if (!engines.length) { say('Select at least one STT engine', 'error'); return; }
+ const btn = q('bench-stt-run');
+ const st = q('bench-stt-status');
+ const tbody = q('bench-stt-table')?.querySelector('tbody');
+ if (btn) btn.disabled = true;
+ if (st) st.textContent = `Running ${engines.length} STT benchmark${engines.length === 1 ? '' : 's'}...`;
+ if (tbody) tbody.innerHTML = `| Benchmarking ${engines.length} engines... |
`;
+ try {
+ const fd = new FormData();
+ fd.append('audio', audio, audio.name);
+ if (refFile) fd.append('reference_file', refFile, refFile.name);
+ fd.append('reference_text', refText);
+ fd.append('engines_json', JSON.stringify(engines));
+ const d = await fetchJson('/api/stt-benchmark', { method: 'POST', body: fd });
+ renderSttResults(d);
+ if (st) st.textContent = `Finished ${d.results?.length || 0} STT engines.`;
+ say('STT benchmark complete', 'success');
+ } catch (e) {
+ if (st) st.textContent = 'STT benchmark failed';
+ if (tbody) tbody.innerHTML = `| ${esc(e.message)} |
`;
+ say('STT benchmark failed: ' + e.message, 'error');
+ } finally {
+ if (btn) btn.disabled = false;
+ }
+ }
+
+ function renderSttResults(d) {
+ const tbody = q('bench-stt-table')?.querySelector('tbody');
+ const summary = q('bench-stt-summary');
+ const rows = d.results || [];
+ if (summary) {
+ summary.innerHTML = [
+ ['Engines', String(rows.length)],
+ ['Reference words', String(d.reference_words || 0)],
+ ['Best time', d.best_time_sec == null ? '—' : fmtSec(d.best_time_sec)],
+ ['Best accuracy', d.best_accuracy == null ? '—' : d.best_accuracy.toFixed(1) + '%'],
+ ].map(([label, val]) => `${esc(val)} ${esc(label)}`).join('');
+ }
+ if (!tbody) return;
+ tbody.innerHTML = rows.map(r => {
+ const acc = r.accuracy == null ? '—' : `${Number(r.accuracy).toFixed(1)}%`;
+ const out = r.ok ? (r.output || '') : `⚠ ${r.error || 'failed'}`;
+ return `
+ | ${esc(r.engine)}${esc(r.url || '')} |
+ ${esc(r.model || '')} |
+ ${esc(r.device || 'Unknown')} |
+ ${fmtSec(r.time_sec)} |
+ ${esc(acc)} |
+ ${esc(out)} |
+
`;
+ }).join('');
+ }
+
+ async function runTtsBenchmark() {
+ const text = q('bench-tts-text')?.value.trim() || '';
+ if (!text) { say('Enter a sample sentence first', 'error'); return; }
+ const activeOnly = (q('bench-tts-scope')?.value || 'active') === 'active';
+ const btn = q('bench-tts-run');
+ const st = q('bench-tts-status');
+ const tbody = q('bench-tts-table')?.querySelector('tbody');
+ if (btn) btn.disabled = true;
+ if (st) st.textContent = 'Running TTS benchmark...';
+ if (tbody) tbody.innerHTML = '| Benchmarking voices... |
';
+ try {
+ const d = await fetchJson('/api/voices/benchmark', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ text, active_only: activeOnly }),
+ });
+ renderTtsResults(d);
+ if (st) st.textContent = `Finished ${d.benchmarked || 0} voices.`;
+ say('TTS benchmark complete', d.errors?.length ? 'error' : 'success');
+ } catch (e) {
+ if (st) st.textContent = 'TTS benchmark failed';
+ if (tbody) tbody.innerHTML = `| ${esc(e.message)} |
`;
+ say('TTS benchmark failed: ' + e.message, 'error');
+ } finally {
+ if (btn) btn.disabled = false;
+ }
+ }
+
+ function renderTtsResults(d) {
+ const tbody = q('bench-tts-table')?.querySelector('tbody');
+ const summary = q('bench-tts-summary');
+ const rows = d.voices || [];
+ const okRows = rows.map(r => r.benchmark).filter(b => b && b.ok);
+ const avg = (key) => {
+ const vals = okRows.map(b => Number(b[key])).filter(Number.isFinite);
+ return vals.length ? vals.reduce((a, b) => a + b, 0) / vals.length : null;
+ };
+ if (summary) {
+ summary.innerHTML = [
+ ['Voices', String(rows.length)],
+ ['OK', String(okRows.length)],
+ ['Avg TTFA', avg('ttfa_ms') == null ? '—' : Math.round(avg('ttfa_ms')) + 'ms'],
+ ['Avg total', avg('elapsed_sec') == null ? '—' : fmtSec(avg('elapsed_sec'))],
+ ].map(([label, val]) => `${esc(val)} ${esc(label)}`).join('');
+ }
+ if (!tbody) return;
+ tbody.innerHTML = rows.length ? rows.map(row => {
+ const b = row.benchmark || {};
+ const status = b.ok ? (b.realtime_ok === false ? 'Slow' : 'OK') : (b.error || 'Failed');
+ return `
+ | ${esc(row.voice_id)} |
+ ${b.ttfa_ms == null ? '—' : Math.round(b.ttfa_ms) + 'ms'} |
+ ${fmtSec(b.elapsed_sec)} |
+ ${fmtSec(b.audio_sec)} |
+ ${b.rtf == null ? '—' : Number(b.rtf).toFixed(2)} |
+ ${b.speed == null ? '—' : Number(b.speed).toFixed(2) + 'x'} |
+ ${esc(status)} |
+
`;
+ }).join('') : '| No voices benchmarked. |
';
+ }
+
+ async function initTurnControls() {
+ try {
+ const settings = await fetchJson('/api/settings');
+ if (q('bench-turn-llm-url')) q('bench-turn-llm-url').value = settings.conv_llm_url || settings.llm_url || '';
+ } catch (_) {}
+ try {
+ const d = await fetchJson('/api/stt-backends');
+ const sel = q('bench-turn-stt');
+ if (sel) sel.innerHTML = (d.backends || []).map(b => ``).join('') || '';
+ } catch (_) {}
+ try {
+ if (typeof refreshTtsBackendAvailability === 'function') await refreshTtsBackendAvailability();
+ const sel = q('bench-turn-tts-backend');
+ const backends = (window._ttsBackends || (typeof _ttsBackends !== 'undefined' ? _ttsBackends : []) || []).filter(Boolean);
+ if (sel) sel.innerHTML = backends.map(b => ``).join('') || '';
+ } catch (_) {}
+ }
+
+ async function fetchTurnModels() {
+ const btn = q('bench-turn-fetch-llm');
+ const sel = q('bench-turn-llm-model');
+ const url = q('bench-turn-llm-url')?.value.trim() || '';
+ if (btn) btn.disabled = true;
+ try {
+ const d = await fetchJson('/api/conversation/llm-models' + (url ? '?url=' + encodeURIComponent(url) : ''));
+ const models = d.models || [];
+ if (sel) sel.innerHTML = models.length ? models.map(m => ``).join('') : '';
+ } catch (e) {
+ if (sel) sel.innerHTML = '';
+ say('Model fetch failed: ' + e.message, 'error');
+ } finally { if (btn) btn.disabled = false; }
+ }
+
+ async function fetchTurnVoices() {
+ const btn = q('bench-turn-fetch-voices');
+ const sel = q('bench-turn-voice');
+ const backend = q('bench-turn-tts-backend')?.value || 'voice_clone';
+ if (btn) btn.disabled = true;
+ try {
+ const raw = await fetchJson('/api/tts-voices?backend=' + encodeURIComponent(backend));
+ const ids = (Array.isArray(raw) ? raw : []).map(v => typeof backendVoiceId === 'function' ? backendVoiceId(v) : (typeof v === 'string' ? v : (v.id || v.voice || v.name))).filter(Boolean);
+ if (sel) sel.innerHTML = ids.length ? ids.map(id => ``).join('') : '';
+ } catch (e) {
+ if (sel) sel.innerHTML = '';
+ say('Voice fetch failed: ' + e.message, 'error');
+ } finally { if (btn) btn.disabled = false; }
+ }
+
+ function updateTurnStats(stats) {
+ const max = stats.total_ms || 1;
+ const pairs = [
+ ['stt', stats.stt_ms], ['ttft', stats.llm_ttft_ms], ['llm', stats.llm_total_ms], ['tts', stats.tts_ms], ['total', stats.total_ms],
+ ];
+ pairs.forEach(([key, ms]) => {
+ const val = q('bench-turn-val-' + key);
+ const fill = q('bench-turn-fill-' + key);
+ if (val) val.textContent = fmtMs(ms);
+ if (fill) fill.style.width = max > 0 ? Math.min(100, ((ms || 0) / max) * 100) + '%' : '0%';
+ });
+ }
+
+ function addTurnLog(role, text) {
+ const log = q('bench-turn-log');
+ if (!log) return null;
+ log.querySelector('.conv-chat-welcome')?.remove();
+ const wrap = document.createElement('div');
+ wrap.className = `conv-bubble-wrap conv-bubble-wrap--${role}`;
+ const bubble = document.createElement('div');
+ bubble.className = `conv-bubble conv-bubble--${role}`;
+ bubble.textContent = text || '';
+ wrap.appendChild(bubble);
+ log.appendChild(wrap);
+ log.scrollTop = log.scrollHeight;
+ return bubble;
+ }
+
+ function addTurnHistory(total, ok) {
+ const hist = q('bench-turn-history');
+ if (!hist) return;
+ hist.querySelector('.conv-history-empty')?.remove();
+ turnHistoryCount++;
+ const item = document.createElement('div');
+ item.className = 'conv-hist-item';
+ item.innerHTML = `#${turnHistoryCount}${fmtMs(total)}`;
+ hist.prepend(item);
+ }
+
+ async function runTurnBenchmark() {
+ const audio = q('bench-turn-audio')?.files?.[0];
+ const text = q('bench-turn-text')?.value.trim() || '';
+ if (!audio && !text) { say('Choose turn audio or enter fallback text', 'error'); return; }
+ const btn = q('bench-turn-run');
+ const st = q('bench-turn-status');
+ if (btn) btn.disabled = true;
+ if (st) st.textContent = 'Running conversation turn...';
+ const t0 = Date.now();
+ const userBubble = addTurnLog('user', text || 'Transcribing audio...');
+ const assistantBubble = addTurnLog('assistant', '...');
+ let assistantText = '';
+ let lastStats = null;
+ try {
+ const fd = new FormData();
+ if (audio) fd.append('audio', audio, audio.name);
+ if (text) fd.append('text', text);
+ fd.append('stt_backend', q('bench-turn-stt')?.value || 'configured');
+ fd.append('llm_url', q('bench-turn-llm-url')?.value.trim() || '');
+ fd.append('llm_model', q('bench-turn-llm-model')?.value || '');
+ fd.append('tts_backend', q('bench-turn-tts-backend')?.value || 'voice_clone');
+ fd.append('tts_voice', q('bench-turn-voice')?.value || '');
+ fd.append('system_prompt', q('bench-turn-system')?.value.trim() || 'You are a helpful voice assistant.');
+ fd.append('history', '[]');
+ const resp = await fetch('/api/conversation/turn', { method: 'POST', body: fd });
+ if (!resp.ok) throw new Error('Server error ' + resp.status);
+ const reader = resp.body.getReader();
+ const dec = new TextDecoder();
+ let buf = '';
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ buf += dec.decode(value, { stream: true });
+ const lines = buf.split('\n');
+ buf = lines.pop();
+ for (const line of lines) {
+ if (!line.startsWith('data:')) continue;
+ let evt;
+ try { evt = JSON.parse(line.slice(5).trim()); } catch (_) { continue; }
+ if (evt.type === 'transcript' && userBubble) userBubble.textContent = evt.text || '(empty)';
+ if (evt.type === 'token') {
+ assistantText += evt.delta || '';
+ if (assistantBubble) assistantBubble.textContent = assistantText;
+ }
+ if (evt.type === 'llm_done') {
+ assistantText = evt.text || assistantText;
+ if (assistantBubble) assistantBubble.textContent = assistantText;
+ }
+ if (evt.type === 'audio' && evt.b64) {
+ const bytes = Uint8Array.from(atob(evt.b64), c => c.charCodeAt(0));
+ const url = URL.createObjectURL(new Blob([bytes], { type: evt.mime || 'audio/wav' }));
+ const audioEl = document.createElement('audio');
+ audioEl.controls = true;
+ audioEl.src = url;
+ audioEl.addEventListener('ended', () => URL.revokeObjectURL(url), { once: true });
+ q('bench-turn-log')?.appendChild(audioEl);
+ }
+ if (evt.type === 'stats') {
+ lastStats = evt;
+ updateTurnStats(evt);
+ }
+ if (evt.type === 'error') throw new Error(`[${evt.stage || 'turn'}] ${evt.message || 'Unknown error'}`);
+ }
+ }
+ const total = lastStats?.total_ms ?? (Date.now() - t0);
+ addTurnHistory(total, true);
+ if (st) st.textContent = `Finished in ${fmtMs(total)}.`;
+ say('Turn benchmark complete', 'success');
+ } catch (e) {
+ if (assistantBubble) assistantBubble.textContent = e.message;
+ addTurnHistory(Date.now() - t0, false);
+ if (st) st.textContent = 'Turn benchmark failed';
+ say('Turn benchmark failed: ' + e.message, 'error');
+ } finally {
+ if (btn) btn.disabled = false;
+ }
+ }
+
+ function bindBenchmarkSection() {
+ if (initialized || !q('bench-stt-run')) return;
+ initialized = true;
+ document.querySelectorAll('.bench-tab').forEach(btn => btn.addEventListener('click', () => setBenchTab(btn.dataset.benchTab)));
+ q('bench-stt-refresh')?.addEventListener('click', loadBenchmarkSttEngines);
+ q('bench-stt-run')?.addEventListener('click', runSttBenchmark);
+ q('bench-tts-run')?.addEventListener('click', runTtsBenchmark);
+ q('bench-turn-fetch-llm')?.addEventListener('click', fetchTurnModels);
+ q('bench-turn-fetch-voices')?.addEventListener('click', fetchTurnVoices);
+ q('bench-turn-run')?.addEventListener('click', runTurnBenchmark);
+ q('bench-turn-tts-backend')?.addEventListener('change', () => { if (q('bench-turn-voice')) q('bench-turn-voice').innerHTML = ''; });
+ loadBenchmarkSttEngines();
+ initTurnControls();
+ }
+
+ bindBenchmarkSection();
+ if (!initialized && document.body) {
+ const obs = new MutationObserver(() => bindBenchmarkSection());
+ obs.observe(document.body, { childList: true, subtree: true });
+ }
+})();
diff --git a/static/sections/s-performance.html b/static/sections/s-performance.html
index 653e734..6567622 100644
--- a/static/sections/s-performance.html
+++ b/static/sections/s-performance.html
@@ -2,10 +2,64 @@
Benchmark
-
Measure synthesis latency and real-time factor. Track trends across backends and voices over time.
+
Measure speech recognition, voice synthesis, and full conversation turns with the same local engines used by the app.
+
+
+
+
+
+
+
+
+
+
+
+
Speech to Text benchmark
+
Run every selected STT endpoint against the same reference clip and compare speed, accuracy, model, and compute device.
+
+
+
+
+
+
+ STT engines
+ No engines loaded.
+
+
+
+
+
+ Ready
+
+
+
+
+
+ | Engine | Model | Device | Time | Accuracy | Output |
|---|
+ | Run a benchmark to see results. |
+
+
+
Run benchmark
@@ -140,3 +194,73 @@
No benchmark history yet. Run a benchmark above to start tracking.
+
+
+
+
+
+
+
Conversation turn benchmark
+
Runs the same STT → LLM → TTS pipeline as Conversation Playground and records turn latency.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Ready
+
+
+
+
Run a turn to see transcript, reply, and timing.
+
+
+
Latency
+
+
Turn history
+
+
+