// ── Batch benchmark ─────────────────────────────────────────────────────── (function initBatchBenchmark() { const batchBackendSel = $('batch-backend-select'); const batchRunsSel = $('batch-runs'); const batchLoadBtn = $('batch-load-voices-btn'); const batchSearchInput = $('batch-voice-search'); const batchSelectAllBtn = $('batch-select-all-btn'); const batchSelectNoneBtn = $('batch-select-none-btn'); const batchVoiceList = $('batch-voice-list'); const batchSelCount = $('batch-selected-count'); const batchRunBtn = $('batch-run-btn'); const batchStopBtn = $('batch-stop-btn'); const perfRunSelectedBtn = $('perf-run-selected-btn'); const batchProgress = $('batch-progress'); const batchProgLabel = $('batch-progress-label'); const batchProgCount = $('batch-progress-count'); const batchProgBar = $('batch-progress-bar'); const batchResultsCard = $('batch-results-card'); const batchResultsLabel = $('batch-results-label'); const batchTbody = $('batch-tbody'); if (!batchRunBtn) return; let batchStopped = false; let batchResults = []; let batchVoices = []; let batchSelected = new Set(); let batchSortField = 'factor'; let batchSortDir = -1; // -1 = desc (fastest first) function populateBatchBackends() { if (!batchBackendSel) return; const cur = batchBackendSel.value; batchBackendSel.innerHTML = availableTtsBackends().map(b => `` ).join('') || ''; } populateBatchBackends(); function libraryVoice(id) { return (window._voices || []).find(v => v && v.id === id) || null; } function normalizeBatchVoice(value) { const id = typeof value === 'string' ? value : (value?.id || value?.voice || value?.name || value?.voice_id || ''); const meta = typeof value === 'object' ? (value.meta || libraryVoice(id) || value) : (libraryVoice(id) || null); return id ? { id, label: meta?.display_name || value?.label || meta?.name || value?.name || id, meta, } : null; } function batchVoiceStats(v) { const meta = v?.meta || libraryVoice(v?.id) || {}; const b = meta.benchmark || {}; const parts = []; const dur = Number(meta.duration); const speed = Number(b.speed ?? b.avg_speed); const rtf = Number(b.rtf ?? b.avg_rtf); if (Number.isFinite(dur) && dur > 0) parts.push(`${dur.toFixed(1)}s`); if (Number.isFinite(speed) && speed > 0) parts.push(`${speed.toFixed(2)}x`); else if (Number.isFinite(rtf) && rtf > 0) parts.push(`RTF ${rtf.toFixed(2)}`); return parts.join(' · '); } function batchColor(id) { const palette = ['#3b82f6', '#10b981', '#8b5cf6', '#f59e0b', '#ef4444', '#ec4899', '#06b6d4', '#84cc16']; let h = 0; for (let i = 0; i < String(id || '').length; i++) h = (h * 31 + String(id).charCodeAt(i)) >>> 0; return palette[h % palette.length]; } function batchAvatar(v) { const meta = v?.meta || libraryVoice(v?.id) || {}; if (meta.has_picture) return ``; const icon = window.voiceAvatarIcon ? window.voiceAvatarIcon(meta.avatar, 28) : null; if (icon) return icon.replace(/vp-avatar/g, 'batch-voice-avatar'); const init = (v.label || v.id || '?')[0].toUpperCase(); return `${escHtml(init)}`; } function batchHistoryExtraFields(voice, backend) { const item = batchVoices.find(v => v.id === voice) || {}; const meta = item.meta || libraryVoice(voice) || {}; return { voiceMeta: { label: meta.display_name || meta.name || item.label || voice, lang: meta.lang || meta.language || '', gender: meta.gender || '', flag: meta.flag || '', avatar: meta.avatar || '', has_picture: !!meta.has_picture, }, device: typeof backendComputeDevice === 'function' ? backendComputeDevice(backend) : '', }; } function batchSearchText(v) { const meta = v.meta || {}; return [v.id, v.label, meta.display_name, meta.name, meta.lang, meta.language, Array.isArray(meta.tags) ? meta.tags.join(' ') : meta.tags, batchVoiceStats(v)] .filter(Boolean).join(' ').toLowerCase(); } function syncRunSelectedLabel() { const count = batchSelected.size; if (perfRunSelectedBtn) { perfRunSelectedBtn.disabled = count === 0; perfRunSelectedBtn.innerHTML = ` ${count ? `Run ${count} selected voice${count === 1 ? '' : 's'}` : 'Run selected voices'}`; } } function updateSelCount() { const total = batchVoices.length; const checked = batchSelected.size; const visible = filteredBatchVoices().length; if (batchSelCount) batchSelCount.textContent = total ? `${checked} of ${total} selected${visible !== total ? ` · ${visible} shown` : ''}` : ''; batchRunBtn.disabled = checked === 0; syncRunSelectedLabel(); } function filteredBatchVoices() { const q = (batchSearchInput?.value || '').trim().toLowerCase(); return q ? batchVoices.filter(v => batchSearchText(v).includes(q)) : batchVoices; } function renderBatchVoiceList() { if (!batchVoiceList) return; const visible = filteredBatchVoices(); if (!batchVoices.length) { batchVoiceList.innerHTML = '
No voices found. Fetch voices above or reload from backend.
'; updateSelCount(); return; } if (!visible.length) { batchVoiceList.innerHTML = '
No matching voices.
'; updateSelCount(); return; } const activeIds = new Set(activeVoiceIds()); batchVoiceList.innerHTML = visible.map(v => { const checked = batchSelected.has(v.id); const isActive = activeIds.has(v.id); const stats = batchVoiceStats(v); const meta = v.meta || libraryVoice(v.id) || {}; const flag = meta.flag || ''; const lang = meta.lang || meta.language || ''; const genderMap = { M: '♂', F: '♀', N: '⚬' }; const gender = genderMap[meta.gender] || ''; const langGender = [flag || lang, gender].filter(Boolean).join(' '); return ``; }).join(''); batchVoiceList.querySelectorAll('.batch-voice-cb').forEach(cb => cb.addEventListener('change', () => { if (cb.checked) batchSelected.add(cb.value); else batchSelected.delete(cb.value); renderBatchVoiceList(); })); updateSelCount(); } function buildVoiceList(voices, opts = {}) { const previous = opts.preserve ? new Set(batchSelected) : null; const byId = new Map(); (voices || []).map(normalizeBatchVoice).filter(Boolean).forEach(v => byId.set(v.id, v)); batchVoices = Array.from(byId.values()).sort((a, b) => (a.label || a.id).localeCompare(b.label || b.id)); if (previous) { batchSelected = new Set(batchVoices.filter(v => previous.has(v.id)).map(v => v.id)); } else if (opts.keepSelection) { batchSelected = new Set(Array.from(batchSelected).filter(id => byId.has(id))); } else { const activeIds = new Set(activeVoiceIds()); batchSelected = new Set(batchVoices.filter(v => activeIds.has(v.id)).map(v => v.id)); } renderBatchVoiceList(); } async function populateFromLibrary() { try { if ((!window._voices || !window._voices.length) && typeof loadVoiceLibrary === 'function') await loadVoiceLibrary(); const voices = (window._voices || []).filter(v => v.enabled !== false).map(v => ({ id: v.id, label: v.display_name || v.name || v.id, meta: v })); buildVoiceList(voices); } catch (e) { if (batchVoiceList) batchVoiceList.innerHTML = `
Voice library unavailable: ${escHtml(e.message)}
`; updateSelCount(); } } populateFromLibrary(); async function loadBatchVoicesFromBackend() { const backend = batchBackendSel.value; if (!backend) { toast('Select a backend first', 'error'); return; } batchLoadBtn.disabled = true; batchVoiceList.innerHTML = '
Loading from backend...
'; try { const raw = await fetch('/api/tts-voices?backend=' + encodeURIComponent(backend)).then(r => r.json()); const voices = raw.map(v => { const id = backendVoiceId(v); return { id, label: id, meta: libraryVoice(id) || (typeof v === 'object' ? v : null) }; }).filter(v => v.id); buildVoiceList(voices); } catch(e) { batchVoiceList.innerHTML = `
Failed: ${escHtml(e.message)}
`; updateSelCount(); } finally { batchLoadBtn.disabled = false; } } batchLoadBtn.addEventListener('click', loadBatchVoicesFromBackend); batchSearchInput?.addEventListener('input', renderBatchVoiceList); batchSelectAllBtn.addEventListener('click', () => { batchVoices.forEach(v => batchSelected.add(v.id)); renderBatchVoiceList(); }); batchSelectNoneBtn.addEventListener('click', () => { batchSelected.clear(); renderBatchVoiceList(); }); window.addEventListener('benchmark:tts-voices-fetched', e => { const detail = e.detail || {}; if (detail.backend && batchBackendSel) batchBackendSel.value = detail.backend; buildVoiceList(detail.voices || []); }); function batchSortValue(r, field) { const factor = r.ok && r.avgRtf > 0 ? 1 / r.avgRtf : null; switch (field) { case 'voice': return (r.voice || '').toLowerCase(); case 'lang': return (r.flag || r.lang || '').toLowerCase(); case 'gender': return (r.gender || '').toLowerCase(); case 'device': return typeof backendComputeDevice === 'function' ? backendComputeDevice(r.backend || '').toLowerCase() : ''; case 'latency': return r.ok ? r.avgLatency : 999999; case 'best': return r.ok ? r.minLatency : 999999; case 'duration': return r.ok && r.avgAudio > 0 ? r.avgAudio : 999999; case 'factor': return factor != null ? factor : -1; case 'time': return r.ok ? r.avgLatency / 1000 : 999999; case 'wpm': return r.ok && r.avgAudio > 0 && r.wordCount > 0 ? Math.round(r.wordCount / (r.avgAudio / 60)) : -1; default: return 0; } } function renderBatchResults() { if (!batchResults.length) { batchResultsCard.style.display = 'none'; return; } batchResultsCard.style.display = ''; const sorted = batchResults.slice().sort((a, b) => { if (a.ok && !b.ok) return -1; if (!a.ok && b.ok) return 1; const av = batchSortValue(a, batchSortField), bv = batchSortValue(b, batchSortField); return typeof av === 'string' ? av.localeCompare(bv) * batchSortDir : (av - bv) * batchSortDir; }); const okCount = batchResults.filter(r => r.ok).length; if (batchResultsLabel) batchResultsLabel.textContent = `${okCount} / ${batchResults.length} voices — ${batchBackendSel.value}`; // Update header sort indicators document.querySelectorAll('#batch-table th[data-sort]').forEach(th => { th.classList.remove('perf-sort-asc', 'perf-sort-desc'); if (th.dataset.sort === batchSortField) th.classList.add(batchSortDir === 1 ? 'perf-sort-asc' : 'perf-sort-desc'); }); batchTbody.innerHTML = sorted.map(r => { const factor = r.ok && r.avgRtf > 0 ? 1 / r.avgRtf : null; const timeSec = r.ok && r.avgLatency > 0 ? r.avgLatency / 1000 : null; const wpm = r.ok && r.avgAudio > 0 && r.wordCount > 0 ? Math.round(r.wordCount / (r.avgAudio / 60)) : null; const factorCls = factor == null ? '' : factor >= 1 ? 'perf-good' : 'perf-slow'; const trend = r.trend ? `${r.trend.label}` : ''; const device = typeof backendComputeDevice === 'function' ? backendComputeDevice(r.backend || batchBackendSel.value) : 'Unknown'; const deviceCls = typeof backendComputeDeviceClass === 'function' ? backendComputeDeviceClass(r.backend || batchBackendSel.value) : ''; const genderMap = { M: '♂', F: '♀', N: '⚬' }; return ` ${escHtml(r.voice)} ${escHtml(r.flag || r.lang || '—')} ${escHtml(genderMap[r.gender] || '—')} ${escHtml(device)} ${r.ok ? Math.round(r.avgLatency) + ' ms' : '—'} ${r.ok ? r.minLatency + ' ms' : '—'} ${r.ok && r.avgAudio > 0 ? r.avgAudio.toFixed(1) + 's' : '—'} ${factor != null ? factor.toFixed(2) + '×' : '—'}${trend} ${timeSec != null ? timeSec.toFixed(1) + 's' : '—'} ${wpm != null ? wpm + ' wpm' : '—'} ${r.ok ? 'OK' : `${escHtml(r.error || 'Failed')}`} `; }).join(''); } // Wire sortable headers document.querySelector('#batch-table')?.addEventListener('click', e => { const th = e.target.closest('th[data-sort]'); if (!th) return; const field = th.dataset.sort; if (field === batchSortField) batchSortDir *= -1; else { batchSortField = field; batchSortDir = field === 'voice' || field === 'lang' || field === 'gender' ? 1 : -1; } renderBatchResults(); }); async function runBatchBenchmark() { const topBackend = $('perf-backend-select')?.value || ''; if (topBackend && batchBackendSel && batchBackendSel.value !== topBackend) batchBackendSel.value = topBackend; const topRuns = $('perf-runs')?.value || ''; if (topRuns && batchRunsSel && Array.from(batchRunsSel.options).some(o => o.value === topRuns)) batchRunsSel.value = topRuns; const singleVoice = $('perf-voice-select')?.value || ''; if (!batchSelected.size && singleVoice && batchVoices.some(v => v.id === singleVoice)) { batchSelected.add(singleVoice); renderBatchVoiceList(); } const backend = batchBackendSel.value; const text = $('perf-text')?.value.trim(); const runs = parseInt(batchRunsSel.value) || 1; const validIds = new Set(batchVoices.map(v => v.id)); const selected = Array.from(batchSelected).filter(id => validIds.has(id)); if (!backend) { toast('Select a backend first', 'error'); return; } if (!text) { toast('Enter sample text in the single-voice form above', 'error'); return; } if (!selected.length) { toast('Select one or more voices first', 'error'); return; } batchStopped = false; batchResults = []; batchRunBtn.disabled = true; if (perfRunSelectedBtn) perfRunSelectedBtn.disabled = true; batchStopBtn.disabled = false; batchProgress.style.display = ''; batchResultsCard.style.display = 'none'; for (let vi = 0; vi < selected.length; vi++) { if (batchStopped) break; const voice = selected[vi]; batchProgLabel.textContent = `${voice} (${vi + 1} / ${selected.length})`; batchProgCount.textContent = `${vi + 1} / ${selected.length}`; batchProgBar.style.width = `${Math.round((vi / selected.length) * 100)}%`; const _vm = batchVoices.find(bv => bv.id === voice); const _meta = _vm?.meta || libraryVoice(voice) || {}; const entry = { voice, backend, ok: false, avgLatency: 0, minLatency: 0, avgRtf: 0, avgAudio: 0, wordCount: text ? text.trim().split(/\s+/).length : 0, lang: _meta.lang || _meta.language || '', flag: _meta.flag || '', gender: _meta.gender || '', error: '' }; const rowRunResults = []; for (let ri = 0; ri < runs; ri++) { if (batchStopped) break; try { const t0 = performance.now(); const blob = await fetchTtsPreviewBlob(voice, text, 'wav', '', backend); const lat = Math.round(performance.now() - t0); let dur = 0; try { const ac = new (window.AudioContext || window.webkitAudioContext)(); const buf = await ac.decodeAudioData(await blob.arrayBuffer()); dur = buf.duration; ac.close(); } catch(_) {} rowRunResults.push({ lat, dur }); } catch(e) { entry.error = e.message; break; } } if (rowRunResults.length) { entry.ok = true; entry.avgLatency = rowRunResults.reduce((sum, r) => sum + r.lat, 0) / rowRunResults.length; entry.minLatency = Math.min(...rowRunResults.map(r => r.lat)); const durRows = rowRunResults.filter(r => r.dur > 0); entry.avgAudio = durRows.length ? durRows.reduce((sum, r) => sum + r.dur, 0) / durRows.length : 0; const rtfArr = durRows.map(r => r.lat / 1000 / r.dur); entry.avgRtf = rtfArr.length ? rtfArr.reduce((sum, value) => sum + value, 0) / rtfArr.length : 0; if (entry.avgRtf > 0) { const prev = perfHistoryLoad().filter(e => e.backend === backend && e.voice === voice && typeof e.avgRtf === 'number'); if (prev.length) { const prevRtf = prev[prev.length - 1].avgRtf; const delta = entry.avgRtf - prevRtf; const pct = Math.abs(delta / Math.max(prevRtf, 0.01)) * 100; if (pct < 5) entry.trend = { cls: 'perf-trend-stable', label: 'stable' }; else if (delta < 0) entry.trend = { cls: 'perf-trend-better', label: `${pct.toFixed(0)}% faster` }; else entry.trend = { cls: 'perf-trend-worse', label: `${pct.toFixed(0)}% slower` }; } perfHistoryAdd({ ts: Date.now(), backend, voice, textLen: text.length, ...batchHistoryExtraFields(voice, backend), avgLatencyMs: entry.avgLatency, minLatencyMs: entry.minLatency, maxLatencyMs: Math.max(...rowRunResults.map(r => r.lat)), avgRtf: entry.avgRtf, runCount: rowRunResults.length, allOk: true, }); } } batchResults.push(entry); renderBatchResults(); } batchProgBar.style.width = '100%'; batchProgLabel.textContent = batchStopped ? `Stopped after ${batchResults.length} voice${batchResults.length !== 1 ? 's' : ''}.` : `Done - ${batchResults.length} voice${batchResults.length !== 1 ? 's' : ''} benchmarked.`; batchStopBtn.disabled = true; batchRunBtn.disabled = false; syncRunSelectedLabel(); renderPerfHistory(); const ok = batchResults.filter(r => r.ok); const best = ok.slice().sort((a, b) => a.avgRtf - b.avgRtf)[0]; toast( `Batch done: ${ok.length}/${batchResults.length} OK` + (best ? `, best RTF ${best.avgRtf.toFixed(2)} (${best.voice})` : ''), ok.length < batchResults.length ? 'error' : 'success' ); // Sync results into voice meta so My Voices Factor/WPM columns update const syncable = batchResults.filter(r => r.ok && r.avgRtf > 0 && r.avgAudio > 0); if (syncable.length && typeof window.loadVoiceLibrary === 'function') { const benchText = ($('perf-text')?.value || '').trim(); await Promise.allSettled(syncable.map(r => fetch('/api/voice/meta', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ voice_id: r.voice, benchmark: { ok: true, text: benchText, elapsed_sec: r.avgLatency / 1000, audio_sec: r.avgAudio, speed: 1 / r.avgRtf, rtf: r.avgRtf, ttfa_ms: r.minLatency, } }) }) )); await window.loadVoiceLibrary(); } } batchRunBtn.addEventListener('click', runBatchBenchmark); perfRunSelectedBtn?.addEventListener('click', runBatchBenchmark); batchStopBtn.addEventListener('click', () => { batchStopped = true; batchStopBtn.disabled = true; batchProgLabel.textContent = 'Stopping after current voice...'; }); })(); // ── Benchmark section ───────────────────────────────────────────────────── (function initBenchmarkSection() { let initialized = false; let sttEngines = []; let turnHistoryCount = 0; const q = (id) => document.getElementById(id); const fmtMs = (ms) => ms == null ? '—' : ms >= 1000 ? (ms / 1000).toFixed(2) + 's' : Math.round(ms) + 'ms'; const fmtSec = (sec) => sec == null ? '—' : Number(sec).toFixed(Number(sec) < 10 ? 2 : 1) + 's'; const esc = (s) => typeof escHtml === 'function' ? escHtml(s) : String(s ?? '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); const say = (msg, type = '') => typeof toast === 'function' ? toast(msg, type) : console.log(msg); function setBenchTab(name) { document.querySelectorAll('.bench-tab').forEach(btn => btn.classList.toggle('active', btn.dataset.benchTab === name)); document.querySelectorAll('.bench-pane').forEach(pane => pane.classList.toggle('active', pane.dataset.benchPane === name)); } async function fetchJson(url, opts) { const r = await fetch(url, opts); if (!r.ok) { let detail = r.statusText || `HTTP ${r.status}`; try { const d = await r.json(); detail = typeof d.detail === 'string' ? d.detail : JSON.stringify(d.detail || d); } catch (_) { try { detail = await r.text(); } catch (_) {} } throw new Error(detail || `HTTP ${r.status}`); } return r.json(); } function dedupeEngines(items) { const seen = new Set(); const out = []; for (const item of items) { const key = `${item.url || ''}|${item.model || ''}|${item.id || item.label || ''}`; if (seen.has(key) || !item.url) continue; seen.add(key); out.push(item); } return out; } function benchmarkModelList(engine) { const values = []; const add = (v) => { const s = String(v || '').trim(); if (s && !values.includes(s)) values.push(s); }; add(engine.model); (engine.models || []).forEach(add); if (!values.length) add('whisper-1'); return values; } async function fetchBenchmarkModels(engine) { if (!engine?.url) return []; try { const d = await fetchJson('/api/engine-models?type=stt&url=' + encodeURIComponent(engine.url)); return Array.isArray(d.models) ? d.models.filter(Boolean) : []; } catch (e) { console.warn('[benchmark models]', engine.label || engine.id, e); return []; } } async function hydrateBenchmarkModels(engines) { await Promise.all(engines.map(async (engine) => { const found = await fetchBenchmarkModels(engine); engine.models = benchmarkModelList({ ...engine, models: found.length ? found : engine.models }); if (!engine.models.includes(engine.model)) engine.model = engine.models[0] || engine.model || 'whisper-1'; })); } function renderBenchmarkModelOptions(index, filter = '') { const engine = sttEngines[index]; const picker = document.querySelector(`.bench-model-picker[data-index="${index}"]`); const list = picker?.querySelector('.bench-model-options'); if (!engine || !list) return; const qv = String(filter || '').trim().toLowerCase(); const models = benchmarkModelList(engine).filter(m => !qv || m.toLowerCase().includes(qv)); list.innerHTML = models.length ? models.map(m => ` `).join('') : '
No matching models
'; } function setBenchmarkModel(index, model) { const engine = sttEngines[index]; if (!engine) return; engine.model = String(model || '').trim() || engine.model || 'whisper-1'; const picker = document.querySelector(`.bench-model-picker[data-index="${index}"]`); const hidden = picker?.querySelector('.bench-engine-model'); const label = picker?.querySelector('.bench-model-label'); if (hidden) hidden.value = engine.model; if (label) label.textContent = engine.model; picker?.querySelector('.bench-model-menu')?.setAttribute('hidden', ''); picker?.querySelector('.bench-model-trigger')?.classList.remove('open'); } function closeBenchmarkModelPickers(except = null) { document.querySelectorAll('.bench-model-picker').forEach(picker => { if (except && picker === except) return; picker.querySelector('.bench-model-menu')?.setAttribute('hidden', ''); picker.querySelector('.bench-model-trigger')?.classList.remove('open'); }); } function bindBenchmarkModelPickers() { document.querySelectorAll('.bench-model-picker').forEach(picker => { picker.addEventListener('click', (ev) => ev.stopPropagation()); const index = Number(picker.dataset.index); const trigger = picker.querySelector('.bench-model-trigger'); const menu = picker.querySelector('.bench-model-menu'); const search = picker.querySelector('.bench-model-search'); renderBenchmarkModelOptions(index); trigger?.addEventListener('click', (ev) => { ev.preventDefault(); ev.stopPropagation(); const opening = menu?.hasAttribute('hidden'); closeBenchmarkModelPickers(picker); if (opening) { menu?.removeAttribute('hidden'); trigger.classList.add('open'); search.value = ''; renderBenchmarkModelOptions(index); setTimeout(() => search?.focus(), 0); } else { menu?.setAttribute('hidden', ''); trigger.classList.remove('open'); } }); search?.addEventListener('input', () => renderBenchmarkModelOptions(index, search.value)); picker.querySelector('.bench-model-options')?.addEventListener('click', (ev) => { const opt = ev.target.closest('.bench-model-option'); if (!opt) return; setBenchmarkModel(index, opt.dataset.model || ''); }); }); } async function loadBenchmarkSttEngines() { const list = q('bench-stt-engines'); const count = q('bench-stt-engine-count'); if (!list) return; list.innerHTML = '
Checking STT engines...
'; let settings = {}; const items = []; try { settings = await fetchJson('/api/settings'); } catch (_) {} try { const d = await fetchJson('/api/stt-backends'); (d.backends || []).forEach(b => items.push({ id: b.id, backend: b.id, label: b.label || b.id, url: b.url, model: (b.models && b.models[0]) || b.model || 'whisper-1', models: b.models || [], available: !!b.available, source: 'configured', })); } catch (e) { console.warn('[benchmark stt backends]', e); } try { const d = await fetchJson('/api/local-containers'); (d.containers || []).filter(c => c.role === 'stt' || c.role === 'stt+tts').forEach(c => { const url = c.port ? `http://host.docker.internal:${c.port}` : ''; const model = settings?.engine_local_models?.['dc-' + c.name] || c.model || 'whisper-1'; items.push({ id: c.name, label: c.label || c.name, url, model, models: c.models || [], available: c.running === true || c.status === 'running', source: c.stack || 'docker', }); }); } catch (e) { console.warn('[benchmark local containers]', e); } sttEngines = dedupeEngines(items); if (count) count.textContent = sttEngines.length ? `Loading models for ${sttEngines.length} engine${sttEngines.length === 1 ? '' : 's'}...` : 'No engines found.'; await hydrateBenchmarkModels(sttEngines); if (count) count.textContent = sttEngines.length ? `${sttEngines.length} engine${sttEngines.length === 1 ? '' : 's'} loaded.` : 'No engines found.'; if (!sttEngines.length) { list.innerHTML = '
No STT engines found. Check Engines -> Speech to Text.
'; return; } list.innerHTML = sttEngines.map((e, i) => ` `).join(''); bindBenchmarkModelPickers(); } async function loadBenchmarkVoiceLibrary() { const sel = q('bench-stt-library-voice'); if (!sel) return; const picker = window.BenchmarkVoicePicker; if (picker) picker.populate('bench-stt-library-voice', [], { placeholder: 'Loading voices...', empty: 'Loading voices...' }); else sel.innerHTML = ''; try { const voices = await fetchJson('/api/voices'); const usable = (voices || []) .filter(v => v && v.enabled !== false && v.has_ref && (v.transcript || '').trim()) .sort((a, b) => String(a.display_name || a.id).localeCompare(String(b.display_name || b.id))); const items = usable.map(v => ({ id: v.id, label: v.display_name || v.name || v.id, meta: v })); if (picker) { picker.populate('bench-stt-library-voice', items, { placeholder: '-- choose from voice library --', empty: 'No library voices with reference transcripts found', }); } else { sel.innerHTML = '' + usable .map(v => ``) .join(''); } const note = q('bench-stt-source-note'); if (note) note.textContent = usable.length ? `${usable.length} voices with reference text available.` : 'No library voices with reference transcripts found.'; } catch (e) { if (picker) picker.populate('bench-stt-library-voice', [], { placeholder: 'Voice library unavailable', empty: 'Voice library unavailable' }); else sel.innerHTML = ''; const note = q('bench-stt-source-note'); if (note) note.textContent = 'Could not load voice library: ' + e.message; } } async function useBenchmarkLibraryVoice() { const sel = q('bench-stt-library-voice'); const voiceId = sel?.value || ''; if (!voiceId) { say('Choose a voice library sample first', 'error'); return; } const btn = q('bench-stt-load-voice'); const note = q('bench-stt-source-note'); if (btn) btn.disabled = true; if (note) note.textContent = 'Loading voice sample...'; try { const d = await fetchJson('/api/voice-load', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ voice_id: voiceId }), }); if (q('bench-stt-source-id')) q('bench-stt-source-id').value = d.id || ''; if (q('bench-stt-ref-text')) q('bench-stt-ref-text').value = d.transcript || ''; if (q('bench-stt-audio')) q('bench-stt-audio').value = ''; if (q('bench-stt-ref-file')) q('bench-stt-ref-file').value = ''; if (note) note.textContent = `${d.voice_id || voiceId} loaded from library${d.duration ? ` (${Number(d.duration).toFixed(1)}s)` : ''}.`; say('Voice library sample loaded', 'success'); } catch (e) { if (note) note.textContent = 'Voice load failed.'; say('Voice load failed: ' + e.message, 'error'); } finally { if (btn) btn.disabled = false; } } function selectedSttEngines() { return [...document.querySelectorAll('.bench-stt-engine-check:checked')].map(cb => { const i = Number(cb.dataset.index); const item = { ...sttEngines[i] }; const modelInput = document.querySelector(`.bench-engine-model[data-index="${i}"]`); item.model = modelInput?.value.trim() || item.model || 'whisper-1'; return item; }).filter(Boolean); } async function runSttBenchmark() { const audio = q('bench-stt-audio')?.files?.[0]; const sourceId = q('bench-stt-source-id')?.value || ''; const refFile = q('bench-stt-ref-file')?.files?.[0]; const refText = q('bench-stt-ref-text')?.value || ''; const engines = selectedSttEngines(); if (!audio && !sourceId) { say('Choose an audio file or load a voice library sample first', 'error'); return; } if (!refFile && !refText.trim()) { say('Add a reference .txt or paste reference text', 'error'); return; } if (!engines.length) { say('Select at least one STT engine', 'error'); return; } const btn = q('bench-stt-run'); const st = q('bench-stt-status'); const tbody = q('bench-stt-table')?.querySelector('tbody'); if (btn) btn.disabled = true; if (st) st.textContent = `Running ${engines.length} STT benchmark${engines.length === 1 ? '' : 's'}...`; if (tbody) tbody.innerHTML = `Benchmarking ${engines.length} engines...`; try { const fd = new FormData(); if (audio) fd.append('audio', audio, audio.name); if (sourceId) fd.append('source_id', sourceId); if (refFile) fd.append('reference_file', refFile, refFile.name); fd.append('reference_text', refText); fd.append('engines_json', JSON.stringify(engines)); const d = await fetchJson('/api/stt-benchmark', { method: 'POST', body: fd }); renderSttResults(d); if (st) st.textContent = `Finished ${d.results?.length || 0} STT engines.`; say('STT benchmark complete', 'success'); } catch (e) { if (st) st.textContent = 'STT benchmark failed'; if (tbody) tbody.innerHTML = `${esc(e.message)}`; say('STT benchmark failed: ' + e.message, 'error'); } finally { if (btn) btn.disabled = false; } } function renderSttResults(d) { const tbody = q('bench-stt-table')?.querySelector('tbody'); const summary = q('bench-stt-summary'); const rows = d.results || []; if (summary) { summary.innerHTML = [ ['Engines', String(rows.length)], ['Reference words', String(d.reference_words || 0)], ['Best time', d.best_time_sec == null ? '—' : fmtSec(d.best_time_sec)], ['Best accuracy', d.best_accuracy == null ? '—' : d.best_accuracy.toFixed(1) + '%'], ].map(([label, val]) => `${esc(val)} ${esc(label)}`).join(''); } if (!tbody) return; tbody.innerHTML = rows.map(r => { const acc = r.accuracy == null ? '—' : `${Number(r.accuracy).toFixed(1)}%`; const out = r.ok ? (r.output || '') : `⚠ ${r.error || 'failed'}`; return ` ${esc(r.engine)}${esc(r.url || '')} ${esc(r.model || '')} ${esc(r.device || 'Unknown')} ${fmtSec(r.time_sec)} ${esc(acc)} ${esc(out)} `; }).join(''); } async function runTtsBenchmark() { const text = q('bench-tts-text')?.value.trim() || ''; if (!text) { say('Enter a sample sentence first', 'error'); return; } const activeOnly = (q('bench-tts-scope')?.value || 'active') === 'active'; const btn = q('bench-tts-run'); const st = q('bench-tts-status'); const tbody = q('bench-tts-table')?.querySelector('tbody'); if (btn) btn.disabled = true; if (st) st.textContent = 'Running TTS benchmark...'; if (tbody) tbody.innerHTML = 'Benchmarking voices...'; try { const d = await fetchJson('/api/voices/benchmark', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text, active_only: activeOnly }), }); renderTtsResults(d); if (st) st.textContent = `Finished ${d.benchmarked || 0} voices.`; say('TTS benchmark complete', d.errors?.length ? 'error' : 'success'); } catch (e) { if (st) st.textContent = 'TTS benchmark failed'; if (tbody) tbody.innerHTML = `${esc(e.message)}`; say('TTS benchmark failed: ' + e.message, 'error'); } finally { if (btn) btn.disabled = false; } } function renderTtsResults(d) { const tbody = q('bench-tts-table')?.querySelector('tbody'); const summary = q('bench-tts-summary'); const rows = d.voices || []; const okRows = rows.map(r => r.benchmark).filter(b => b && b.ok); const avg = (key) => { const vals = okRows.map(b => Number(b[key])).filter(Number.isFinite); return vals.length ? vals.reduce((a, b) => a + b, 0) / vals.length : null; }; if (summary) { summary.innerHTML = [ ['Voices', String(rows.length)], ['OK', String(okRows.length)], ['Avg TTFA', avg('ttfa_ms') == null ? '—' : Math.round(avg('ttfa_ms')) + 'ms'], ['Avg total', avg('elapsed_sec') == null ? '—' : fmtSec(avg('elapsed_sec'))], ].map(([label, val]) => `${esc(val)} ${esc(label)}`).join(''); } if (!tbody) return; tbody.innerHTML = rows.length ? rows.map(row => { const b = row.benchmark || {}; const status = b.ok ? (b.realtime_ok === false ? 'Slow' : 'OK') : (b.error || 'Failed'); return ` ${esc(row.voice_id)} ${b.ttfa_ms == null ? '—' : Math.round(b.ttfa_ms) + 'ms'} ${fmtSec(b.elapsed_sec)} ${fmtSec(b.audio_sec)} ${b.rtf == null ? '—' : Number(b.rtf).toFixed(2)} ${b.speed == null ? '—' : Number(b.speed).toFixed(2) + 'x'} ${esc(status)} `; }).join('') : 'No voices benchmarked.'; } async function initTurnControls() { try { const settings = await fetchJson('/api/settings'); if (q('bench-turn-llm-url')) q('bench-turn-llm-url').value = settings.conv_llm_url || settings.llm_url || ''; } catch (_) {} try { const d = await fetchJson('/api/stt-backends'); const sel = q('bench-turn-stt'); if (sel) sel.innerHTML = (d.backends || []).map(b => ``).join('') || ''; } catch (_) {} try { if (typeof refreshTtsBackendAvailability === 'function') await refreshTtsBackendAvailability(); const sel = q('bench-turn-tts-backend'); const backends = (window._ttsBackends || (typeof _ttsBackends !== 'undefined' ? _ttsBackends : []) || []).filter(Boolean); if (sel) sel.innerHTML = backends.map(b => ``).join('') || ''; } catch (_) {} } async function fetchTurnModels() { const btn = q('bench-turn-fetch-llm'); const sel = q('bench-turn-llm-model'); const url = q('bench-turn-llm-url')?.value.trim() || ''; if (btn) btn.disabled = true; try { const d = await fetchJson('/api/conversation/llm-models' + (url ? '?url=' + encodeURIComponent(url) : '')); const models = d.models || []; if (sel) sel.innerHTML = models.length ? models.map(m => ``).join('') : ''; } catch (e) { if (sel) sel.innerHTML = ''; say('Model fetch failed: ' + e.message, 'error'); } finally { if (btn) btn.disabled = false; } } async function fetchTurnVoices() { const btn = q('bench-turn-fetch-voices'); const sel = q('bench-turn-voice'); const backend = q('bench-turn-tts-backend')?.value || 'voice_clone'; const picker = window.BenchmarkVoicePicker; if (btn) btn.disabled = true; try { const raw = await fetchJson('/api/tts-voices?backend=' + encodeURIComponent(backend)); const items = (Array.isArray(raw) ? raw : []).map(v => { const id = typeof backendVoiceId === 'function' ? backendVoiceId(v) : (typeof v === 'string' ? v : (v.id || v.voice || v.name)); return id ? { id, label: id, meta: (window._voices || []).find(x => x && x.id === id) || (typeof v === 'object' ? v : null) } : null; }).filter(Boolean); if (picker) picker.populate('bench-turn-voice', items, { placeholder: 'Fetch voices', empty: 'No voices' }); else if (sel) sel.innerHTML = items.length ? items.map(v => ``).join('') : ''; } catch (e) { if (picker) picker.populate('bench-turn-voice', [], { placeholder: 'Fetch failed', empty: 'Fetch failed' }); else if (sel) sel.innerHTML = ''; say('Voice fetch failed: ' + e.message, 'error'); } finally { if (btn) btn.disabled = false; } } function updateTurnStats(stats) { const max = stats.total_ms || 1; const pairs = [ ['stt', stats.stt_ms], ['ttft', stats.llm_ttft_ms], ['llm', stats.llm_total_ms], ['tts', stats.tts_ms], ['total', stats.total_ms], ]; pairs.forEach(([key, ms]) => { const val = q('bench-turn-val-' + key); const fill = q('bench-turn-fill-' + key); if (val) val.textContent = fmtMs(ms); if (fill) fill.style.width = max > 0 ? Math.min(100, ((ms || 0) / max) * 100) + '%' : '0%'; }); } function addTurnLog(role, text) { const log = q('bench-turn-log'); if (!log) return null; log.querySelector('.conv-chat-welcome')?.remove(); const wrap = document.createElement('div'); wrap.className = `conv-bubble-wrap conv-bubble-wrap--${role}`; const bubble = document.createElement('div'); bubble.className = `conv-bubble conv-bubble--${role}`; bubble.textContent = text || ''; wrap.appendChild(bubble); log.appendChild(wrap); log.scrollTop = log.scrollHeight; return bubble; } function addTurnHistory(total, ok) { const hist = q('bench-turn-history'); if (!hist) return; hist.querySelector('.conv-history-empty')?.remove(); turnHistoryCount++; const item = document.createElement('div'); item.className = 'conv-hist-item'; item.innerHTML = `#${turnHistoryCount}${fmtMs(total)}`; hist.prepend(item); } async function runTurnBenchmark() { const audio = q('bench-turn-audio')?.files?.[0]; const text = q('bench-turn-text')?.value.trim() || ''; if (!audio && !text) { say('Choose turn audio or enter fallback text', 'error'); return; } const btn = q('bench-turn-run'); const st = q('bench-turn-status'); if (btn) btn.disabled = true; if (st) st.textContent = 'Running conversation turn...'; const t0 = Date.now(); const userBubble = addTurnLog('user', text || 'Transcribing audio...'); const assistantBubble = addTurnLog('assistant', '...'); let assistantText = ''; let lastStats = null; try { const fd = new FormData(); if (audio) fd.append('audio', audio, audio.name); if (text) fd.append('text', text); fd.append('stt_backend', q('bench-turn-stt')?.value || 'configured'); fd.append('llm_url', q('bench-turn-llm-url')?.value.trim() || ''); fd.append('llm_model', q('bench-turn-llm-model')?.value || ''); fd.append('tts_backend', q('bench-turn-tts-backend')?.value || 'voice_clone'); fd.append('tts_voice', q('bench-turn-voice')?.value || ''); fd.append('system_prompt', q('bench-turn-system')?.value.trim() || 'You are a helpful voice assistant.'); fd.append('history', '[]'); const resp = await fetch('/api/conversation/turn', { method: 'POST', body: fd }); if (!resp.ok) throw new Error('Server error ' + resp.status); const reader = resp.body.getReader(); const dec = new TextDecoder(); let buf = ''; while (true) { const { done, value } = await reader.read(); if (done) break; buf += dec.decode(value, { stream: true }); const lines = buf.split('\n'); buf = lines.pop(); for (const line of lines) { if (!line.startsWith('data:')) continue; let evt; try { evt = JSON.parse(line.slice(5).trim()); } catch (_) { continue; } if (evt.type === 'transcript' && userBubble) userBubble.textContent = evt.text || '(empty)'; if (evt.type === 'token') { assistantText += evt.delta || ''; if (assistantBubble) assistantBubble.textContent = assistantText; } if (evt.type === 'llm_done') { assistantText = evt.text || assistantText; if (assistantBubble) assistantBubble.textContent = assistantText; } if (evt.type === 'audio' && evt.b64) { const bytes = Uint8Array.from(atob(evt.b64), c => c.charCodeAt(0)); const url = URL.createObjectURL(new Blob([bytes], { type: evt.mime || 'audio/wav' })); const audioEl = document.createElement('audio'); audioEl.controls = true; audioEl.src = url; audioEl.addEventListener('ended', () => URL.revokeObjectURL(url), { once: true }); q('bench-turn-log')?.appendChild(audioEl); } if (evt.type === 'stats') { lastStats = evt; updateTurnStats(evt); } if (evt.type === 'error') throw new Error(`[${evt.stage || 'turn'}] ${evt.message || 'Unknown error'}`); } } const total = lastStats?.total_ms ?? (Date.now() - t0); addTurnHistory(total, true); if (st) st.textContent = `Finished in ${fmtMs(total)}.`; say('Turn benchmark complete', 'success'); } catch (e) { if (assistantBubble) assistantBubble.textContent = e.message; addTurnHistory(Date.now() - t0, false); if (st) st.textContent = 'Turn benchmark failed'; say('Turn benchmark failed: ' + e.message, 'error'); } finally { if (btn) btn.disabled = false; } } function bindBenchmarkSection() { if (initialized || !q('bench-stt-run')) return; initialized = true; document.querySelectorAll('.bench-tab').forEach(btn => btn.addEventListener('click', () => setBenchTab(btn.dataset.benchTab))); if (window.BenchmarkVoicePicker) { BenchmarkVoicePicker.upgrade('bench-stt-library-voice', { placeholder: '-- choose from voice library --', empty: 'No library voices with reference transcripts found' }); BenchmarkVoicePicker.upgrade('perf-voice-select', { placeholder: '-- select after fetch --', empty: 'No voices' }); BenchmarkVoicePicker.upgrade('bench-turn-voice', { placeholder: 'Fetch voices', empty: 'No voices' }); } document.addEventListener('click', () => closeBenchmarkModelPickers()); q('bench-stt-refresh')?.addEventListener('click', loadBenchmarkSttEngines); q('bench-stt-load-voice')?.addEventListener('click', useBenchmarkLibraryVoice); q('bench-stt-audio')?.addEventListener('change', () => { if (q('bench-stt-source-id')) q('bench-stt-source-id').value = ''; }); q('bench-stt-run')?.addEventListener('click', runSttBenchmark); q('bench-tts-run')?.addEventListener('click', runTtsBenchmark); q('bench-turn-fetch-llm')?.addEventListener('click', fetchTurnModels); q('bench-turn-fetch-voices')?.addEventListener('click', fetchTurnVoices); q('bench-turn-run')?.addEventListener('click', runTurnBenchmark); q('bench-turn-tts-backend')?.addEventListener('change', () => { if (window.BenchmarkVoicePicker) BenchmarkVoicePicker.populate('bench-turn-voice', [], { placeholder: 'Fetch voices', empty: 'No voices' }); else if (q('bench-turn-voice')) q('bench-turn-voice').innerHTML = ''; }); loadBenchmarkSttEngines(); loadBenchmarkVoiceLibrary(); initTurnControls(); } bindBenchmarkSection(); if (!initialized && document.body) { const obs = new MutationObserver(() => bindBenchmarkSection()); obs.observe(document.body, { childList: true, subtree: true }); } })();