286 lines
19 KiB
HTML
286 lines
19 KiB
HTML
<div class="section-head">
|
|
<span class="section-icon"><span class="mdi mdi-speedometer"></span></span>
|
|
<div class="section-title">
|
|
<h2>Benchmark</h2>
|
|
<p>Measure speech recognition, voice synthesis, and full conversation turns with the same local engines used by the app.</p>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="bench-tabs" role="tablist" aria-label="Benchmark modes">
|
|
<button class="bench-tab" type="button" data-bench-tab="stt"><span class="mdi mdi-microphone-outline"></span> STT <span>Speech to Text</span></button>
|
|
<button class="bench-tab active" type="button" data-bench-tab="tts"><span class="mdi mdi-account-voice"></span> TTS <span>Text to Speech</span></button>
|
|
<button class="bench-tab" type="button" data-bench-tab="turn"><span class="mdi mdi-swap-horizontal"></span> Turn <span>STT → TTS</span></button>
|
|
</div>
|
|
|
|
<section class="bench-pane" id="bench-pane-stt" data-bench-pane="stt">
|
|
<div class="card bench-card">
|
|
<div class="bench-card-head">
|
|
<div>
|
|
<h3>Speech to Text benchmark</h3>
|
|
<p>Run every selected STT endpoint against the same reference clip and compare speed, accuracy, model, and compute device.</p>
|
|
</div>
|
|
<button class="btn-secondary" id="bench-stt-refresh" type="button"><span class="mdi mdi-refresh"></span> Refresh engines</button>
|
|
</div>
|
|
<div class="bench-library-source">
|
|
<div class="s-field">
|
|
<label>Use voice library sample</label>
|
|
<div class="bench-source-row">
|
|
<select id="bench-stt-library-voice"><option value="">Loading voices...</option></select>
|
|
<button class="btn-secondary" id="bench-stt-load-voice" type="button"><span class="mdi mdi-database-import-outline"></span> Use voice</button>
|
|
</div>
|
|
<div class="bench-source-note" id="bench-stt-source-note">Pick a library voice to use its WAV and reference transcript.</div>
|
|
</div>
|
|
</div>
|
|
<input id="bench-stt-source-id" type="hidden" value="">
|
|
<div class="settings-grid three bench-input-grid">
|
|
<div class="s-field">
|
|
<label>Audio (.wav or audio file)</label>
|
|
<input id="bench-stt-audio" type="file" accept="audio/*,.wav">
|
|
</div>
|
|
<div class="s-field">
|
|
<label>Reference (.txt)</label>
|
|
<input id="bench-stt-ref-file" type="file" accept="text/plain,.txt">
|
|
</div>
|
|
<div class="s-field">
|
|
<label>Reference text</label>
|
|
<textarea id="bench-stt-ref-text" rows="3" placeholder="Paste the expected transcript here..."></textarea>
|
|
</div>
|
|
</div>
|
|
<div class="bench-engine-panel">
|
|
<div class="bench-engine-head">
|
|
<strong>STT engines</strong>
|
|
<span id="bench-stt-engine-count">No engines loaded.</span>
|
|
</div>
|
|
<div class="bench-engine-list" id="bench-stt-engines"></div>
|
|
</div>
|
|
<div class="btn-row">
|
|
<button class="btn-primary" id="bench-stt-run" type="button"><span class="mdi mdi-play"></span> Run STT benchmark</button>
|
|
<span class="bench-status" id="bench-stt-status">Ready</span>
|
|
</div>
|
|
</div>
|
|
<div class="card bench-card">
|
|
<div class="bench-summary" id="bench-stt-summary"></div>
|
|
<div class="perf-table-wrap bench-table-wrap">
|
|
<table class="perf-table bench-table" id="bench-stt-table">
|
|
<thead><tr><th>Engine</th><th>Model</th><th>Device</th><th>Time</th><th>Accuracy</th><th>Output</th></tr></thead>
|
|
<tbody><tr><td colspan="6" class="bench-empty">Run a benchmark to see results.</td></tr></tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<section class="bench-pane active" id="bench-pane-tts" data-bench-pane="tts">
|
|
<!-- Run benchmark -->
|
|
<div class="card">
|
|
<h2>Run benchmark</h2>
|
|
<p class="card-subtitle">Pick a backend and voice, set run count, then measure synthesis latency and real-time factor (RTF). RTF < 1.0 means the backend generates faster than real-time.</p>
|
|
<div class="engine-setup-row" style="margin-bottom:12px">
|
|
<div class="engine-setup-col" style="flex:2">
|
|
<label class="engine-setup-label"><span class="mdi mdi-account-voice"></span> Text to Speech <a href="#" class="engine-setup-link" onclick="navTo('s-llms'); navEnginesCat('tts'); return false;" title="Configure Text to Speech settings"><span class="mdi mdi-cog-outline"></span></a></label>
|
|
<div class="engine-setup-controls">
|
|
<select id="perf-backend-select" aria-label="TTS backend"><option value="">Checking backends…</option></select>
|
|
<button class="btn-secondary" id="perf-fetch-voices-btn" type="button" title="Fetch voices"><span class="mdi mdi-refresh"></span></button>
|
|
<select id="perf-voice-select" aria-label="Voice"><option value="">— select after fetch —</option></select>
|
|
</div>
|
|
</div>
|
|
<div class="engine-setup-col no-border" style="flex:1">
|
|
<label class="engine-setup-label"><span class="mdi mdi-tune"></span> Benchmark Options</label>
|
|
<div class="engine-setup-controls">
|
|
<select id="perf-runs" aria-label="Runs">
|
|
<option value="1">1 run</option>
|
|
<option value="3" selected>3 runs</option>
|
|
<option value="5">5 runs</option>
|
|
<option value="10">10 runs</option>
|
|
</select>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="field">
|
|
<label>Sample text</label>
|
|
<textarea id="perf-text" rows="3">Hello, how are you today? Please read this sample clearly for a fair voice benchmark.</textarea>
|
|
</div>
|
|
<div class="btn-row">
|
|
<button class="btn-primary" id="perf-run-btn" type="button"><span class="mdi mdi-play"></span> Run benchmark</button>
|
|
<button class="btn-secondary" id="perf-run-selected-btn" type="button" disabled title="Benchmark every checked voice in the Batch benchmark list below" aria-describedby="batch-benchmark-help"><span class="mdi mdi-playlist-play"></span> Run selected voices</button>
|
|
<button class="btn-secondary" id="perf-clear-btn" type="button">Clear results</button>
|
|
</div>
|
|
<div id="perf-progress" class="note" style="display:none"></div>
|
|
</div>
|
|
|
|
<!-- Current results -->
|
|
<div class="card" id="perf-results-card" style="display:none">
|
|
<h2>Results <span id="perf-results-label" class="s-label-note"></span></h2>
|
|
<p class="card-subtitle">Latency per run. RTF = synthesis time ÷ audio duration (lower is better, <1.0 is real-time capable).</p>
|
|
<div class="perf-summary" id="perf-summary"></div>
|
|
<div class="perf-trend-row" id="perf-trend-row" style="display:none">
|
|
<span class="perf-trend-label">vs. previous session:</span>
|
|
<span id="perf-trend-badge" class="perf-trend-badge"></span>
|
|
<span class="perf-sparkline-wrap" title="RTF history for this backend/voice (oldest → newest)">
|
|
<svg id="perf-sparkline" class="perf-sparkline" viewBox="0 0 120 32"></svg>
|
|
</span>
|
|
</div>
|
|
<div class="perf-table-wrap">
|
|
<table class="perf-table" id="perf-table">
|
|
<thead>
|
|
<tr>
|
|
<th>#</th><th>Backend</th><th>Device</th><th>Voice</th>
|
|
<th>Latency (ms)</th><th>Audio (s)</th><th>RTF</th><th>Status</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="perf-tbody"></tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Batch benchmark -->
|
|
<div class="card">
|
|
<h2>Batch benchmark</h2>
|
|
<p class="card-subtitle">Benchmark multiple voices in one run. Pre-populated from your active My Voices — or reload from the backend. Results are sorted fastest first and saved to History.</p>
|
|
<div class="engine-setup-row" style="margin-bottom:12px">
|
|
<div class="engine-setup-col" style="flex:2">
|
|
<label class="engine-setup-label"><span class="mdi mdi-account-voice"></span> Text to Speech <a href="#" class="engine-setup-link" onclick="navTo('s-llms'); navEnginesCat('tts'); return false;" title="Configure Text to Speech settings"><span class="mdi mdi-cog-outline"></span></a></label>
|
|
<div class="engine-setup-controls">
|
|
<select id="batch-backend-select" aria-label="TTS backend"><option value="">Checking backends…</option></select>
|
|
</div>
|
|
</div>
|
|
<div class="engine-setup-col no-border" style="flex:1">
|
|
<label class="engine-setup-label"><span class="mdi mdi-tune"></span> Benchmark Options</label>
|
|
<div class="engine-setup-controls">
|
|
<select id="batch-runs" aria-label="Runs per voice">
|
|
<option value="1" selected>1 run</option>
|
|
<option value="3">3 runs</option>
|
|
<option value="5">5 runs</option>
|
|
<option value="10">10 runs</option>
|
|
</select>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<p class="note" id="batch-benchmark-help" style="margin:4px 0 8px">Uses the sample text and Runs value from the single-voice form above. Check one voice, a few voices, or Select all, then run them together.</p>
|
|
<div class="batch-voice-toolbar">
|
|
<button class="btn-secondary btn-sm" id="batch-load-voices-btn" type="button" title="Load all voices exposed by the selected backend"><span class="mdi mdi-reload"></span> Reload from backend</button>
|
|
<div class="batch-search-wrap"><span class="mdi mdi-magnify" aria-hidden="true"></span><input id="batch-voice-search" type="search" placeholder="Search voices..." autocomplete="off" spellcheck="false" aria-label="Search voices to benchmark"></div>
|
|
<button class="btn-secondary btn-sm" id="batch-select-all-btn" type="button" title="Select every loaded voice for the batch benchmark">Select all</button>
|
|
<button class="btn-secondary btn-sm" id="batch-select-none-btn" type="button" title="Clear the batch benchmark voice selection">Deselect all</button>
|
|
<span id="batch-selected-count" class="note" aria-live="polite"></span>
|
|
</div>
|
|
<div id="batch-voice-list" class="batch-voice-list" role="group" aria-label="Voices selected for batch benchmark" aria-describedby="batch-benchmark-help"></div>
|
|
<div class="btn-row" style="margin-top:12px">
|
|
<button class="btn-primary" id="batch-run-btn" type="button" disabled><span class="mdi mdi-play"></span> Run batch</button>
|
|
<button class="btn-secondary" id="batch-stop-btn" type="button" disabled><span class="mdi mdi-stop"></span> Stop</button>
|
|
</div>
|
|
<div id="batch-progress" class="batch-progress" style="display:none">
|
|
<div class="batch-progress-head">
|
|
<span id="batch-progress-label">Benchmarking…</span>
|
|
<span id="batch-progress-count"></span>
|
|
</div>
|
|
<div class="batch-progress-track" role="progressbar">
|
|
<div id="batch-progress-bar" class="batch-progress-bar"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Batch results -->
|
|
<div class="card" id="batch-results-card" style="display:none">
|
|
<h2>Batch results <span id="batch-results-label" class="s-label-note"></span></h2>
|
|
<p class="card-subtitle">Sorted fastest RTF first. Green = real-time capable (<1.0). Trend compares to the previous session for each voice.</p>
|
|
<div class="perf-table-wrap">
|
|
<table class="perf-table" id="batch-table">
|
|
<thead>
|
|
<tr>
|
|
<th>Voice</th><th>Device</th><th>Avg latency</th><th>Best</th><th>Audio (s)</th><th>Avg RTF</th><th>Status</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="batch-tbody"></tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- History -->
|
|
<div class="card" id="perf-history-card">
|
|
<h2>History</h2>
|
|
<p class="card-subtitle">Last 50 benchmark sessions saved in your browser. Each row is one run session — click the backend/voice to pre-fill the form above.</p>
|
|
<div class="bench-history-toolbar">
|
|
<label class="bench-history-filter">
|
|
<input type="checkbox" id="perf-history-filter-current">
|
|
Show current backend & voice only
|
|
</label>
|
|
<button class="btn-secondary btn-sm" id="perf-history-clear-btn" type="button">Clear history</button>
|
|
</div>
|
|
<div id="perf-history-list" class="perf-history-list">
|
|
<div class="perf-history-empty">No benchmark history yet. Run a benchmark above to start tracking.</div>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<section class="bench-pane" id="bench-pane-turn" data-bench-pane="turn">
|
|
<div class="card bench-card conv-config-bar">
|
|
<div class="bench-card-head">
|
|
<div>
|
|
<h3>Conversation turn benchmark</h3>
|
|
<p>Runs the same STT → LLM → TTS pipeline as Conversation Playground and records turn latency.</p>
|
|
</div>
|
|
</div>
|
|
<div class="engine-setup-row" style="margin-bottom:12px; margin-top:12px">
|
|
<div class="engine-setup-col" style="flex:1">
|
|
<label class="engine-setup-label"><span class="mdi mdi-microphone"></span> Speech to Text <a href="#" class="engine-setup-link" onclick="navTo('s-llms'); navEnginesCat('stt'); return false;" title="Configure Speech to Text settings"><span class="mdi mdi-cog-outline"></span></a></label>
|
|
<div class="engine-setup-controls">
|
|
<select id="bench-turn-stt" aria-label="STT backend"><option value="configured">Checking...</option></select>
|
|
</div>
|
|
</div>
|
|
<div class="engine-setup-col" style="flex:1.5">
|
|
<label class="engine-setup-label"><span class="mdi mdi-brain"></span> Language Model <a href="#" class="engine-setup-link" onclick="navTo('s-llms'); navEnginesCat('llm'); return false;" title="Configure Language Model settings"><span class="mdi mdi-cog-outline"></span></a></label>
|
|
<div class="engine-setup-controls">
|
|
<input id="bench-turn-llm-url" type="text" placeholder="http://localhost:11434/v1" spellcheck="false" aria-label="LLM URL">
|
|
<button class="btn-secondary" id="bench-turn-fetch-llm" type="button" title="Fetch models"><span class="mdi mdi-refresh"></span></button>
|
|
<select id="bench-turn-llm-model" aria-label="LLM Model"><option value="">Fetch models</option></select>
|
|
</div>
|
|
</div>
|
|
<div class="engine-setup-col no-border" style="flex:1.5">
|
|
<label class="engine-setup-label"><span class="mdi mdi-account-voice"></span> Text to Speech <a href="#" class="engine-setup-link" onclick="navTo('s-llms'); navEnginesCat('tts'); return false;" title="Configure Text to Speech settings"><span class="mdi mdi-cog-outline"></span></a></label>
|
|
<div class="engine-setup-controls">
|
|
<select id="bench-turn-tts-backend" aria-label="TTS backend"><option value="">Checking...</option></select>
|
|
<button class="btn-secondary" id="bench-turn-fetch-voices" type="button" title="Fetch voices"><span class="mdi mdi-refresh"></span></button>
|
|
<select id="bench-turn-voice" aria-label="Voice"><option value="">Fetch voices</option></select>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="settings-grid bench-input-grid">
|
|
<div class="s-field">
|
|
<label>Turn audio (full STT → LLM → TTS)</label>
|
|
<input id="bench-turn-audio" type="file" accept="audio/*,.wav,.webm,.ogg">
|
|
</div>
|
|
<div class="s-field">
|
|
<label>Text fallback</label>
|
|
<input id="bench-turn-text" type="text" placeholder="Hello, please answer in one short sentence.">
|
|
</div>
|
|
</div>
|
|
<div class="conv-prompt-row">
|
|
<label class="conv-cfg-label"><span class="mdi mdi-text-box-outline"></span> System prompt</label>
|
|
<textarea id="bench-turn-system" class="conv-system-textarea" rows="1" spellcheck="false">You are a helpful voice assistant. Keep replies short and conversational.</textarea>
|
|
</div>
|
|
<div class="btn-row">
|
|
<button class="btn-primary" id="bench-turn-run" type="button"><span class="mdi mdi-play"></span> Run turn benchmark</button>
|
|
<span class="bench-status" id="bench-turn-status">Ready</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="conv-main bench-turn-main">
|
|
<div class="conv-chat-panel bench-turn-log" id="bench-turn-log">
|
|
<div class="conv-chat-welcome"><span class="mdi mdi-forum-outline" style="font-size:32px;opacity:.25"></span><p>Run a turn to see transcript, reply, and timing.</p></div>
|
|
</div>
|
|
<div class="conv-stats-panel">
|
|
<div class="conv-stats-head">Latency</div>
|
|
<div class="conv-pipeline">
|
|
<div class="conv-pipe-step"><div class="conv-pipe-label"><span class="mdi mdi-microphone-outline"></span> STT</div><div class="conv-pipe-bar"><div class="conv-pipe-fill" id="bench-turn-fill-stt"></div></div><div class="conv-pipe-val" id="bench-turn-val-stt">—</div></div>
|
|
<div class="conv-pipe-step"><div class="conv-pipe-label"><span class="mdi mdi-timer-outline"></span> LLM first token</div><div class="conv-pipe-bar"><div class="conv-pipe-fill" id="bench-turn-fill-ttft"></div></div><div class="conv-pipe-val" id="bench-turn-val-ttft">—</div></div>
|
|
<div class="conv-pipe-step"><div class="conv-pipe-label"><span class="mdi mdi-brain"></span> LLM total</div><div class="conv-pipe-bar"><div class="conv-pipe-fill" id="bench-turn-fill-llm"></div></div><div class="conv-pipe-val" id="bench-turn-val-llm">—</div></div>
|
|
<div class="conv-pipe-step"><div class="conv-pipe-label"><span class="mdi mdi-account-voice"></span> TTS</div><div class="conv-pipe-bar"><div class="conv-pipe-fill" id="bench-turn-fill-tts"></div></div><div class="conv-pipe-val" id="bench-turn-val-tts">—</div></div>
|
|
<div class="conv-pipe-step conv-pipe-total"><div class="conv-pipe-label"><span class="mdi mdi-timer-check-outline"></span> Total</div><div class="conv-pipe-bar"><div class="conv-pipe-fill" id="bench-turn-fill-total" style="background:var(--accent)"></div></div><div class="conv-pipe-val" id="bench-turn-val-total">—</div></div>
|
|
</div>
|
|
<div class="conv-stats-head" style="margin-top:14px">Turn history</div>
|
|
<div class="conv-turn-history" id="bench-turn-history"><div class="conv-history-empty">No turns yet.</div></div>
|
|
</div>
|
|
</div>
|
|
</section>
|