mARTin-B78
1b0b4bbc1f
New benchmark.py: word-error-rate accuracy + a run() that times each STT engine on a reference clip. Settings gains a Benchmark tab: pick a .wav and a matching .txt, run all STT engines, see Time + Accuracy per engine and a summary of the fastest and most accurate. Add presets for each model you want compared. Fix "Local engine selected but the model isn't loaded." on Test: a cached _transcriber_for() loads the local model on demand (the daemon only preloads it when a local engine is active). Tested: local small 2.17s/100% vs remote :8010. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
83 lines
2.5 KiB
Python
83 lines
2.5 KiB
Python
"""Benchmark STT engines against a reference clip.
|
||
|
||
Given a WAV and its reference transcript, transcribe with each engine, measure
|
||
the time, and score accuracy as 1 − WER (word error rate). Used by the Settings
|
||
Benchmark tab to find the fastest and most accurate engine/model.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
from . import stt
|
||
from .routing import normalize
|
||
|
||
|
||
@dataclass
|
||
class BenchRow:
|
||
engine: str
|
||
model: str
|
||
ok: bool
|
||
seconds: float
|
||
wer: float
|
||
accuracy: float # percent, max(0, 1-wer)*100
|
||
text: str
|
||
error: str = ""
|
||
|
||
|
||
def _edit_distance(a: list[str], b: list[str]) -> int:
|
||
m, n = len(a), len(b)
|
||
dp = list(range(n + 1))
|
||
for i in range(1, m + 1):
|
||
prev = dp[0]
|
||
dp[0] = i
|
||
for j in range(1, n + 1):
|
||
cur = dp[j]
|
||
dp[j] = min(dp[j] + 1, dp[j - 1] + 1, prev + (a[i - 1] != b[j - 1]))
|
||
prev = cur
|
||
return dp[n]
|
||
|
||
|
||
def wer(reference: str, hypothesis: str) -> float:
|
||
"""Word error rate (0 = perfect). Text is normalized (case/punct-insensitive)."""
|
||
ref = normalize(reference)
|
||
hyp = normalize(hypothesis)
|
||
if not ref:
|
||
return 0.0 if not hyp else 1.0
|
||
return _edit_distance(ref, hyp) / len(ref)
|
||
|
||
|
||
def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
||
get_local_transcriber=None, progress=None) -> list[BenchRow]:
|
||
"""Benchmark each engine; calls progress(row) as each finishes."""
|
||
rows: list[BenchRow] = []
|
||
for e in engines:
|
||
tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None
|
||
res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr)
|
||
w = wer(reference, res.text) if res.ok else 1.0
|
||
row = BenchRow(
|
||
engine=e.name,
|
||
model=e.model or ("local" if e.is_local else "(default)"),
|
||
ok=res.ok,
|
||
seconds=res.seconds,
|
||
wer=w,
|
||
accuracy=max(0.0, 1.0 - w) * 100.0,
|
||
text=res.text,
|
||
error=res.error,
|
||
)
|
||
rows.append(row)
|
||
if progress:
|
||
progress(row)
|
||
return rows
|
||
|
||
|
||
def best(rows: list[BenchRow]) -> tuple[BenchRow | None, BenchRow | None]:
|
||
"""Return (fastest, most_accurate) among successful rows."""
|
||
ok = [r for r in rows if r.ok]
|
||
if not ok:
|
||
return None, None
|
||
fastest = min(ok, key=lambda r: r.seconds)
|
||
most_accurate = max(ok, key=lambda r: (r.accuracy, -r.seconds))
|
||
return fastest, most_accurate
|