benchmark: add 'Best for' column, rename GPU→CUDA (v2.0.4)

- BenchRow gains `best_for` field: "Short clips" / "Short / medium" /
  "Long / batch" / "Streaming" — derived from engine type and model name
- Device now shows "CUDA" instead of "GPU" for clarity
- Benchmark table gains a "Best for" column between Device and Time(s)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
mARTin-B78 2026-06-09 17:28:55 +02:00
parent 0ff0f604c2
commit 4d59853da6
3 changed files with 30 additions and 7 deletions

View File

@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
(not in a container) so it can type into any application via xdotool. (not in a container) so it can type into any application via xdotool.
""" """
__version__ = "2.0.3" __version__ = "2.0.4"

View File

@ -19,7 +19,8 @@ from .routing import normalize
class BenchRow: class BenchRow:
engine: str engine: str
model: str model: str
device: str # "CPU" | "GPU" | "remote" device: str # "CPU" | "CUDA" | "remote"
best_for: str # "Short clips" | "Short / medium" | "Long / batch" | "Streaming"
ok: bool ok: bool
seconds: float seconds: float
wer: float wer: float
@ -60,10 +61,29 @@ def wer(reference: str, hypothesis: str, *, case_sensitive: bool = False) -> flo
def _engine_device(engine, transcriber) -> str: def _engine_device(engine, transcriber) -> str:
if engine.is_local: if engine.is_local:
return "GPU" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU" return "CUDA" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU"
return "remote" return "remote"
def _engine_best_for(engine) -> str:
if engine.type == "riva_realtime":
return "Streaming"
model = (engine.model or "").lower()
name = engine.name.lower()
if any(x in name for x in ("stream", "realtime", "real-time", "live")):
return "Streaming"
if engine.is_local:
if any(x in model for x in ("tiny", "base")):
return "Short clips"
if any(x in model for x in ("large",)):
return "Long / batch"
return "Short / medium"
# Remote endpoint
if any(x in name for x in ("large", "batch")):
return "Long / batch"
return "Short / medium"
def run(engines, wav_path: Path, reference: str, *, language: str = "", def run(engines, wav_path: Path, reference: str, *, language: str = "",
case_sensitive: bool = True, get_local_transcriber=None, progress=None) -> list[BenchRow]: case_sensitive: bool = True, get_local_transcriber=None, progress=None) -> list[BenchRow]:
"""Benchmark each engine; calls progress(row) as each finishes. """Benchmark each engine; calls progress(row) as each finishes.
@ -79,6 +99,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
engine=e.name, engine=e.name,
model=e.model or ("local" if e.is_local else "(default)"), model=e.model or ("local" if e.is_local else "(default)"),
device=_engine_device(e, tr), device=_engine_device(e, tr),
best_for=_engine_best_for(e),
ok=res.ok, ok=res.ok,
seconds=res.seconds, seconds=res.seconds,
wer=w, wer=w,

View File

@ -1857,12 +1857,13 @@ notebook.bt-nb tab:checked label {
run.set_halign(Gtk.Align.START) run.set_halign(Gtk.Align.START)
page.pack_start(run, False, False, 6) page.pack_start(run, False, False, 6)
self.bench_store = Gtk.ListStore(str, str, str, str, str, str) self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str)
tree = Gtk.TreeView(model=self.bench_store) tree = Gtk.TreeView(model=self.bench_store)
for title, i, expand in [("Engine", 0, False), ("Model", 1, False), ("Device", 2, False), for title, i, expand in [("Engine", 0, False), ("Model", 1, False), ("Device", 2, False),
("Time (s)", 3, False), ("Accuracy", 4, False), ("Output", 5, True)]: ("Best for", 3, False), ("Time (s)", 4, False),
("Accuracy", 5, False), ("Output", 6, True)]:
r = Gtk.CellRendererText() r = Gtk.CellRendererText()
if i == 5: if i == 6:
r.set_property("ellipsize", Pango.EllipsizeMode.END) r.set_property("ellipsize", Pango.EllipsizeMode.END)
col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True) col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True)
col.set_expand(expand) col.set_expand(expand)
@ -1951,7 +1952,8 @@ notebook.bt-nb tab:checked label {
def _bench_add_row(self, row) -> bool: def _bench_add_row(self, row) -> bool:
acc = f"{row.accuracy:.1f}%" if row.ok else "" acc = f"{row.accuracy:.1f}%" if row.ok else ""
out = row.text if row.ok else f"{row.error}" out = row.text if row.ok else f"{row.error}"
self.bench_store.append([row.engine, row.model, row.device, f"{row.seconds:.2f}", acc, out]) self.bench_store.append([row.engine, row.model, row.device, row.best_for,
f"{row.seconds:.2f}", acc, out])
return False return False
def _bench_done(self, rows) -> bool: def _bench_done(self, rows) -> bool: