benchmark: add 'Best for' column, rename GPU→CUDA (v2.0.4)
- BenchRow gains `best_for` field: "Short clips" / "Short / medium" / "Long / batch" / "Streaming" — derived from engine type and model name - Device now shows "CUDA" instead of "GPU" for clarity - Benchmark table gains a "Best for" column between Device and Time(s) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
0ff0f604c2
commit
4d59853da6
@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
|
||||
(not in a container) so it can type into any application via xdotool.
|
||||
"""
|
||||
|
||||
__version__ = "2.0.3"
|
||||
__version__ = "2.0.4"
|
||||
|
||||
@ -19,7 +19,8 @@ from .routing import normalize
|
||||
class BenchRow:
|
||||
engine: str
|
||||
model: str
|
||||
device: str # "CPU" | "GPU" | "remote"
|
||||
device: str # "CPU" | "CUDA" | "remote"
|
||||
best_for: str # "Short clips" | "Short / medium" | "Long / batch" | "Streaming"
|
||||
ok: bool
|
||||
seconds: float
|
||||
wer: float
|
||||
@ -60,10 +61,29 @@ def wer(reference: str, hypothesis: str, *, case_sensitive: bool = False) -> flo
|
||||
|
||||
def _engine_device(engine, transcriber) -> str:
|
||||
if engine.is_local:
|
||||
return "GPU" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU"
|
||||
return "CUDA" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU"
|
||||
return "remote"
|
||||
|
||||
|
||||
def _engine_best_for(engine) -> str:
|
||||
if engine.type == "riva_realtime":
|
||||
return "Streaming"
|
||||
model = (engine.model or "").lower()
|
||||
name = engine.name.lower()
|
||||
if any(x in name for x in ("stream", "realtime", "real-time", "live")):
|
||||
return "Streaming"
|
||||
if engine.is_local:
|
||||
if any(x in model for x in ("tiny", "base")):
|
||||
return "Short clips"
|
||||
if any(x in model for x in ("large",)):
|
||||
return "Long / batch"
|
||||
return "Short / medium"
|
||||
# Remote endpoint
|
||||
if any(x in name for x in ("large", "batch")):
|
||||
return "Long / batch"
|
||||
return "Short / medium"
|
||||
|
||||
|
||||
def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
||||
case_sensitive: bool = True, get_local_transcriber=None, progress=None) -> list[BenchRow]:
|
||||
"""Benchmark each engine; calls progress(row) as each finishes.
|
||||
@ -79,6 +99,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
||||
engine=e.name,
|
||||
model=e.model or ("local" if e.is_local else "(default)"),
|
||||
device=_engine_device(e, tr),
|
||||
best_for=_engine_best_for(e),
|
||||
ok=res.ok,
|
||||
seconds=res.seconds,
|
||||
wer=w,
|
||||
|
||||
@ -1857,12 +1857,13 @@ notebook.bt-nb tab:checked label {
|
||||
run.set_halign(Gtk.Align.START)
|
||||
page.pack_start(run, False, False, 6)
|
||||
|
||||
self.bench_store = Gtk.ListStore(str, str, str, str, str, str)
|
||||
self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str)
|
||||
tree = Gtk.TreeView(model=self.bench_store)
|
||||
for title, i, expand in [("Engine", 0, False), ("Model", 1, False), ("Device", 2, False),
|
||||
("Time (s)", 3, False), ("Accuracy", 4, False), ("Output", 5, True)]:
|
||||
("Best for", 3, False), ("Time (s)", 4, False),
|
||||
("Accuracy", 5, False), ("Output", 6, True)]:
|
||||
r = Gtk.CellRendererText()
|
||||
if i == 5:
|
||||
if i == 6:
|
||||
r.set_property("ellipsize", Pango.EllipsizeMode.END)
|
||||
col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True)
|
||||
col.set_expand(expand)
|
||||
@ -1951,7 +1952,8 @@ notebook.bt-nb tab:checked label {
|
||||
def _bench_add_row(self, row) -> bool:
|
||||
acc = f"{row.accuracy:.1f}%" if row.ok else "—"
|
||||
out = row.text if row.ok else f"⚠ {row.error}"
|
||||
self.bench_store.append([row.engine, row.model, row.device, f"{row.seconds:.2f}", acc, out])
|
||||
self.bench_store.append([row.engine, row.model, row.device, row.best_for,
|
||||
f"{row.seconds:.2f}", acc, out])
|
||||
return False
|
||||
|
||||
def _bench_done(self, rows) -> bool:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user