benchmark: add 'Best for' column, rename GPU→CUDA (v2.0.4)
- BenchRow gains `best_for` field: "Short clips" / "Short / medium" / "Long / batch" / "Streaming" — derived from engine type and model name - Device now shows "CUDA" instead of "GPU" for clarity - Benchmark table gains a "Best for" column between Device and Time(s) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
0ff0f604c2
commit
4d59853da6
@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
|
|||||||
(not in a container) so it can type into any application via xdotool.
|
(not in a container) so it can type into any application via xdotool.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__version__ = "2.0.3"
|
__version__ = "2.0.4"
|
||||||
|
|||||||
@ -19,7 +19,8 @@ from .routing import normalize
|
|||||||
class BenchRow:
|
class BenchRow:
|
||||||
engine: str
|
engine: str
|
||||||
model: str
|
model: str
|
||||||
device: str # "CPU" | "GPU" | "remote"
|
device: str # "CPU" | "CUDA" | "remote"
|
||||||
|
best_for: str # "Short clips" | "Short / medium" | "Long / batch" | "Streaming"
|
||||||
ok: bool
|
ok: bool
|
||||||
seconds: float
|
seconds: float
|
||||||
wer: float
|
wer: float
|
||||||
@ -60,10 +61,29 @@ def wer(reference: str, hypothesis: str, *, case_sensitive: bool = False) -> flo
|
|||||||
|
|
||||||
def _engine_device(engine, transcriber) -> str:
|
def _engine_device(engine, transcriber) -> str:
|
||||||
if engine.is_local:
|
if engine.is_local:
|
||||||
return "GPU" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU"
|
return "CUDA" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU"
|
||||||
return "remote"
|
return "remote"
|
||||||
|
|
||||||
|
|
||||||
|
def _engine_best_for(engine) -> str:
|
||||||
|
if engine.type == "riva_realtime":
|
||||||
|
return "Streaming"
|
||||||
|
model = (engine.model or "").lower()
|
||||||
|
name = engine.name.lower()
|
||||||
|
if any(x in name for x in ("stream", "realtime", "real-time", "live")):
|
||||||
|
return "Streaming"
|
||||||
|
if engine.is_local:
|
||||||
|
if any(x in model for x in ("tiny", "base")):
|
||||||
|
return "Short clips"
|
||||||
|
if any(x in model for x in ("large",)):
|
||||||
|
return "Long / batch"
|
||||||
|
return "Short / medium"
|
||||||
|
# Remote endpoint
|
||||||
|
if any(x in name for x in ("large", "batch")):
|
||||||
|
return "Long / batch"
|
||||||
|
return "Short / medium"
|
||||||
|
|
||||||
|
|
||||||
def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
||||||
case_sensitive: bool = True, get_local_transcriber=None, progress=None) -> list[BenchRow]:
|
case_sensitive: bool = True, get_local_transcriber=None, progress=None) -> list[BenchRow]:
|
||||||
"""Benchmark each engine; calls progress(row) as each finishes.
|
"""Benchmark each engine; calls progress(row) as each finishes.
|
||||||
@ -79,6 +99,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
|||||||
engine=e.name,
|
engine=e.name,
|
||||||
model=e.model or ("local" if e.is_local else "(default)"),
|
model=e.model or ("local" if e.is_local else "(default)"),
|
||||||
device=_engine_device(e, tr),
|
device=_engine_device(e, tr),
|
||||||
|
best_for=_engine_best_for(e),
|
||||||
ok=res.ok,
|
ok=res.ok,
|
||||||
seconds=res.seconds,
|
seconds=res.seconds,
|
||||||
wer=w,
|
wer=w,
|
||||||
|
|||||||
@ -1857,12 +1857,13 @@ notebook.bt-nb tab:checked label {
|
|||||||
run.set_halign(Gtk.Align.START)
|
run.set_halign(Gtk.Align.START)
|
||||||
page.pack_start(run, False, False, 6)
|
page.pack_start(run, False, False, 6)
|
||||||
|
|
||||||
self.bench_store = Gtk.ListStore(str, str, str, str, str, str)
|
self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str)
|
||||||
tree = Gtk.TreeView(model=self.bench_store)
|
tree = Gtk.TreeView(model=self.bench_store)
|
||||||
for title, i, expand in [("Engine", 0, False), ("Model", 1, False), ("Device", 2, False),
|
for title, i, expand in [("Engine", 0, False), ("Model", 1, False), ("Device", 2, False),
|
||||||
("Time (s)", 3, False), ("Accuracy", 4, False), ("Output", 5, True)]:
|
("Best for", 3, False), ("Time (s)", 4, False),
|
||||||
|
("Accuracy", 5, False), ("Output", 6, True)]:
|
||||||
r = Gtk.CellRendererText()
|
r = Gtk.CellRendererText()
|
||||||
if i == 5:
|
if i == 6:
|
||||||
r.set_property("ellipsize", Pango.EllipsizeMode.END)
|
r.set_property("ellipsize", Pango.EllipsizeMode.END)
|
||||||
col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True)
|
col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True)
|
||||||
col.set_expand(expand)
|
col.set_expand(expand)
|
||||||
@ -1951,7 +1952,8 @@ notebook.bt-nb tab:checked label {
|
|||||||
def _bench_add_row(self, row) -> bool:
|
def _bench_add_row(self, row) -> bool:
|
||||||
acc = f"{row.accuracy:.1f}%" if row.ok else "—"
|
acc = f"{row.accuracy:.1f}%" if row.ok else "—"
|
||||||
out = row.text if row.ok else f"⚠ {row.error}"
|
out = row.text if row.ok else f"⚠ {row.error}"
|
||||||
self.bench_store.append([row.engine, row.model, row.device, f"{row.seconds:.2f}", acc, out])
|
self.bench_store.append([row.engine, row.model, row.device, row.best_for,
|
||||||
|
f"{row.seconds:.2f}", acc, out])
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _bench_done(self, rows) -> bool:
|
def _bench_done(self, rows) -> bool:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user