From 1c16cfeb0f3106eea2ab780d70aac789ba133db9 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Tue, 9 Jun 2026 18:19:42 +0200 Subject: [PATCH] benchmark: remote GPU detection, URL column, test-all-models (v2.0.6) - stt.detect_remote_device(): probes /info (faster-whisper-server) then /metadata (NVIDIA NIM) to detect CUDA vs CPU; cached per unique URL - BenchRow gains url field; Device column now shows "CUDA" for GPU remotes instead of the generic "remote" - Benchmark table gains URL column (scheme stripped, max 180px wide) - "Test all models per engine" checkbox: fetches list_models() for each remote engine and expands to one row per model when checked - benchmark.run() gains expand_models parameter Co-Authored-By: Claude Sonnet 4.6 --- linux/blitztext/__init__.py | 2 +- linux/blitztext/benchmark.py | 32 ++++++++++++++++++++++++----- linux/blitztext/gtksettings.py | 37 ++++++++++++++++++++++++---------- linux/blitztext/stt.py | 32 +++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 17 deletions(-) diff --git a/linux/blitztext/__init__.py b/linux/blitztext/__init__.py index 1290067..944024a 100644 --- a/linux/blitztext/__init__.py +++ b/linux/blitztext/__init__.py @@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host (not in a container) so it can type into any application via xdotool. """ -__version__ = "2.0.5" +__version__ = "2.0.6" diff --git a/linux/blitztext/benchmark.py b/linux/blitztext/benchmark.py index 131c4ea..a02e47a 100644 --- a/linux/blitztext/benchmark.py +++ b/linux/blitztext/benchmark.py @@ -7,6 +7,7 @@ Benchmark tab to find the fastest and most accurate engine/model. from __future__ import annotations +import dataclasses import re from dataclasses import dataclass from pathlib import Path @@ -18,6 +19,7 @@ from .routing import normalize @dataclass class BenchRow: engine: str + url: str # base URL of the engine (empty for local) model: str device: str # "CPU" | "CUDA" | "remote" best_for: str # "Short clips" | "Short / medium" | "Long / batch" | "Streaming" @@ -59,10 +61,13 @@ def wer(reference: str, hypothesis: str, *, case_sensitive: bool = False) -> flo return _edit_distance(ref, hyp) / len(ref) -def _engine_device(engine, transcriber) -> str: +def _engine_device(engine, transcriber, _cache: dict) -> str: if engine.is_local: return "CUDA" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU" - return "remote" + url = engine.url + if url not in _cache: + _cache[url] = stt.detect_remote_device(url) + return _cache[url] def _engine_best_for(engine) -> str: @@ -85,20 +90,37 @@ def _engine_best_for(engine) -> str: def run(engines, wav_path: Path, reference: str, *, language: str = "", - case_sensitive: bool = True, get_local_transcriber=None, progress=None) -> list[BenchRow]: + case_sensitive: bool = True, get_local_transcriber=None, progress=None, + expand_models: bool = False) -> list[BenchRow]: """Benchmark each engine; calls progress(row) as each finishes. + expand_models=True fetches available models for each remote engine and + runs one benchmark row per model instead of just the configured one. Accuracy is case-sensitive by default so wrong capitalisation counts. """ - rows: list[BenchRow] = [] + # Build the run list, optionally expanding remote engines by their models + run_list: list = [] for e in engines: + if expand_models and not e.is_local and not e.is_streaming: + models = stt.list_models(e.url, e.api_key_env) + if len(models) > 1: + for m in models: + run_list.append(dataclasses.replace(e, model=m, + name=f"{e.name} [{m}]")) + continue + run_list.append(e) + + device_cache: dict = {} + rows: list[BenchRow] = [] + for e in run_list: tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr) w = wer(reference, res.text, case_sensitive=case_sensitive) if res.ok else 1.0 row = BenchRow( engine=e.name, + url=e.url, model=e.model or ("local" if e.is_local else "(default)"), - device=_engine_device(e, tr), + device=_engine_device(e, tr, device_cache), best_for=_engine_best_for(e), ok=res.ok, seconds=res.seconds, diff --git a/linux/blitztext/gtksettings.py b/linux/blitztext/gtksettings.py index 187a67e..ff6b037 100644 --- a/linux/blitztext/gtksettings.py +++ b/linux/blitztext/gtksettings.py @@ -1853,23 +1853,33 @@ notebook.bt-nb tab:checked label { break wavf.connect("file-set", _on_wav_set) + run_row = Gtk.Box(spacing=16) + run_row.set_margin_bottom(2) run = Gtk.Button(label="Run benchmark"); run.connect("clicked", self._run_bench) run.set_halign(Gtk.Align.START) - page.pack_start(run, False, False, 6) + run_row.pack_start(run, False, False, 0) + self.bench_expand = Gtk.CheckButton(label="Test all models per engine") + self.bench_expand.set_tooltip_text( + "Fetch every available model from each remote engine and run a separate " + "benchmark row per model — useful to compare models on the same server.") + run_row.pack_start(self.bench_expand, False, False, 0) + page.pack_start(run_row, False, False, 6) - # cols: engine, model, device, best_for, time, accuracy, output_friendly, tooltip_full - self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str, str) + # cols: engine, url, model, device, best_for, time, accuracy, output_friendly, tooltip_full + self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str, str, str) tree = Gtk.TreeView(model=self.bench_store) tree.set_has_tooltip(True) - tree.set_tooltip_column(7) # hover any row → full error text - for title, i, expand in [("Engine", 0, False), ("Model", 1, False), ("Device", 2, False), - ("Best for", 3, False), ("Time (s)", 4, False), - ("Accuracy", 5, False), ("Output", 6, True)]: + tree.set_tooltip_column(8) # hover any row → full error / output text + for title, i, expand in [("Engine", 0, False), ("URL", 1, False), + ("Model", 2, False), ("Device", 3, False), + ("Best for", 4, False), ("Time (s)", 5, False), + ("Accuracy", 6, False), ("Output", 7, True)]: r = Gtk.CellRendererText() - if i == 6: - r.set_property("ellipsize", Pango.EllipsizeMode.END) + r.set_property("ellipsize", Pango.EllipsizeMode.END) col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True) col.set_expand(expand) + if i == 1: + col.set_max_width(180) # keep URL column from dominating tree.append_column(col) sw = Gtk.ScrolledWindow(); sw.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC) sw.add(tree); page.pack_start(sw, True, True, 4) @@ -1959,11 +1969,14 @@ notebook.bt-nb tab:checked label { else: self.bench_summary.set_markup("Running… (local models load on first use)") + expand = self.bench_expand.get_active() + def work(): def prog(row): GLib.idle_add(self._bench_add_row, row) rows = benchmark.run(engines, Path(wav), reference, language=self.cfg.language, - get_local_transcriber=self._transcriber_for, progress=prog) + get_local_transcriber=self._transcriber_for, progress=prog, + expand_models=expand) GLib.idle_add(self._bench_done, rows) threading.Thread(target=work, daemon=True).start() @@ -1995,7 +2008,9 @@ notebook.bt-nb tab:checked label { else: out_friendly = f"⚠ {self._bench_friendly_error(row.error)}" tooltip = row.error - self.bench_store.append([row.engine, row.model, row.device, row.best_for, + # Strip scheme from URL for display brevity (http://192.168.1.1:8080 → 192.168.1.1:8080) + url_display = row.url.removeprefix("https://").removeprefix("http://").rstrip("/") + self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for, f"{row.seconds:.2f}", acc, out_friendly, tooltip]) return False diff --git a/linux/blitztext/stt.py b/linux/blitztext/stt.py index 0f36fa7..00cd3ff 100644 --- a/linux/blitztext/stt.py +++ b/linux/blitztext/stt.py @@ -105,6 +105,38 @@ def list_models(base_url: str, api_key_env: str = "", timeout: float = 5.0) -> l return [] +def detect_remote_device(base_url: str, timeout: float = 3.0) -> str: + """Best-effort GPU/CPU detection for a remote STT server. + + Tries faster-whisper-server's /info endpoint (returns {"device":"cuda",...}), + then NVIDIA NIM /metadata (GPU-only service). Falls back to "remote". + """ + if not base_url: + return "remote" + base = base_url.rstrip("/") + try: + req = urllib.request.Request(base + "/info") + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + if isinstance(data, dict): + dev = str(data.get("device") or data.get("compute_type") or "") + if "cuda" in dev.lower(): + return "CUDA" + if dev: + return dev.upper()[:16] + except Exception: + pass + try: + req = urllib.request.Request(base + "/metadata") + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + if isinstance(data, dict) and data.get("modelInfo"): + return "CUDA" # NVIDIA NIM is always GPU + except Exception: + pass + return "remote" + + def _host_port(url: str) -> tuple[str | None, int]: try: u = urlparse(url if "://" in url else "http://" + url)