From 4667144b7279a3d58157c92f90ff0418737eda0b Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Fri, 5 Jun 2026 14:12:16 +0200 Subject: [PATCH] Benchmark: add Device column (CPU/GPU/remote); case-sensitive accuracy - BenchRow gains a device field; Transcriber records its resolved device, so local engines report CPU or GPU and remote engines show "remote". New Device column in the results table. - Accuracy is now case-sensitive by default (capitalisation counts) so an all-lowercase transcript no longer scores 100%. Punctuation is still ignored. Co-Authored-By: Claude Opus 4.8 --- linux/blitztext/benchmark.py | 34 +++++++-- linux/blitztext/gtksettings.py | 132 ++++++++++++++++++++++++++++----- linux/blitztext/transcribe.py | 2 + 3 files changed, 143 insertions(+), 25 deletions(-) diff --git a/linux/blitztext/benchmark.py b/linux/blitztext/benchmark.py index 9a907ff..b003f36 100644 --- a/linux/blitztext/benchmark.py +++ b/linux/blitztext/benchmark.py @@ -7,6 +7,7 @@ Benchmark tab to find the fastest and most accurate engine/model. from __future__ import annotations +import re from dataclasses import dataclass from pathlib import Path @@ -18,6 +19,7 @@ from .routing import normalize class BenchRow: engine: str model: str + device: str # "CPU" | "GPU" | "remote" ok: bool seconds: float wer: float @@ -26,6 +28,14 @@ class BenchRow: error: str = "" +def _tokens(text: str, case_sensitive: bool) -> list[str]: + """Word tokens. case_sensitive keeps capitalisation (and accents); both + drop punctuation so only the words/spelling are compared.""" + if not case_sensitive: + return normalize(text) + return re.sub(r"[^\w\s]", " ", text, flags=re.UNICODE).split() + + def _edit_distance(a: list[str], b: list[str]) -> int: m, n = len(a), len(b) dp = list(range(n + 1)) @@ -39,26 +49,36 @@ def _edit_distance(a: list[str], b: list[str]) -> int: return dp[n] -def wer(reference: str, hypothesis: str) -> float: - """Word error rate (0 = perfect). Text is normalized (case/punct-insensitive).""" - ref = normalize(reference) - hyp = normalize(hypothesis) +def wer(reference: str, hypothesis: str, *, case_sensitive: bool = False) -> float: + """Word error rate (0 = perfect). case_sensitive makes capitalisation count.""" + ref = _tokens(reference, case_sensitive) + hyp = _tokens(hypothesis, case_sensitive) if not ref: return 0.0 if not hyp else 1.0 return _edit_distance(ref, hyp) / len(ref) +def _engine_device(engine, transcriber) -> str: + if engine.is_local: + return "GPU" if getattr(transcriber, "device", "cpu") == "cuda" else "CPU" + return "remote" + + def run(engines, wav_path: Path, reference: str, *, language: str = "", - get_local_transcriber=None, progress=None) -> list[BenchRow]: - """Benchmark each engine; calls progress(row) as each finishes.""" + case_sensitive: bool = True, get_local_transcriber=None, progress=None) -> list[BenchRow]: + """Benchmark each engine; calls progress(row) as each finishes. + + Accuracy is case-sensitive by default so wrong capitalisation counts. + """ rows: list[BenchRow] = [] for e in engines: tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr) - w = wer(reference, res.text) if res.ok else 1.0 + w = wer(reference, res.text, case_sensitive=case_sensitive) if res.ok else 1.0 row = BenchRow( engine=e.name, model=e.model or ("local" if e.is_local else "(default)"), + device=_engine_device(e, tr), ok=res.ok, seconds=res.seconds, wer=w, diff --git a/linux/blitztext/gtksettings.py b/linux/blitztext/gtksettings.py index d6e2aa2..5f6cd4c 100644 --- a/linux/blitztext/gtksettings.py +++ b/linux/blitztext/gtksettings.py @@ -5,6 +5,9 @@ Tabs: Engines - STT and LLM engine presets with online/offline status + STT test Input - input scheme, keys, quality gate General - mic (with live level meter), output, language, notifications, autostart + Benchmark- compare STT engines against a reference clip + Log - runtime log output + About - version, source, changelog, and license """ from __future__ import annotations @@ -21,7 +24,7 @@ gi.require_version("Gtk", "3.0") gi.require_version("Gdk", "3.0") from gi.repository import Gdk, GLib, Gtk, Pango # noqa: E402 -from . import audio, autostart, benchmark, llm, logbuffer, stt # noqa: E402 +from . import __version__, audio, autostart, benchmark, llm, logbuffer, stt # noqa: E402 from .config import Config, save # noqa: E402 from .llm import LLMEngine # noqa: E402 from .stt import STTEngine # noqa: E402 @@ -190,6 +193,44 @@ def _page(nb: Gtk.Notebook, title: str) -> Gtk.Box: return box +def _read_first(paths: list[Path], fallback: str = "Not available in this install.") -> str: + for path in paths: + try: + if path.exists(): + return path.read_text(encoding="utf-8", errors="replace").strip() + except OSError: + continue + return fallback + + +def _app_paths() -> dict[str, list[Path]]: + pkg_dir = Path(__file__).resolve().parent + linux_dir = pkg_dir.parent + repo_dir = linux_dir.parent + return { + "changelog": [linux_dir / "CHANGELOG.md", Path("/opt/blitztext/CHANGELOG.md")], + "license": [repo_dir / "LICENSE", Path("/usr/share/doc/blitztext/copyright")], + } + + +def _text_panel(text: str, *, monospace: bool = True, height: int = 180) -> Gtk.ScrolledWindow: + sw = Gtk.ScrolledWindow() + sw.set_min_content_height(height) + sw.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC) + view = Gtk.TextView() + view.set_editable(False) + view.set_cursor_visible(False) + view.set_wrap_mode(Gtk.WrapMode.WORD_CHAR) + view.set_left_margin(8) + view.set_right_margin(8) + view.set_top_margin(8) + view.set_bottom_margin(8) + view.set_monospace(monospace) + view.get_buffer().set_text(text) + sw.add(view) + return sw + + class SettingsDialog: def __init__(self, parent: Gtk.Window, cfg: Config, daemon=None): self.cfg = cfg @@ -212,6 +253,7 @@ class SettingsDialog: self._build_general(_page(nb, "General")) self._build_benchmark(_page(nb, "Benchmark")) self._build_log(_page(nb, "Log")) + self._build_about(_page(nb, "About")) self._bind_entry = None self._bind_pressed: list[str] = [] @@ -241,7 +283,7 @@ class SettingsDialog: self.wf_desc = _labeled(form, "Description", _entry(placeholder="Short description shown in the panel")) self.wf_keywords = _labeled(form, "Keywords (comma)", _entry(placeholder="nicer email, bessere email")) self.wf_hotkey = self._key_field(form, "Hotkey (optional)", "", placeholder="click Set, or e.g. ++e", width=130) - self.wf_mode = _labeled(form, "Mode", _combo(["transcribe", "rewrite"])) + self.wf_mode = _labeled(form, "Mode", _combo(["transcribe", "rewrite", "stream"])) self.wf_model = _labeled(form, "LLM model (opt.)", _entry(placeholder="blank = use the active LLM engine's model")) self.wf_temp = _labeled(form, "Temperature (opt.)", _entry(placeholder="blank = engine default (e.g. 0.3)")) @@ -262,7 +304,7 @@ class SettingsDialog: self.wf_desc.set_text(wf.description) self.wf_keywords.set_text(", ".join(wf.keywords)) self.wf_hotkey.set_text(wf.hotkey) - self.wf_mode.set_active(["transcribe", "rewrite"].index(wf.mode) if wf.mode in ("transcribe", "rewrite") else 0) + self.wf_mode.set_active(["transcribe", "rewrite", "stream"].index(wf.mode) if wf.mode in ("transcribe", "rewrite", "stream") else 0) self.wf_model.set_text(wf.model or "") self.wf_temp.set_text("" if wf.temperature is None else str(wf.temperature)) self.wf_prompt.get_buffer().set_text(wf.prompt) @@ -332,17 +374,18 @@ class SettingsDialog: self.stt_dot = Gtk.Label(); self.stt_dot.set_markup(_dot(GREY)) bar.pack_start(self.stt_dot, False, False, 0) bar.pack_start(self.stt_combo, True, True, 0) - for label, cb in (("+ Add", self._stt_add), ("Delete", self._stt_delete), - ("Test", self._stt_test), ("Refresh", lambda _b: self._refresh_status())): + for label, cb in (("+ Add", self._stt_add), ("+ Stream", self._stt_add_stream), + ("Delete", self._stt_delete), ("Test", self._stt_test), + ("Refresh", lambda _b: self._refresh_status())): b = Gtk.Button(label=label); b.connect("clicked", cb); bar.pack_start(b, False, False, 0) box.pack_start(bar, False, False, 2) form = Gtk.Box(orientation=Gtk.Orientation.VERTICAL); box.pack_start(form, False, False, 2) self.stt_name = _labeled(form, "Name", _entry(placeholder="e.g. faster-whisper GPU")) - self.stt_type = _labeled(form, "Type", _combo(["local", "openai"])) - self.stt_url = _url_field(form, "URL", "http://localhost:8010/v1 (blank for local)", + self.stt_type = _labeled(form, "Type", _combo(["local", "openai", "riva_realtime"])) + self.stt_url = _url_field(form, "URL", "http://localhost:8010/v1 · realtime: http://localhost:8006/v1", lambda: self._populate_models(self.stt_model, self.stt_url.get_text().strip(), self.stt_key.get_text().strip())) - self.stt_model = _labeled(form, "Model", _model_combo("pick after entering URL · tiny/base/small… for local")) + self.stt_model = _labeled(form, "Model", _model_combo("blank = server default · tiny/base/small… for local")) self.stt_key = _labeled(form, "API key env", _entry(placeholder="env var name, e.g. GROQ_API_KEY (optional)")) self.stt_url.connect("changed", lambda _e: self._schedule_models("stt")) self.stt_key.connect("changed", lambda _e: self._schedule_models("stt")) @@ -395,13 +438,15 @@ class SettingsDialog: return e = self.cfg.stt_engines[idx] self.stt_name.set_text(e.name) - self.stt_type.set_active(["local", "openai"].index(e.type) if e.type in ("local", "openai") else 0) + self.stt_type.set_active(["local", "openai", "riva_realtime"].index(e.type) if e.type in ("local", "openai", "riva_realtime") else 0) self.stt_url.set_text(e.url); self.stt_key.set_text(e.api_key_env) if e.type == "local": _fill_combo(self.stt_model, ["tiny", "base", "small", "medium", "large-v3"], e.model or self.cfg.model) - else: + elif e.type == "openai": _fill_combo(self.stt_model, [], e.model) self._populate_models(self.stt_model, e.url, e.api_key_env) + else: + _fill_combo(self.stt_model, [], e.model) self._stt_idx = idx def _stt_commit(self) -> None: @@ -431,6 +476,13 @@ class SettingsDialog: self.stt_combo.set_active(len(self.cfg.stt_engines) - 1) self._stt_load(len(self.cfg.stt_engines) - 1) + def _stt_add_stream(self, _b): + self._stt_commit() + e = STTEngine("Nemotron ASR Streaming", "riva_realtime", "http://127.0.0.1:8006/v1", "") + self.cfg.stt_engines.append(e); self.stt_combo.append_text(e.name) + self.stt_combo.set_active(len(self.cfg.stt_engines) - 1) + self._stt_load(len(self.cfg.stt_engines) - 1) + def _stt_delete(self, _b): if len(self.cfg.stt_engines) <= 1: return @@ -441,6 +493,9 @@ class SettingsDialog: def _stt_test(self, _b): self._stt_commit() e = self.cfg.stt_engines[self._stt_idx] + if e.is_streaming: + self.stt_result.set_markup("Streaming engines are live-only. Use a preset with mode = stream.") + return self.stt_result.set_markup("Recording 4s — speak now…") threading.Thread(target=self._run_stt_test, args=(e,), daemon=True).start() @@ -528,10 +583,13 @@ class SettingsDialog: self.llm_combo.set_active(0); self._llm_load(0) def _stt_type_changed(self, _c) -> None: - if self.stt_type.get_active_text() == "local": + typ = self.stt_type.get_active_text() + if typ == "local": _fill_combo(self.stt_model, ["tiny", "base", "small", "medium", "large-v3"], _combo_text(self.stt_model)) - else: + elif typ == "openai": self._schedule_models("stt") + else: + _fill_combo(self.stt_model, [], _combo_text(self.stt_model)) # -- model dropdowns (fetched from {url}/models) --- def _populate_models(self, combo, url: str, key_env: str) -> None: @@ -553,7 +611,7 @@ class SettingsDialog: def fire(): setattr(self, attr, 0) - if which == "stt" and self.stt_type.get_active_text() != "local": + if which == "stt" and self.stt_type.get_active_text() == "openai": self._populate_models(self.stt_model, self.stt_url.get_text().strip(), self.stt_key.get_text().strip()) elif which == "llm": @@ -685,12 +743,12 @@ class SettingsDialog: run.set_halign(Gtk.Align.START) page.pack_start(run, False, False, 6) - self.bench_store = Gtk.ListStore(str, str, str, str, str) + self.bench_store = Gtk.ListStore(str, str, str, str, str, str) tree = Gtk.TreeView(model=self.bench_store) - for title, i, expand in [("Engine", 0, False), ("Model", 1, False), - ("Time (s)", 2, False), ("Accuracy", 3, False), ("Output", 4, True)]: + for title, i, expand in [("Engine", 0, False), ("Model", 1, False), ("Device", 2, False), + ("Time (s)", 3, False), ("Accuracy", 4, False), ("Output", 5, True)]: r = Gtk.CellRendererText() - if i == 4: + if i == 5: r.set_property("ellipsize", Pango.EllipsizeMode.END) col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True) col.set_expand(expand) @@ -724,7 +782,7 @@ class SettingsDialog: def _bench_add_row(self, row) -> bool: acc = f"{row.accuracy:.1f}%" if row.ok else "—" out = row.text if row.ok else f"⚠ {row.error}" - self.bench_store.append([row.engine, row.model, f"{row.seconds:.2f}", acc, out]) + self.bench_store.append([row.engine, row.model, row.device, f"{row.seconds:.2f}", acc, out]) return False def _bench_done(self, rows) -> bool: @@ -737,6 +795,44 @@ class SettingsDialog: f" · Most accurate: {GLib.markup_escape_text(acc.engine)} ({acc.accuracy:.1f}%)") return False + # ===== About ============================================================ + def _build_about(self, page: Gtk.Box) -> None: + paths = _app_paths() + changelog = _read_first(paths["changelog"]) + license_text = _read_first(paths["license"]) + + title = Gtk.Label(label="Blitztext", xalign=0.0) + title.set_markup("Blitztext") + page.pack_start(title, False, False, 0) + + version = Gtk.Label(label=f"Version {__version__}", xalign=0.0) + version.set_selectable(True) + page.pack_start(version, False, False, 2) + + source = Gtk.LinkButton.new_with_label( + "https://github.com/mARTin-B78/blitztext-app", + "Source: github.com/mARTin-B78/blitztext-app", + ) + source.set_halign(Gtk.Align.START) + page.pack_start(source, False, False, 4) + + license_label = Gtk.Label(label="License: MIT", xalign=0.0) + license_label.set_selectable(True) + page.pack_start(license_label, False, False, 2) + + nb = Gtk.Notebook() + nb.set_margin_top(8) + page.pack_start(nb, True, True, 0) + + changelog_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL) + changelog_box.pack_start(_text_panel(changelog, height=300), True, True, 0) + nb.append_page(changelog_box, Gtk.Label(label="Changelog")) + + license_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL) + license_box.pack_start(_text_panel(license_text, height=300), True, True, 0) + nb.append_page(license_box, Gtk.Label(label="License")) + + # ===== Log ============================================================== def _build_log(self, page: Gtk.Box) -> None: sw = Gtk.ScrolledWindow(); sw.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC) diff --git a/linux/blitztext/transcribe.py b/linux/blitztext/transcribe.py index 915fc92..e0a9271 100644 --- a/linux/blitztext/transcribe.py +++ b/linux/blitztext/transcribe.py @@ -19,6 +19,7 @@ class Transcriber: beam_size: int = 5, ): self.beam_size = beam_size + self.device = "cpu" # resolved actual device, set by _load self._model = self._load(model, device, compute_type) @staticmethod @@ -45,6 +46,7 @@ class Transcriber: log(f"Loading Whisper '{model}' on {dev} ({ct})… (first run may download the model)") m = WhisperModel(model, device=dev, compute_type=ct) log(f"Whisper '{model}' ready on {dev} ({ct})") + self.device = dev return m except Exception as exc: # noqa: BLE001 - CUDA libs may be absent last_err = exc