From d9603c98bf0c6c8370b961c83c37e6dee31151c1 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Tue, 9 Jun 2026 22:11:38 +0200 Subject: [PATCH] feat: RAM usage column in benchmark, docs update (v2.03.01) - benchmark.py: measure RSS delta via /proc/self/status before/after each transcription; add ram_mb field to BenchRow - gtksettings.py: add RAM (MB) column to results table (index 8); tooltip column shifted to index 10 - CHANGELOG.md: full history from v2.02.00 through v2.03.01 - README.md: benchmark description updated to mention RAM column - MANUAL.md: benchmark result columns as table; Log tab documents level filter dropdown Co-Authored-By: Claude Sonnet 4.6 --- MANUAL.md | 27 +++++++++++--- linux/CHANGELOG.md | 65 ++++++++++++++++++++++++++++++++++ linux/README.md | 2 +- linux/blitztext/__init__.py | 2 +- linux/blitztext/benchmark.py | 17 +++++++++ linux/blitztext/gtksettings.py | 12 ++++--- 6 files changed, 114 insertions(+), 11 deletions(-) diff --git a/MANUAL.md b/MANUAL.md index 38dbe09..98a5eb9 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -194,8 +194,22 @@ settings — it's a one-off tool. a matching `*.txt` / `*.reference.txt` sits next to the WAV.) 3. **Run benchmark** — fills the table with one row per engine. -Result columns: **Engine · Model · Device · Time (s) · Accuracy · Output**. A -summary line names the **fastest** and **most accurate** engine. +Result columns: + +| Column | Description | +|---|---| +| **Engine** | Engine preset name | +| **URL** | Server address (blank for local) | +| **Model** | Model name used | +| **Device** | `CPU`, `CUDA`, or `remote` | +| **Best for** | `Short clips` · `Short / medium` · `Long / batch` · `Streaming` | +| **Lang** | Supported languages from the server's `/v1/models` (`—` if unknown) | +| **Time (s)** | Wall-clock seconds for this transcription | +| **Accuracy** | `1 − WER` × 100 %. 100 % = word-perfect, case-sensitive | +| **RAM (MB)** | RSS increase while the engine ran. Captures model load cost on first run. Remote engines show `—`. | +| **Output** | Transcribed text (hover for full error on failure) | + +A summary line names the **fastest** and **most accurate** engine. Click any column header to sort. --- @@ -203,8 +217,13 @@ summary line names the **fastest** and **most accurate** engine. A live activity log — useful to watch a model load/download or to diagnose a problem (recording, transcription, routing, and wakeword events all appear here). -Press **Copy** to put the log on the clipboard when reporting an issue. No -settings. + +| Control | Description | +|---|---| +| **Level** dropdown | Filter by severity: **Verbose** (all), **Info** (default), **Warning**, **Error**. Switch to Warning or Error to cut noise when troubleshooting. | +| **Copy** | Put the log on the clipboard when reporting an issue. | +| **Clear** | Discard all current log entries. | +| **Auto-scroll** | Keep the view scrolled to the latest entry. | --- diff --git a/linux/CHANGELOG.md b/linux/CHANGELOG.md index 078c2b6..987c9b0 100644 --- a/linux/CHANGELOG.md +++ b/linux/CHANGELOG.md @@ -9,6 +9,71 @@ The version is defined in [`blitztext/__init__.py`](blitztext/__init__.py). ## [Unreleased] +## [2.03.01] - 2026-06-09 + +### Added +- **RAM usage column in benchmark.** The results table now shows a **RAM (MB)** + column — the increase in process RSS while the transcription ran. For local + models this captures the memory cost of loading the model on first use; for + remote engines it shows `—` (work happens server-side). Values are measured via + `/proc/self/status` (VmRSS), so they reflect actual resident memory, not + virtual address space. + +## [2.03.00] - 2026-06-09 + +### Fixed +- **"Not responding" / system instability on Save.** `_collect()` was calling + `socket.create_connection()` *synchronously* on the GTK main thread when + wakeword is enabled — freezing the UI for up to 1.5 s (longer if DNS is slow). + The check is now done on a daemon thread and the result is logged instead of + blocking the save path. +- **GTK thread-safety crash in wakeword model load.** `_ww_load()` read + `self.ww_uri.get_text()` from inside a background thread — unsafe. The URI is + now captured on the main thread before the thread is spawned. +- **HTTP 404 with WhisperX and other non-standard endpoints.** The remote + transcription call always appended `/audio/transcriptions` to the base URL, but + services like WhisperX use `/transcribe` as the full path. The URL path is now + inspected: if it is anything other than empty / `/v1` / `/v1.0`, the URL is + used as the complete endpoint with nothing appended — so + `http://host:8081/transcribe` works out of the box. +- **Log levels.** `logbuffer` now stores `(timestamp, level, message)` tuples and + accepts a `level=` keyword (`DEBUG` / `INFO` / `WARNING` / `ERROR`). The Log + tab gains a **Level** dropdown (Verbose · Info · Warning · Error) that filters + the displayed entries live. Wakeword and socket errors are now tagged + `WARNING`; library records are forwarded at their native level. + +### Added +- **Wakeword server preset dropdown** (Input → Hands-free wakeword). A + **Server preset** combo lists all configured wakeword server engines by name. + Picking one auto-fills the URI and model fields and re-probes reachability. + The selection is persisted as `wakeword_active` in config. + +## [2.02.03] - 2026-06-09 + +### Added +- Wakeword server preset dropdown in Input tab. + +## [2.02.02] - 2026-06-09 + +### Fixed +- License tab now renders with markdown styling. +- Benchmark pane minimum height (320 px, `shrink=False`) prevents the engine + list or results table from collapsing to zero when the window is small. + +## [2.02.01] - 2026-06-09 + +### Added +- Last benchmark time and accuracy shown on the selected STT engine in the + Engines tab. Persisted to config so it survives restarts. + +## [2.02.00] - 2026-06-09 + +### Added +- **Language metadata in benchmark.** The engine checkbox list shows supported + language codes next to each engine (fetched async). Filter box searches by + language code. Results table has a **Lang** column. Data comes from the + `/v1/models` `language` field (faster-whisper-server) or NVIDIA NIM `/metadata`. + ## [1.9.5] - 2026-06-09 ### Added diff --git a/linux/README.md b/linux/README.md index aabf7cf..5382811 100644 --- a/linux/README.md +++ b/linux/README.md @@ -91,7 +91,7 @@ screen-reader (ATK) support. Click any image to open it full size.

Benchmark settings tab
- Benchmark — compare every configured STT engine against a reference WAV + transcript to find the fastest and most accurate, with a Device column (CPU / GPU / remote). Also includes a wakeword benchmark: point it at any OpenAI-compatible TTS server (Kokoro, XTTS, …), and it synthesizes your wake phrase in random voices, streams it to your wyoming-openwakeword server, and reports recall + false-fires per voice. + Benchmark — compare every configured STT engine against a reference WAV + transcript. The results table shows time, accuracy (WER), device (CPU / GPU / remote), language support, and RAM usage (MB loaded during first run — useful for sizing local models). Also includes a wakeword benchmark: point it at any OpenAI-compatible TTS server (Kokoro, XTTS, …), and it synthesizes your wake phrase in random voices, streams it to your wyoming-openwakeword server, and reports recall + false-fires per voice.

diff --git a/linux/blitztext/__init__.py b/linux/blitztext/__init__.py index 78cc2aa..bb0dc0a 100644 --- a/linux/blitztext/__init__.py +++ b/linux/blitztext/__init__.py @@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host (not in a container) so it can type into any application via xdotool. """ -__version__ = "2.03.00" +__version__ = "2.03.01" diff --git a/linux/blitztext/benchmark.py b/linux/blitztext/benchmark.py index f283080..d9fc36d 100644 --- a/linux/blitztext/benchmark.py +++ b/linux/blitztext/benchmark.py @@ -16,6 +16,18 @@ from . import stt from .routing import normalize +def _rss_mb() -> float: + """Current process RSS in MB via /proc/self/status (Linux only).""" + try: + with open("/proc/self/status") as fh: + for line in fh: + if line.startswith("VmRSS:"): + return int(line.split()[1]) / 1024.0 # kB → MB + except OSError: + pass + return 0.0 + + @dataclass class BenchRow: engine: str @@ -29,6 +41,7 @@ class BenchRow: wer: float accuracy: float # percent, max(0, 1-wer)*100 text: str + ram_mb: float = 0.0 # RSS delta in MB; >0 means model loaded during this run error: str = "" @@ -128,7 +141,10 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "", rows: list[BenchRow] = [] for e in run_list: tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None + rss_before = _rss_mb() res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr) + rss_after = _rss_mb() + ram_delta = max(0.0, rss_after - rss_before) w = wer(reference, res.text, case_sensitive=case_sensitive) if res.ok else 1.0 row = BenchRow( engine=e.name, @@ -142,6 +158,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "", wer=w, accuracy=max(0.0, 1.0 - w) * 100.0, text=res.text, + ram_mb=ram_delta, error=res.error, ) rows.append(row) diff --git a/linux/blitztext/gtksettings.py b/linux/blitztext/gtksettings.py index 40d9a2e..60feef1 100644 --- a/linux/blitztext/gtksettings.py +++ b/linux/blitztext/gtksettings.py @@ -2044,12 +2044,12 @@ notebook.bt-nb tab:checked label { page.pack_start(run_row, False, False, 0) # ── Resizable pane: engine list (top) ↕ results table (bottom) ─────── - # engine, url, model, device, best_for, lang, time, accuracy, output, tooltip - self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str, str, str, str) + # engine, url, model, device, best_for, lang, time, accuracy, ram, output, tooltip + self.bench_store = Gtk.ListStore(str, str, str, str, str, str, str, str, str, str, str) bench_sort = Gtk.TreeModelSort(model=self.bench_store) tree = Gtk.TreeView(model=bench_sort) tree.set_has_tooltip(True) - tree.set_tooltip_column(9) + tree.set_tooltip_column(10) for title, i, expand, max_w in [ ("Engine", 0, False, 0), ("URL", 1, False, 180), @@ -2059,7 +2059,8 @@ notebook.bt-nb tab:checked label { ("Lang", 5, False, 160), ("Time (s)", 6, False, 0), ("Accuracy", 7, False, 0), - ("Output", 8, True, 0)]: + ("RAM (MB)", 8, False, 0), + ("Output", 9, True, 0)]: r = Gtk.CellRendererText() r.set_property("ellipsize", Pango.EllipsizeMode.END) col = Gtk.TreeViewColumn(title, r, text=i); col.set_resizable(True) @@ -2219,8 +2220,9 @@ notebook.bt-nb tab:checked label { # Strip scheme from URL for display brevity (http://192.168.1.1:8080 → 192.168.1.1:8080) url_display = row.url.removeprefix("https://").removeprefix("http://").rstrip("/") lang_display = stt.fmt_languages(row.languages) + ram_display = f"{row.ram_mb:.0f}" if row.ram_mb >= 1.0 else "—" self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for, - lang_display, f"{row.seconds:.2f}", acc, out_friendly, tooltip]) + lang_display, f"{row.seconds:.2f}", acc, ram_display, out_friendly, tooltip]) # Persist result so Engines tab can show it self.cfg.bench_last[row.engine] = { "seconds": row.seconds, "accuracy": row.accuracy, "ok": row.ok,