feat: server RAM via Prometheus /metrics in benchmark (v2.03.09)

Probe remote engines' /metrics for process_resident_memory_bytes or
container_memory_rss; show actual server-side MB in RAM column.
Falls back to "server" when the endpoint is not exposed.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
mARTin-B78 2026-06-09 23:24:40 +02:00
parent 86632e1918
commit c3cf123afe
5 changed files with 70 additions and 6 deletions

View File

@ -9,6 +9,15 @@ The version is defined in [`blitztext/__init__.py`](blitztext/__init__.py).
## [Unreleased]
## [2.03.09] - 2026-06-09
### Added
- **Server RAM in benchmark.** For remote/Docker STT engines the benchmark now
probes the server's Prometheus `/metrics` endpoint for
`process_resident_memory_bytes` (standard Python/Go exporter) or
`container_memory_rss` (cAdvisor) and shows the server-side RSS in MB in the
RAM column. Falls back to `server` when the endpoint is not exposed.
## [2.03.08] - 2026-06-09
### Fixed

View File

@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
(not in a container) so it can type into any application via xdotool.
"""
__version__ = "2.03.08"
__version__ = "2.03.09"

View File

@ -41,7 +41,8 @@ class BenchRow:
wer: float
accuracy: float # percent, max(0, 1-wer)*100
text: str
ram_mb: float = 0.0 # RSS delta in MB; >0 means model loaded during this run
ram_mb: float = 0.0 # local RSS delta in MB
srv_ram_mb: float | None = None # server /metrics RSS in MB, None = not available
error: str = ""
@ -126,6 +127,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
device_cache: dict = {}
meta_cache: dict = {} # url → list[ModelMeta]
server_ram_cache: dict = {} # url → float | None (MB from /metrics)
def _get_langs(e) -> list[str]:
if e.is_local:
@ -141,10 +143,21 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
rows: list[BenchRow] = []
for e in run_list:
tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None
# Snapshot server RAM before (if Prometheus metrics available)
if not e.is_local and e.url not in server_ram_cache:
server_ram_cache[e.url] = None # sentinel — probe once per URL
srv_before: float | None = None
if not e.is_local:
srv_before = stt.probe_server_ram_mb(e.url)
rss_before = _rss_mb()
res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr)
rss_after = _rss_mb()
ram_delta = max(0.0, rss_after - rss_before)
srv_ram: float | None = None
if not e.is_local:
srv_after = stt.probe_server_ram_mb(e.url)
if srv_before is not None and srv_after is not None:
srv_ram = srv_after # report current RSS, not delta (server may not unload)
w = wer(reference, res.text, case_sensitive=case_sensitive) if res.ok else 1.0
row = BenchRow(
engine=e.name,
@ -159,6 +172,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
accuracy=max(0.0, 1.0 - w) * 100.0,
text=res.text,
ram_mb=ram_delta,
srv_ram_mb=srv_ram,
error=res.error,
)
rows.append(row)

View File

@ -2104,8 +2104,10 @@ notebook.bt-nb tab:checked label {
col.set_max_width(max_w)
if i == 8:
col.set_widget(Gtk.Label(label="RAM (MB)", tooltip_text=
"RAM loaded by this engine during the benchmark run (local engines only).\n"
"'server' = model runs in a container or remote server — memory is not visible here.\n"
"RAM used by the engine during the benchmark.\n"
"Local engines: RSS delta measured on this machine.\n"
"Remote engines: current RSS read from the server's Prometheus /metrics endpoint.\n"
"'server' = server does not expose /metrics — RAM not measurable.\n"
"'' = local model was already loaded, no delta to measure."))
col.get_widget().show()
tree.append_column(col)
@ -2265,9 +2267,12 @@ notebook.bt-nb tab:checked label {
if row.ram_mb >= 1.0:
ram_display = f"{row.ram_mb:.0f}"
elif row.url:
ram_display = "server" # model runs remotely — RAM not measurable here
if row.srv_ram_mb is not None:
ram_display = f"{row.srv_ram_mb:.0f}" # from server /metrics
else:
ram_display = "server" # metrics not exposed
else:
ram_display = "" # local model already loaded, no delta
ram_display = "" # local model already loaded, no delta
self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for,
lang_display, f"{row.seconds:.2f}", acc, ram_display, out_friendly, tooltip])
# Persist result so Engines tab can show it

View File

@ -148,6 +148,42 @@ def list_models_meta(base_url: str, api_key_env: str = "", timeout: float = 5.0)
return []
def probe_server_ram_mb(base_url: str, timeout: float = 3.0) -> float | None:
"""Probe a remote server's Prometheus /metrics endpoint for RSS memory.
Returns RSS in MB if the server exposes Prometheus metrics with either
``process_resident_memory_bytes`` (standard Python/Go exporter) or
``container_memory_rss`` (cAdvisor). Returns None if not available.
"""
if not base_url:
return None
base = _api_base(base_url)
try:
req = urllib.request.Request(base + "/metrics")
with urllib.request.urlopen(req, timeout=timeout) as resp:
text = resp.read().decode("utf-8", "replace")
except Exception:
return None
# Parse Prometheus text format — we only need two specific metric names.
# Lines look like: process_resident_memory_bytes 1.23456e+08
best: float | None = None
for line in text.splitlines():
line = line.strip()
if line.startswith("#"):
continue
for metric in ("process_resident_memory_bytes", "container_memory_rss"):
if line.startswith(metric):
parts = line.split()
if len(parts) >= 2:
try:
best = float(parts[-1]) / (1024 * 1024) # bytes → MB
return best
except ValueError:
pass
return best
def detect_remote_device(base_url: str, timeout: float = 3.0) -> str:
"""Best-effort GPU/CPU detection for a remote STT server.