feat: server RAM via Prometheus /metrics in benchmark (v2.03.09)
Probe remote engines' /metrics for process_resident_memory_bytes or container_memory_rss; show actual server-side MB in RAM column. Falls back to "server" when the endpoint is not exposed. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
86632e1918
commit
c3cf123afe
@ -9,6 +9,15 @@ The version is defined in [`blitztext/__init__.py`](blitztext/__init__.py).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.03.09] - 2026-06-09
|
||||
|
||||
### Added
|
||||
- **Server RAM in benchmark.** For remote/Docker STT engines the benchmark now
|
||||
probes the server's Prometheus `/metrics` endpoint for
|
||||
`process_resident_memory_bytes` (standard Python/Go exporter) or
|
||||
`container_memory_rss` (cAdvisor) and shows the server-side RSS in MB in the
|
||||
RAM column. Falls back to `server` when the endpoint is not exposed.
|
||||
|
||||
## [2.03.08] - 2026-06-09
|
||||
|
||||
### Fixed
|
||||
|
||||
@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
|
||||
(not in a container) so it can type into any application via xdotool.
|
||||
"""
|
||||
|
||||
__version__ = "2.03.08"
|
||||
__version__ = "2.03.09"
|
||||
|
||||
@ -41,7 +41,8 @@ class BenchRow:
|
||||
wer: float
|
||||
accuracy: float # percent, max(0, 1-wer)*100
|
||||
text: str
|
||||
ram_mb: float = 0.0 # RSS delta in MB; >0 means model loaded during this run
|
||||
ram_mb: float = 0.0 # local RSS delta in MB
|
||||
srv_ram_mb: float | None = None # server /metrics RSS in MB, None = not available
|
||||
error: str = ""
|
||||
|
||||
|
||||
@ -126,6 +127,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
||||
|
||||
device_cache: dict = {}
|
||||
meta_cache: dict = {} # url → list[ModelMeta]
|
||||
server_ram_cache: dict = {} # url → float | None (MB from /metrics)
|
||||
|
||||
def _get_langs(e) -> list[str]:
|
||||
if e.is_local:
|
||||
@ -141,10 +143,21 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
||||
rows: list[BenchRow] = []
|
||||
for e in run_list:
|
||||
tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None
|
||||
# Snapshot server RAM before (if Prometheus metrics available)
|
||||
if not e.is_local and e.url not in server_ram_cache:
|
||||
server_ram_cache[e.url] = None # sentinel — probe once per URL
|
||||
srv_before: float | None = None
|
||||
if not e.is_local:
|
||||
srv_before = stt.probe_server_ram_mb(e.url)
|
||||
rss_before = _rss_mb()
|
||||
res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr)
|
||||
rss_after = _rss_mb()
|
||||
ram_delta = max(0.0, rss_after - rss_before)
|
||||
srv_ram: float | None = None
|
||||
if not e.is_local:
|
||||
srv_after = stt.probe_server_ram_mb(e.url)
|
||||
if srv_before is not None and srv_after is not None:
|
||||
srv_ram = srv_after # report current RSS, not delta (server may not unload)
|
||||
w = wer(reference, res.text, case_sensitive=case_sensitive) if res.ok else 1.0
|
||||
row = BenchRow(
|
||||
engine=e.name,
|
||||
@ -159,6 +172,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
||||
accuracy=max(0.0, 1.0 - w) * 100.0,
|
||||
text=res.text,
|
||||
ram_mb=ram_delta,
|
||||
srv_ram_mb=srv_ram,
|
||||
error=res.error,
|
||||
)
|
||||
rows.append(row)
|
||||
|
||||
@ -2104,8 +2104,10 @@ notebook.bt-nb tab:checked label {
|
||||
col.set_max_width(max_w)
|
||||
if i == 8:
|
||||
col.set_widget(Gtk.Label(label="RAM (MB)", tooltip_text=
|
||||
"RAM loaded by this engine during the benchmark run (local engines only).\n"
|
||||
"'server' = model runs in a container or remote server — memory is not visible here.\n"
|
||||
"RAM used by the engine during the benchmark.\n"
|
||||
"Local engines: RSS delta measured on this machine.\n"
|
||||
"Remote engines: current RSS read from the server's Prometheus /metrics endpoint.\n"
|
||||
"'server' = server does not expose /metrics — RAM not measurable.\n"
|
||||
"'—' = local model was already loaded, no delta to measure."))
|
||||
col.get_widget().show()
|
||||
tree.append_column(col)
|
||||
@ -2265,9 +2267,12 @@ notebook.bt-nb tab:checked label {
|
||||
if row.ram_mb >= 1.0:
|
||||
ram_display = f"{row.ram_mb:.0f}"
|
||||
elif row.url:
|
||||
ram_display = "server" # model runs remotely — RAM not measurable here
|
||||
if row.srv_ram_mb is not None:
|
||||
ram_display = f"{row.srv_ram_mb:.0f}" # from server /metrics
|
||||
else:
|
||||
ram_display = "server" # metrics not exposed
|
||||
else:
|
||||
ram_display = "—" # local model already loaded, no delta
|
||||
ram_display = "—" # local model already loaded, no delta
|
||||
self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for,
|
||||
lang_display, f"{row.seconds:.2f}", acc, ram_display, out_friendly, tooltip])
|
||||
# Persist result so Engines tab can show it
|
||||
|
||||
@ -148,6 +148,42 @@ def list_models_meta(base_url: str, api_key_env: str = "", timeout: float = 5.0)
|
||||
return []
|
||||
|
||||
|
||||
def probe_server_ram_mb(base_url: str, timeout: float = 3.0) -> float | None:
|
||||
"""Probe a remote server's Prometheus /metrics endpoint for RSS memory.
|
||||
|
||||
Returns RSS in MB if the server exposes Prometheus metrics with either
|
||||
``process_resident_memory_bytes`` (standard Python/Go exporter) or
|
||||
``container_memory_rss`` (cAdvisor). Returns None if not available.
|
||||
"""
|
||||
if not base_url:
|
||||
return None
|
||||
base = _api_base(base_url)
|
||||
try:
|
||||
req = urllib.request.Request(base + "/metrics")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
text = resp.read().decode("utf-8", "replace")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Parse Prometheus text format — we only need two specific metric names.
|
||||
# Lines look like: process_resident_memory_bytes 1.23456e+08
|
||||
best: float | None = None
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
for metric in ("process_resident_memory_bytes", "container_memory_rss"):
|
||||
if line.startswith(metric):
|
||||
parts = line.split()
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
best = float(parts[-1]) / (1024 * 1024) # bytes → MB
|
||||
return best
|
||||
except ValueError:
|
||||
pass
|
||||
return best
|
||||
|
||||
|
||||
def detect_remote_device(base_url: str, timeout: float = 3.0) -> str:
|
||||
"""Best-effort GPU/CPU detection for a remote STT server.
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user