feat: server RAM via Prometheus /metrics in benchmark (v2.03.09)
Probe remote engines' /metrics for process_resident_memory_bytes or container_memory_rss; show actual server-side MB in RAM column. Falls back to "server" when the endpoint is not exposed. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
86632e1918
commit
c3cf123afe
@ -9,6 +9,15 @@ The version is defined in [`blitztext/__init__.py`](blitztext/__init__.py).
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [2.03.09] - 2026-06-09
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- **Server RAM in benchmark.** For remote/Docker STT engines the benchmark now
|
||||||
|
probes the server's Prometheus `/metrics` endpoint for
|
||||||
|
`process_resident_memory_bytes` (standard Python/Go exporter) or
|
||||||
|
`container_memory_rss` (cAdvisor) and shows the server-side RSS in MB in the
|
||||||
|
RAM column. Falls back to `server` when the endpoint is not exposed.
|
||||||
|
|
||||||
## [2.03.08] - 2026-06-09
|
## [2.03.08] - 2026-06-09
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|||||||
@ -6,4 +6,4 @@ counterpart to the macOS Blitztext menu bar app: it runs natively on the host
|
|||||||
(not in a container) so it can type into any application via xdotool.
|
(not in a container) so it can type into any application via xdotool.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__version__ = "2.03.08"
|
__version__ = "2.03.09"
|
||||||
|
|||||||
@ -41,7 +41,8 @@ class BenchRow:
|
|||||||
wer: float
|
wer: float
|
||||||
accuracy: float # percent, max(0, 1-wer)*100
|
accuracy: float # percent, max(0, 1-wer)*100
|
||||||
text: str
|
text: str
|
||||||
ram_mb: float = 0.0 # RSS delta in MB; >0 means model loaded during this run
|
ram_mb: float = 0.0 # local RSS delta in MB
|
||||||
|
srv_ram_mb: float | None = None # server /metrics RSS in MB, None = not available
|
||||||
error: str = ""
|
error: str = ""
|
||||||
|
|
||||||
|
|
||||||
@ -126,6 +127,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
|||||||
|
|
||||||
device_cache: dict = {}
|
device_cache: dict = {}
|
||||||
meta_cache: dict = {} # url → list[ModelMeta]
|
meta_cache: dict = {} # url → list[ModelMeta]
|
||||||
|
server_ram_cache: dict = {} # url → float | None (MB from /metrics)
|
||||||
|
|
||||||
def _get_langs(e) -> list[str]:
|
def _get_langs(e) -> list[str]:
|
||||||
if e.is_local:
|
if e.is_local:
|
||||||
@ -141,10 +143,21 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
|||||||
rows: list[BenchRow] = []
|
rows: list[BenchRow] = []
|
||||||
for e in run_list:
|
for e in run_list:
|
||||||
tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None
|
tr = get_local_transcriber(e) if (e.is_local and get_local_transcriber) else None
|
||||||
|
# Snapshot server RAM before (if Prometheus metrics available)
|
||||||
|
if not e.is_local and e.url not in server_ram_cache:
|
||||||
|
server_ram_cache[e.url] = None # sentinel — probe once per URL
|
||||||
|
srv_before: float | None = None
|
||||||
|
if not e.is_local:
|
||||||
|
srv_before = stt.probe_server_ram_mb(e.url)
|
||||||
rss_before = _rss_mb()
|
rss_before = _rss_mb()
|
||||||
res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr)
|
res = stt.benchmark(e, wav_path, language=language, local_transcriber=tr)
|
||||||
rss_after = _rss_mb()
|
rss_after = _rss_mb()
|
||||||
ram_delta = max(0.0, rss_after - rss_before)
|
ram_delta = max(0.0, rss_after - rss_before)
|
||||||
|
srv_ram: float | None = None
|
||||||
|
if not e.is_local:
|
||||||
|
srv_after = stt.probe_server_ram_mb(e.url)
|
||||||
|
if srv_before is not None and srv_after is not None:
|
||||||
|
srv_ram = srv_after # report current RSS, not delta (server may not unload)
|
||||||
w = wer(reference, res.text, case_sensitive=case_sensitive) if res.ok else 1.0
|
w = wer(reference, res.text, case_sensitive=case_sensitive) if res.ok else 1.0
|
||||||
row = BenchRow(
|
row = BenchRow(
|
||||||
engine=e.name,
|
engine=e.name,
|
||||||
@ -159,6 +172,7 @@ def run(engines, wav_path: Path, reference: str, *, language: str = "",
|
|||||||
accuracy=max(0.0, 1.0 - w) * 100.0,
|
accuracy=max(0.0, 1.0 - w) * 100.0,
|
||||||
text=res.text,
|
text=res.text,
|
||||||
ram_mb=ram_delta,
|
ram_mb=ram_delta,
|
||||||
|
srv_ram_mb=srv_ram,
|
||||||
error=res.error,
|
error=res.error,
|
||||||
)
|
)
|
||||||
rows.append(row)
|
rows.append(row)
|
||||||
|
|||||||
@ -2104,8 +2104,10 @@ notebook.bt-nb tab:checked label {
|
|||||||
col.set_max_width(max_w)
|
col.set_max_width(max_w)
|
||||||
if i == 8:
|
if i == 8:
|
||||||
col.set_widget(Gtk.Label(label="RAM (MB)", tooltip_text=
|
col.set_widget(Gtk.Label(label="RAM (MB)", tooltip_text=
|
||||||
"RAM loaded by this engine during the benchmark run (local engines only).\n"
|
"RAM used by the engine during the benchmark.\n"
|
||||||
"'server' = model runs in a container or remote server — memory is not visible here.\n"
|
"Local engines: RSS delta measured on this machine.\n"
|
||||||
|
"Remote engines: current RSS read from the server's Prometheus /metrics endpoint.\n"
|
||||||
|
"'server' = server does not expose /metrics — RAM not measurable.\n"
|
||||||
"'—' = local model was already loaded, no delta to measure."))
|
"'—' = local model was already loaded, no delta to measure."))
|
||||||
col.get_widget().show()
|
col.get_widget().show()
|
||||||
tree.append_column(col)
|
tree.append_column(col)
|
||||||
@ -2265,7 +2267,10 @@ notebook.bt-nb tab:checked label {
|
|||||||
if row.ram_mb >= 1.0:
|
if row.ram_mb >= 1.0:
|
||||||
ram_display = f"{row.ram_mb:.0f}"
|
ram_display = f"{row.ram_mb:.0f}"
|
||||||
elif row.url:
|
elif row.url:
|
||||||
ram_display = "server" # model runs remotely — RAM not measurable here
|
if row.srv_ram_mb is not None:
|
||||||
|
ram_display = f"{row.srv_ram_mb:.0f}" # from server /metrics
|
||||||
|
else:
|
||||||
|
ram_display = "server" # metrics not exposed
|
||||||
else:
|
else:
|
||||||
ram_display = "—" # local model already loaded, no delta
|
ram_display = "—" # local model already loaded, no delta
|
||||||
self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for,
|
self.bench_store.append([row.engine, url_display, row.model, row.device, row.best_for,
|
||||||
|
|||||||
@ -148,6 +148,42 @@ def list_models_meta(base_url: str, api_key_env: str = "", timeout: float = 5.0)
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def probe_server_ram_mb(base_url: str, timeout: float = 3.0) -> float | None:
|
||||||
|
"""Probe a remote server's Prometheus /metrics endpoint for RSS memory.
|
||||||
|
|
||||||
|
Returns RSS in MB if the server exposes Prometheus metrics with either
|
||||||
|
``process_resident_memory_bytes`` (standard Python/Go exporter) or
|
||||||
|
``container_memory_rss`` (cAdvisor). Returns None if not available.
|
||||||
|
"""
|
||||||
|
if not base_url:
|
||||||
|
return None
|
||||||
|
base = _api_base(base_url)
|
||||||
|
try:
|
||||||
|
req = urllib.request.Request(base + "/metrics")
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
text = resp.read().decode("utf-8", "replace")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Parse Prometheus text format — we only need two specific metric names.
|
||||||
|
# Lines look like: process_resident_memory_bytes 1.23456e+08
|
||||||
|
best: float | None = None
|
||||||
|
for line in text.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if line.startswith("#"):
|
||||||
|
continue
|
||||||
|
for metric in ("process_resident_memory_bytes", "container_memory_rss"):
|
||||||
|
if line.startswith(metric):
|
||||||
|
parts = line.split()
|
||||||
|
if len(parts) >= 2:
|
||||||
|
try:
|
||||||
|
best = float(parts[-1]) / (1024 * 1024) # bytes → MB
|
||||||
|
return best
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
return best
|
||||||
|
|
||||||
|
|
||||||
def detect_remote_device(base_url: str, timeout: float = 3.0) -> str:
|
def detect_remote_device(base_url: str, timeout: float = 3.0) -> str:
|
||||||
"""Best-effort GPU/CPU detection for a remote STT server.
|
"""Best-effort GPU/CPU detection for a remote STT server.
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user